Skip to content

Commit

Permalink
freeze stringy token values
Browse files Browse the repository at this point in the history
  • Loading branch information
@danini-the-panini
danini-the-panini committed Dec 25, 2024
1 parent 62d1e63 commit 25ca7c6
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 23 deletions.
2 changes: 1 addition & 1 deletion lib/kdl/node.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def <=>(other)

def to_s(level = 0, m = :to_s)
indent = ' ' * level
s = +"#{indent}#{type ? "(#{id_to_s type, m })" : ''}#{id_to_s name, m}"
s = "#{indent}#{type ? "(#{id_to_s type, m })" : ''}#{id_to_s name, m}"
unless arguments.empty?
s << " #{arguments.map(&m).join(' ')}"
end
Expand Down
26 changes: 13 additions & 13 deletions lib/kdl/tokenizer.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ def next_token
la = t.next_token
if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF)
traverse_to(t.index)
@buffer = +"#{c}#{la[1].value}"
@buffer = "#{c}#{la[1].value}"
@buffer << "\n" if lan == :NEWLINE
self.context = :whitespace
else
Expand All @@ -214,10 +214,10 @@ def next_token
@buffer = +c
traverse(1)
when *SYMBOLS.keys
return token(SYMBOLS[c], c).tap { traverse(1) }
return token(SYMBOLS[c], -c).tap { traverse(1) }
when *NEWLINES, "\r"
nl = expect_newline
return token(:NEWLINE, nl).tap do
return token(:NEWLINE, -nl).tap do
traverse(nl.length)
end
when "/"
Expand Down Expand Up @@ -250,10 +250,10 @@ def next_token
traverse(1)
when '('
@type_context = true
return token(:LPAREN, c).tap { traverse(1) }
return token(:LPAREN, -c).tap { traverse(1) }
when ')'
@type_context = false
return token(:RPAREN, c).tap { traverse(1) }
return token(:RPAREN, -c).tap { traverse(1) }
else
raise_error "Unexpected character #{c.inspect}"
end
Expand All @@ -269,7 +269,7 @@ def next_token
when /\A\.\d/
raise_error "Identifier cannot look like an illegal float"
else
return token(:IDENT, @buffer)
return token(:IDENT, -@buffer)
end
end
when :keyword
Expand Down Expand Up @@ -305,7 +305,7 @@ def next_token
traverse(2)
end
when '"'
return token(:STRING, unescape(@buffer)).tap { traverse(1) }
return token(:STRING, -unescape(@buffer)).tap { traverse(1) }
when *NEWLINES, "\r"
raise_error "Unexpected NEWLINE in string literal"
when nil
Expand All @@ -322,7 +322,7 @@ def next_token
traverse(2)
when '"'
if self[@index + 1] == '"' && self[@index + 2] == '"'
return token(:STRING, unescape_non_ws(dedent(unescape_ws(@buffer)))).tap { traverse(3) }
return token(:STRING, -unescape_non_ws(dedent(unescape_ws(@buffer)))).tap { traverse(3) }
end
@buffer << c
traverse(1)
Expand All @@ -340,7 +340,7 @@ def next_token
h = 0
h += 1 while self[@index + 1 + h] == '#' && h < @rawstring_hashes
if h == @rawstring_hashes
return token(:RAWSTRING, @buffer).tap { traverse(1 + h) }
return token(:RAWSTRING, -@buffer).tap { traverse(1 + h) }
end
when *NEWLINES, "\r"
raise_error "Unexpected NEWLINE in rawstring literal"
Expand All @@ -355,7 +355,7 @@ def next_token
h = 1
h += 1 while self[@index + 3 + h] == '#' && h < @rawstring_hashes
if h == @rawstring_hashes
return token(:RAWSTRING, dedent(@buffer)).tap { traverse(3 + h) }
return token(:RAWSTRING, -dedent(@buffer)).tap { traverse(3 + h) }
end
end

Expand Down Expand Up @@ -441,7 +441,7 @@ def next_token
raise_error "Unexpected '\\' (#{la[0]})"
end
else
return token(:WS, @buffer)
return token(:WS, -@buffer)
end
when :equals
t = Tokenizer.new(@str, @index)
Expand All @@ -450,7 +450,7 @@ def next_token
@buffer << la[1].value
traverse_to(t.index)
end
return token(:EQUALS, @buffer)
return token(:EQUALS, -@buffer)
else
# :nocov:
raise_error "Unknown context `#{@context}'"
Expand Down Expand Up @@ -542,7 +542,7 @@ def parse_decimal(s)
token(:INTEGER, Integer(munch_underscores(s), 10), format: '%d')
rescue
if s[0] =~ INITIAL_IDENTIFIER_CHARS && s[1..-1].each_char.all? { |c| c =~ IDENTIFIER_CHARS }
token(:IDENT, s)
token(:IDENT, -s)
else
raise
end
Expand Down
18 changes: 9 additions & 9 deletions lib/kdl/v1/tokenizer.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,17 +77,17 @@ def next_token
la = t.next_token
if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF)
traverse_to(t.index)
@buffer = +"#{c}#{la[1].value}"
@buffer = "#{c}#{la[1].value}"
@buffer << "\n" if lan == :NEWLINE
self.context = :whitespace
else
raise_error "Unexpected '\\' (#{la[0]})"
end
when *SYMBOLS.keys
return token(SYMBOLS[c], c).tap { traverse(1) }
return token(SYMBOLS[c], -c).tap { traverse(1) }
when *NEWLINES, "\r"
nl = expect_newline
return token(:NEWLINE, nl).tap do
return token(:NEWLINE, -nl).tap do
traverse(nl.length)
end
when "/"
Expand Down Expand Up @@ -120,10 +120,10 @@ def next_token
traverse(1)
when '('
@type_context = true
return token(:LPAREN, c).tap { traverse(1) }
return token(:LPAREN, -c).tap { traverse(1) }
when ')'
@type_context = false
return token(:RPAREN, c).tap { traverse(1) }
return token(:RPAREN, -c).tap { traverse(1) }
else
raise_error "Unexpected character #{c.inspect}"
end
Expand All @@ -137,7 +137,7 @@ def next_token
when 'true' then return token(:TRUE, true)
when 'false' then return token(:FALSE, false)
when 'null' then return token(:NULL, nil)
else return token(:IDENT, @buffer)
else return token(:IDENT, -@buffer)
end
end
when :string
Expand All @@ -156,7 +156,7 @@ def next_token
traverse(2)
end
when '"'
return token(:STRING, unescape(@buffer)).tap { traverse(1) }
return token(:STRING, -unescape(@buffer)).tap { traverse(1) }
when nil
raise_error "Unterminated string literal"
else
Expand All @@ -170,7 +170,7 @@ def next_token
h = 0
h += 1 while self[@index + 1 + h] == '#' && h < @rawstring_hashes
if h == @rawstring_hashes
return token(:RAWSTRING, @buffer).tap { traverse(1 + h) }
return token(:RAWSTRING, -@buffer).tap { traverse(1 + h) }
end
end

Expand Down Expand Up @@ -252,7 +252,7 @@ def next_token
raise_error "Unexpected '\\' (#{la[0]})"
end
else
return token(:WS, @buffer)
return token(:WS, -@buffer)
end
else
# :nocov:
Expand Down

0 comments on commit 25ca7c6

Please sign in to comment.