From 25ca7c6fcd5efa94768be5fe8841a4225c623ca8 Mon Sep 17 00:00:00 2001 From: Danielle Smith Date: Wed, 25 Dec 2024 11:46:00 +0200 Subject: [PATCH] freeze stringy token values --- lib/kdl/node.rb | 2 +- lib/kdl/tokenizer.rb | 26 +++++++++++++------------- lib/kdl/v1/tokenizer.rb | 18 +++++++++--------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/kdl/node.rb b/lib/kdl/node.rb index dbb1abf..a5e25d3 100644 --- a/lib/kdl/node.rb +++ b/lib/kdl/node.rb @@ -92,7 +92,7 @@ def <=>(other) def to_s(level = 0, m = :to_s) indent = ' ' * level - s = +"#{indent}#{type ? "(#{id_to_s type, m })" : ''}#{id_to_s name, m}" + s = "#{indent}#{type ? "(#{id_to_s type, m })" : ''}#{id_to_s name, m}" unless arguments.empty? s << " #{arguments.map(&m).join(' ')}" end diff --git a/lib/kdl/tokenizer.rb b/lib/kdl/tokenizer.rb index ce037b3..85d422a 100644 --- a/lib/kdl/tokenizer.rb +++ b/lib/kdl/tokenizer.rb @@ -203,7 +203,7 @@ def next_token la = t.next_token if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF) traverse_to(t.index) - @buffer = +"#{c}#{la[1].value}" + @buffer = "#{c}#{la[1].value}" @buffer << "\n" if lan == :NEWLINE self.context = :whitespace else @@ -214,10 +214,10 @@ def next_token @buffer = +c traverse(1) when *SYMBOLS.keys - return token(SYMBOLS[c], c).tap { traverse(1) } + return token(SYMBOLS[c], -c).tap { traverse(1) } when *NEWLINES, "\r" nl = expect_newline - return token(:NEWLINE, nl).tap do + return token(:NEWLINE, -nl).tap do traverse(nl.length) end when "/" @@ -250,10 +250,10 @@ def next_token traverse(1) when '(' @type_context = true - return token(:LPAREN, c).tap { traverse(1) } + return token(:LPAREN, -c).tap { traverse(1) } when ')' @type_context = false - return token(:RPAREN, c).tap { traverse(1) } + return token(:RPAREN, -c).tap { traverse(1) } else raise_error "Unexpected character #{c.inspect}" end @@ -269,7 +269,7 @@ def next_token when /\A\.\d/ raise_error "Identifier cannot look like an illegal float" else - return token(:IDENT, @buffer) + return token(:IDENT, -@buffer) end end when :keyword @@ -305,7 +305,7 @@ def next_token traverse(2) end when '"' - return token(:STRING, unescape(@buffer)).tap { traverse(1) } + return token(:STRING, -unescape(@buffer)).tap { traverse(1) } when *NEWLINES, "\r" raise_error "Unexpected NEWLINE in string literal" when nil @@ -322,7 +322,7 @@ def next_token traverse(2) when '"' if self[@index + 1] == '"' && self[@index + 2] == '"' - return token(:STRING, unescape_non_ws(dedent(unescape_ws(@buffer)))).tap { traverse(3) } + return token(:STRING, -unescape_non_ws(dedent(unescape_ws(@buffer)))).tap { traverse(3) } end @buffer << c traverse(1) @@ -340,7 +340,7 @@ def next_token h = 0 h += 1 while self[@index + 1 + h] == '#' && h < @rawstring_hashes if h == @rawstring_hashes - return token(:RAWSTRING, @buffer).tap { traverse(1 + h) } + return token(:RAWSTRING, -@buffer).tap { traverse(1 + h) } end when *NEWLINES, "\r" raise_error "Unexpected NEWLINE in rawstring literal" @@ -355,7 +355,7 @@ def next_token h = 1 h += 1 while self[@index + 3 + h] == '#' && h < @rawstring_hashes if h == @rawstring_hashes - return token(:RAWSTRING, dedent(@buffer)).tap { traverse(3 + h) } + return token(:RAWSTRING, -dedent(@buffer)).tap { traverse(3 + h) } end end @@ -441,7 +441,7 @@ def next_token raise_error "Unexpected '\\' (#{la[0]})" end else - return token(:WS, @buffer) + return token(:WS, -@buffer) end when :equals t = Tokenizer.new(@str, @index) @@ -450,7 +450,7 @@ def next_token @buffer << la[1].value traverse_to(t.index) end - return token(:EQUALS, @buffer) + return token(:EQUALS, -@buffer) else # :nocov: raise_error "Unknown context `#{@context}'" @@ -542,7 +542,7 @@ def parse_decimal(s) token(:INTEGER, Integer(munch_underscores(s), 10), format: '%d') rescue if s[0] =~ INITIAL_IDENTIFIER_CHARS && s[1..-1].each_char.all? { |c| c =~ IDENTIFIER_CHARS } - token(:IDENT, s) + token(:IDENT, -s) else raise end diff --git a/lib/kdl/v1/tokenizer.rb b/lib/kdl/v1/tokenizer.rb index 4859462..6c88445 100644 --- a/lib/kdl/v1/tokenizer.rb +++ b/lib/kdl/v1/tokenizer.rb @@ -77,17 +77,17 @@ def next_token la = t.next_token if la[0] == :NEWLINE || la[0] == :EOF || (la[0] == :WS && (lan = t.next_token[0]) == :NEWLINE || lan == :EOF) traverse_to(t.index) - @buffer = +"#{c}#{la[1].value}" + @buffer = "#{c}#{la[1].value}" @buffer << "\n" if lan == :NEWLINE self.context = :whitespace else raise_error "Unexpected '\\' (#{la[0]})" end when *SYMBOLS.keys - return token(SYMBOLS[c], c).tap { traverse(1) } + return token(SYMBOLS[c], -c).tap { traverse(1) } when *NEWLINES, "\r" nl = expect_newline - return token(:NEWLINE, nl).tap do + return token(:NEWLINE, -nl).tap do traverse(nl.length) end when "/" @@ -120,10 +120,10 @@ def next_token traverse(1) when '(' @type_context = true - return token(:LPAREN, c).tap { traverse(1) } + return token(:LPAREN, -c).tap { traverse(1) } when ')' @type_context = false - return token(:RPAREN, c).tap { traverse(1) } + return token(:RPAREN, -c).tap { traverse(1) } else raise_error "Unexpected character #{c.inspect}" end @@ -137,7 +137,7 @@ def next_token when 'true' then return token(:TRUE, true) when 'false' then return token(:FALSE, false) when 'null' then return token(:NULL, nil) - else return token(:IDENT, @buffer) + else return token(:IDENT, -@buffer) end end when :string @@ -156,7 +156,7 @@ def next_token traverse(2) end when '"' - return token(:STRING, unescape(@buffer)).tap { traverse(1) } + return token(:STRING, -unescape(@buffer)).tap { traverse(1) } when nil raise_error "Unterminated string literal" else @@ -170,7 +170,7 @@ def next_token h = 0 h += 1 while self[@index + 1 + h] == '#' && h < @rawstring_hashes if h == @rawstring_hashes - return token(:RAWSTRING, @buffer).tap { traverse(1 + h) } + return token(:RAWSTRING, -@buffer).tap { traverse(1 + h) } end end @@ -252,7 +252,7 @@ def next_token raise_error "Unexpected '\\' (#{la[0]})" end else - return token(:WS, @buffer) + return token(:WS, -@buffer) end else # :nocov: