From 0af8bbe7b798d138732891d6be5d5a3b2ff506d3 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Wed, 27 Jun 2018 21:06:59 -0300 Subject: [PATCH 1/2] Fix incorrect parsing of `%r(...)` inside macros --- spec/compiler/parser/parser_spec.cr | 5 +++++ src/compiler/crystal/syntax/lexer.cr | 3 +++ 2 files changed, 8 insertions(+) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index c268ab82577f..ff03cc0ed890 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -1380,6 +1380,11 @@ module Crystal it_parses "foo.Bar", Call.new("foo".call, "Bar") + it_parses "{% begin %}%r(\\A){% end %}", MacroIf.new(true.bool, MacroLiteral.new("%r(\\A)")) + it_parses "{% begin %}%r[\\A]{% end %}", MacroIf.new(true.bool, MacroLiteral.new("%r[\\A]")) + it_parses "{% begin %}%r<\\A>{% end %}", MacroIf.new(true.bool, MacroLiteral.new("%r<\\A>")) + it_parses "{% begin %}%r{\\A}{% end %}", MacroIf.new(true.bool, Expressions.new([MacroLiteral.new("%r"), MacroLiteral.new("{\\A}")] of ASTNode)) + assert_syntax_error "return do\nend", "unexpected token: do" %w(def macro class struct module fun alias abstract include extend lib).each do |keyword| diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index 40e77ea0067d..7e102049ecb8 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -2130,6 +2130,9 @@ module Crystal if char == 'q' && (peek = peek_next_char) && {'(', '<', '[', '{'}.includes?(peek) next_char delimiter_state = Token::DelimiterState.new(:string, char, closing_char, 1) + elsif char == 'r' && (peek = peek_next_char) && {'(', '<', '[', '{'}.includes?(peek) + next_char + delimiter_state = Token::DelimiterState.new(:regex, char, closing_char, 1) else start = current_pos while ident_part?(char) From af546f0d18d11d8b7a8ef5278c2792ccd53fbc5a Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Wed, 27 Jun 2018 21:28:09 -0300 Subject: [PATCH 2/2] Fix more parsing issues related to `%{...}` and variants inside macros --- spec/compiler/parser/parser_spec.cr | 9 +++++---- src/compiler/crystal/syntax/lexer.cr | 8 +++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index ff03cc0ed890..2caba5fd3de6 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -1380,10 +1380,11 @@ module Crystal it_parses "foo.Bar", Call.new("foo".call, "Bar") - it_parses "{% begin %}%r(\\A){% end %}", MacroIf.new(true.bool, MacroLiteral.new("%r(\\A)")) - it_parses "{% begin %}%r[\\A]{% end %}", MacroIf.new(true.bool, MacroLiteral.new("%r[\\A]")) - it_parses "{% begin %}%r<\\A>{% end %}", MacroIf.new(true.bool, MacroLiteral.new("%r<\\A>")) - it_parses "{% begin %}%r{\\A}{% end %}", MacroIf.new(true.bool, Expressions.new([MacroLiteral.new("%r"), MacroLiteral.new("{\\A}")] of ASTNode)) + [{'(', ')'}, {'[', ']'}, {'<', '>'}, {'{', '}'}, {'|', '|'}].each do |open, close| + it_parses "{% begin %}%r#{open}\\A#{close}{% end %}", MacroIf.new(true.bool, MacroLiteral.new("%r#{open}\\A#{close}")) + it_parses "{% begin %}%#{open} %s #{close}{% end %}", MacroIf.new(true.bool, MacroLiteral.new("%#{open} %s #{close}")) + it_parses "{% begin %}%q#{open} %s #{close}{% end %}", MacroIf.new(true.bool, MacroLiteral.new("%q#{open} %s #{close}")) + end assert_syntax_error "return do\nend", "unexpected token: do" diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index 7e102049ecb8..ad3d839d3238 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -2127,12 +2127,14 @@ module Crystal if !delimiter_state && current_char == '%' && ident_start?(peek_next_char) char = next_char - if char == 'q' && (peek = peek_next_char) && {'(', '<', '[', '{'}.includes?(peek) + if char == 'q' && (peek = peek_next_char) && {'(', '<', '[', '{', '|'}.includes?(peek) next_char delimiter_state = Token::DelimiterState.new(:string, char, closing_char, 1) - elsif char == 'r' && (peek = peek_next_char) && {'(', '<', '[', '{'}.includes?(peek) + next_char + elsif char == 'r' && (peek = peek_next_char) && {'(', '<', '[', '{', '|'}.includes?(peek) next_char delimiter_state = Token::DelimiterState.new(:regex, char, closing_char, 1) + next_char else start = current_pos while ident_part?(char) @@ -2199,7 +2201,7 @@ module Crystal whitespace = false when '%' case char = peek_next_char - when '(', '[', '<', '{' + when '(', '[', '<', '{', '|' next_char delimiter_state = Token::DelimiterState.new(:string, char, closing_char, 1) else