Compiler: add locations to all expanded macro arguments

spec/compiler/macro/macro_expander_spec.cr

@@ -156,4 +156,18 @@ describe "MacroExpander" do
   it "can't use `yield` outside a macro" do
     assert_error %({{yield}}), "can't use `{{yield}}` outside a macro"
   end
+
+  it "outputs invisible location pragmas" do
+    node = 42.int32
+    node.location = Location.new "foo.cr", 10, 20
+    assert_macro "node", %({{node}}), [node] of ASTNode, "42", {
+      0 => [
+        Lexer::LocPushPragma.new,
+        Lexer::LocSetPragma.new("foo.cr", 10, 20),
+      ] of Lexer::LocPragma,
+      2 => [
+        Lexer::LocPopPragma.new,
+      ] of Lexer::LocPragma,
+    }
+  end
 end

spec/spec_helper.cr

@@ -108,25 +108,26 @@ def assert_error(str, message, inject_primitives = true)
   end
 end
 
-def assert_macro(macro_args, macro_body, call_args, expected, flags = nil)
-  assert_macro(macro_args, macro_body, expected, flags) { call_args }
+def assert_macro(macro_args, macro_body, call_args, expected, expected_pragmas = nil, flags = nil)
+  assert_macro(macro_args, macro_body, expected, expected_pragmas, flags) { call_args }
 end
 
-def assert_macro(macro_args, macro_body, expected, flags = nil)
+def assert_macro(macro_args, macro_body, expected, expected_pragmas = nil, flags = nil)
   program = Program.new
   program.flags = flags if flags
   sub_node = yield program
-  assert_macro_internal program, sub_node, macro_args, macro_body, expected
+  assert_macro_internal program, sub_node, macro_args, macro_body, expected, expected_pragmas
 end
 
-def assert_macro_internal(program, sub_node, macro_args, macro_body, expected)
+def assert_macro_internal(program, sub_node, macro_args, macro_body, expected, expected_pragmas)
   macro_def = "macro foo(#{macro_args});#{macro_body};end"
   a_macro = Parser.parse(macro_def).as(Macro)
 
   call = Call.new(nil, "", sub_node)
-  result = program.expand_macro a_macro, call, program, program
+  result, result_pragmas = program.expand_macro a_macro, call, program, program
   result = result.chomp(';')
   result.should eq(expected)
+  result_pragmas.should eq(expected_pragmas) if expected_pragmas
 end
 
 def codegen(code, inject_primitives = true, debug = Crystal::Debug::None)

src/compiler/crystal/macros/interpreter.cr

@@ -2,6 +2,7 @@ module Crystal
   class MacroInterpreter < Visitor
     getter last : ASTNode
     property free_vars : Hash(String, TypeVar)?
+    property macro_expansion_pragmas : Hash(Int32, Array(Lexer::LocPragma))? = nil
 
     def self.new(program, scope : Type, path_lookup : Type, a_macro : Macro, call, a_def : Def? = nil, in_macro = false)
       vars = {} of String => ASTNode
@@ -99,13 +100,17 @@ module Crystal
       node.exp.accept self
 
       if node.output?
-        # In the caseof {{yield}}, we want to paste the block's body
-        # retaining the original node's location, so error messages
-        # are shown in the block instead of in the generated macro source
         is_yield = node.exp.is_a?(Yield) && [email protected]_a?(Nop)
-        @str << " #<loc:push>begin " if is_yield
-        @last.to_s(@str, emit_loc_pragma: is_yield, emit_doc: is_yield)
-        @str << " end#<loc:pop> " if is_yield
+        if (loc = @last.location) && loc.filename.is_a?(String) || is_yield
+          macro_expansion_pragmas = @macro_expansion_pragmas ||= {} of Int32 => Array(Lexer::LocPragma)
+          (macro_expansion_pragmas[@str.pos.to_i32] ||= [] of Lexer::LocPragma) << Lexer::LocPushPragma.new
+          @str << "begin " if is_yield
+          @last.to_s(@str, macro_expansion_pragmas: macro_expansion_pragmas, emit_doc: true)
+          @str << " end" if is_yield
+          (macro_expansion_pragmas[@str.pos.to_i32] ||= [] of Lexer::LocPragma) << Lexer::LocPopPragma.new
+        else
+          @last.to_s(@str)
+        end
       end
 
       false
@@ -161,7 +166,7 @@ module Crystal
             @last.to_s(str)
           end
         end
-      end).at(node)
+      end)
       false
     end
 
@@ -512,33 +517,33 @@ module Crystal
     end
 
     def visit(node : TupleLiteral)
-      @last = TupleLiteral.map(node.elements) { |element| accept element }.at(node)
+      @last = TupleLiteral.map(node.elements) { |element| accept element }
       false
     end
 
     def visit(node : ArrayLiteral)
-      @last = ArrayLiteral.map(node.elements) { |element| accept element }.at(node)
+      @last = ArrayLiteral.map(node.elements) { |element| accept element }
       false
     end
 
     def visit(node : HashLiteral)
       @last =
         HashLiteral.new(node.entries.map do |entry|
           HashLiteral::Entry.new(accept(entry.key), accept(entry.value))
-        end).at(node)
+        end)
       false
     end
 
     def visit(node : NamedTupleLiteral)
       @last =
         NamedTupleLiteral.new(node.entries.map do |entry|
           NamedTupleLiteral::Entry.new(entry.key, accept(entry.value))
-        end).at(node)
+        end)
       false
     end
 
     def visit(node : Nop | NilLiteral | BoolLiteral | NumberLiteral | CharLiteral | StringLiteral | SymbolLiteral | RangeLiteral | RegexLiteral | MacroId | TypeNode | Def)
-      @last = node
+      @last = node.clone_without_location
       false
     end
 

src/compiler/crystal/macros/macros.cr

@@ -39,18 +39,18 @@ class Crystal::Program
   def expand_macro(a_macro : Macro, call : Call, scope : Type, path_lookup : Type? = nil, a_def : Def? = nil)
     interpreter = MacroInterpreter.new self, scope, path_lookup || scope, a_macro, call, a_def, in_macro: true
     a_macro.body.accept interpreter
-    interpreter.to_s
+    {interpreter.to_s, interpreter.macro_expansion_pragmas}
   end
 
   def expand_macro(node : ASTNode, scope : Type, path_lookup : Type? = nil, free_vars = nil, a_def : Def? = nil)
     interpreter = MacroInterpreter.new self, scope, path_lookup || scope, node.location, def: a_def, in_macro: false
     interpreter.free_vars = free_vars
     node.accept interpreter
-    interpreter.to_s
+    {interpreter.to_s, interpreter.macro_expansion_pragmas}
   end
 
-  def parse_macro_source(generated_source, the_macro, node, vars, current_def = nil, inside_type = false, inside_exp = false, mode : MacroExpansionMode = MacroExpansionMode::Normal)
-    parse_macro_source(generated_source, the_macro, node, vars, current_def, inside_type, inside_exp) do |parser|
+  def parse_macro_source(generated_source, macro_expansion_pragmas, the_macro, node, vars, current_def = nil, inside_type = false, inside_exp = false, mode : MacroExpansionMode = MacroExpansionMode::Normal)
+    parse_macro_source(generated_source, macro_expansion_pragmas, the_macro, node, vars, current_def, inside_type, inside_exp) do |parser|
       case mode
       when .lib?
         parser.parse_lib_body
@@ -64,10 +64,11 @@ class Crystal::Program
     end
   end
 
-  def parse_macro_source(generated_source, the_macro, node, vars, current_def = nil, inside_type = false, inside_exp = false)
+  def parse_macro_source(generated_source, macro_expansion_pragmas, the_macro, node, vars, current_def = nil, inside_type = false, inside_exp = false)
     begin
       parser = Parser.new(generated_source, @program.string_pool, [vars.dup])
       parser.filename = VirtualFile.new(the_macro, generated_source, node.location)
+      parser.macro_expansion_pragmas = macro_expansion_pragmas
       parser.visibility = node.visibility
       parser.def_nest = 1 if current_def
       parser.type_nest = 1 if inside_type

src/compiler/crystal/macros/methods.cr

@@ -139,7 +139,7 @@ module Crystal
     end
 
     def interpret_skip_file(node)
-      raise SkipMacroException.new(@str.to_s)
+      raise SkipMacroException.new(@str.to_s, macro_expansion_pragmas)
     end
 
     def interpret_system(node)

src/compiler/crystal/semantic/exception.cr

@@ -19,7 +19,7 @@ module Crystal
       if location
         column_number = node.name_column_number
         name_size = node.name_size
-        if column_number == 0
+        if column_number == 0 || (end_location = node.end_location) && end_location.filename != location.filename
           name_size = 0
           column_number = location.column_number
         end
@@ -296,8 +296,9 @@ module Crystal
 
   class SkipMacroException < ::Exception
     getter expanded_before_skip : String
+    getter macro_expansion_pragmas : Hash(Int32, Array(Lexer::LocPragma))?
 
-    def initialize(@expanded_before_skip)
+    def initialize(@expanded_before_skip, @macro_expansion_pragmas)
       super()
     end
   end

src/compiler/crystal/semantic/method_missing.cr

@@ -65,14 +65,14 @@ module Crystal
         block: block_node.is_a?(Block) ? block_node : nil)
       fake_call = Call.new(nil, "method_missing", [call] of ASTNode)
 
-      expanded_macro = program.expand_macro method_missing, fake_call, self, self
+      expanded_macro, macro_expansion_pragmas = program.expand_macro method_missing, fake_call, self, self
 
       # Check if the expanded macro is a def. We do this
       # by just lexing the result and seeing if the first
       # token is `def`
       expands_to_def = starts_with_def?(expanded_macro)
       generated_nodes =
-        program.parse_macro_source(expanded_macro, method_missing, method_missing, args_nodes_names) do |parser|
+        program.parse_macro_source(expanded_macro, macro_expansion_pragmas, method_missing, method_missing, args_nodes_names) do |parser|
           if expands_to_def
             parser.parse
           else

src/compiler/crystal/semantic/semantic_visitor.cr

@@ -296,9 +296,9 @@ abstract class Crystal::SemanticVisitor < Crystal::Visitor
     generated_nodes = expand_macro(the_macro, node) do
       old_args = node.args
       node.args = args
-      expanded = @program.expand_macro the_macro, node, expansion_scope, expansion_scope, @untyped_def
+      expanded_macro, macro_expansion_pragmas = @program.expand_macro the_macro, node, expansion_scope, expansion_scope, @untyped_def
       node.args = old_args
-      expanded
+      {expanded_macro, macro_expansion_pragmas}
     end
     @exp_nest += 1
 
@@ -310,7 +310,7 @@ abstract class Crystal::SemanticVisitor < Crystal::Visitor
   end
 
   def expand_macro(the_macro, node, mode = nil)
-    expanded_macro =
+    expanded_macro, macro_expansion_pragmas =
       eval_macro(node) do
         yield
       end
@@ -323,7 +323,7 @@ abstract class Crystal::SemanticVisitor < Crystal::Visitor
                Program::MacroExpansionMode::Normal
              end
 
-    generated_nodes = @program.parse_macro_source(expanded_macro, the_macro, node, Set.new(@vars.keys),
+    generated_nodes = @program.parse_macro_source(expanded_macro, macro_expansion_pragmas, the_macro, node, Set.new(@vars.keys),
       current_def: @typed_def,
       inside_type: !current_type.is_a?(Program),
       inside_exp: @exp_nest > 0,
@@ -400,7 +400,7 @@ abstract class Crystal::SemanticVisitor < Crystal::Visitor
         @program.expand_macro node, (@scope || current_type), @path_lookup, free_vars, @untyped_def
       rescue ex : SkipMacroException
         skip_macro_exception = ex
-        ex.expanded_before_skip
+        {ex.expanded_before_skip, ex.macro_expansion_pragmas}
       end
     end
 

src/compiler/crystal/syntax/lexer.cr

@@ -26,6 +26,31 @@ module Crystal
     # Heredocs pushed when found. Should be processed when encountering a newline
     getter heredocs = [] of {Token::DelimiterState, HeredocItem}
 
+    property macro_expansion_pragmas : Hash(Int32, Array(LocPragma))? = nil
+
+    alias LocPragma = LocSetPragma | LocPushPragma | LocPopPragma
+
+    record LocSetPragma,
+      filename : String,
+      line_number : Int32,
+      column_number : Int32 do
+      def run_pragma(lexer)
+        lexer.set_location filename, line_number, column_number
+      end
+    end
+
+    record LocPushPragma do
+      def run_pragma(lexer)
+        lexer.push_location
+      end
+    end
+
+    record LocPopPragma do
+      def run_pragma(lexer)
+        lexer.pop_location
+      end
+    end
+
     def initialize(string, string_pool : StringPool? = nil)
       @reader = Char::Reader.new(string)
       @token = Token.new
@@ -91,6 +116,11 @@ module Crystal
 
       start = current_pos
 
+      # Fix location by `macro_expansion_pragmas`.
+      if pragmas = macro_expansion_pragmas.try &.[start]?
+        pragmas.each &.run_pragma self
+      end
+
       reset_regex_flags = true
 
       case current_char
@@ -2713,9 +2743,7 @@ module Crystal
           next_char
         end
 
-        @token.filename = @filename = filename
-        @token.line_number = @line_number = line_number
-        @token.column_number = @column_number = column_number
+        set_location filename, line_number, column_number
       when 'p'
         # skip 'p'
         next_char_no_column_increment
@@ -2756,6 +2784,12 @@ module Crystal
       end
     end
 
+    def set_location(filename, line_number, column_number)
+      @token.filename = @filename = filename
+      @token.line_number = @line_number = line_number
+      @token.column_number = @column_number = column_number
+    end
+
     def pop_location
       if @stacked
         @stacked = false

src/compiler/crystal/syntax/to_s.cr

@@ -7,16 +7,17 @@ module Crystal
       to_s(io)
     end
 
-    def to_s(io, emit_loc_pragma = false, emit_doc = false)
-      visitor = ToSVisitor.new(io, emit_loc_pragma: emit_loc_pragma, emit_doc: emit_doc)
+    def to_s(io, macro_expansion_pragmas = nil, emit_doc = false)
+      visitor = ToSVisitor.new(io, macro_expansion_pragmas: macro_expansion_pragmas, emit_doc: emit_doc)
       self.accept visitor
     end
   end
 
   class ToSVisitor < Visitor
     @str : IO
+    @macro_expansion_pragmas : Hash(Int32, Array(Lexer::LocPragma))?
 
-    def initialize(@str = IO::Memory.new, @emit_loc_pragma = false, @emit_doc = false)
+    def initialize(@str = IO::Memory.new, @macro_expansion_pragmas = nil, @emit_doc = false)
       @indent = 0
       @inside_macro = 0
       @inside_lib = false
@@ -32,14 +33,9 @@ module Crystal
         @str.puts
       end
 
-      if @emit_loc_pragma && (loc = node.location) && loc.filename.is_a?(String)
-        @str << "#<loc:"
-        loc.filename.inspect(@str)
-        @str << ','
-        @str << loc.line_number
-        @str << ','
-        @str << loc.column_number
-        @str << '>'
+      if (macro_expansion_pragmas = @macro_expansion_pragmas) && (loc = node.location) && (filename = loc.filename).is_a?(String)
+        pragmas = macro_expansion_pragmas[@str.pos.to_i32] ||= [] of Lexer::LocPragma
+        pragmas << Lexer::LocSetPragma.new(filename, loc.line_number, loc.column_number)
       end
 
       true