Don't run doctests for generated functions

lib/livebook/intellisense/docs.ex

@@ -143,21 +143,4 @@ defmodule Livebook.Intellisense.Docs do
   # loads elixir.beam, so we explicitly list it.
   defp ensure_loaded?(Elixir), do: false
   defp ensure_loaded?(module), do: Code.ensure_loaded?(module)
-
-  @doc """
-  Checks if the module has any documentation.
-  """
-  @spec any_docs?(module()) :: boolean()
-  def any_docs?(module) do
-    case Code.fetch_docs(module) do
-      {:docs_v1, _, _, _, %{}, _, _} ->
-        true
-
-      {:docs_v1, _, _, _, _, _, docs} ->
-        Enum.any?(docs, &match?({_, _, _, %{}, _}, &1))
-
-      _ ->
-        false
-    end
-  end
 end

lib/livebook/runtime/evaluator.ex

@@ -460,9 +460,7 @@ defmodule Livebook.Runtime.Evaluator do
       end
 
     if ebin_path() do
-      new_context.env.context_modules
-      |> Enum.filter(&Livebook.Intellisense.Docs.any_docs?/1)
-      |> Livebook.Runtime.Evaluator.Doctests.run(code)
+      Livebook.Runtime.Evaluator.Doctests.run(new_context.env.context_modules, code)
     end
 
     state = put_context(state, ref, new_context)

lib/livebook/runtime/evaluator/doctests.ex

@@ -9,17 +9,45 @@ defmodule Livebook.Runtime.Evaluator.Doctests do
   Runs doctests in the given modules.
   """
   @spec run(list(module()), String.t()) :: :ok
-  def run(modules, code)
+  def run(modules, code) do
+    doctests_specs =
+      for module <- modules, doctests_spec = doctests_spec(module), do: doctests_spec
 
-  def run([], _code), do: :ok
+    do_run(doctests_specs, code)
+  end
 
-  def run(modules, code) do
-    case define_test_module(modules) do
+  defp doctests_spec(module) do
+    case Code.fetch_docs(module) do
+      {:docs_v1, _, _, _, doc_content, _, member_docs} ->
+        funs =
+          for {{:function, name, arity}, annotation, _signatures, _doc, _meta} <- member_docs,
+              do: %{name: name, arity: arity, generated: :erl_anno.generated(annotation)}
+
+        {generated_funs, regular_funs} = Enum.split_with(funs, & &1.generated)
+
+        if regular_funs != [] or is_map(doc_content) do
+          except = Enum.map(generated_funs, &{&1.name, &1.arity})
+          %{module: module, except: except}
+        end
+
+      _ ->
+        nil
+    end
+  end
+
+  defp do_run([], _code), do: :ok
+
+  defp do_run(doctests_specs, code) do
+    case define_test_module(doctests_specs) do
       {:ok, test_module} ->
-        if test_module.tests != [] do
-          lines = String.split(code, ["\r\n", "\n"])
+        lines = String.split(code, ["\r\n", "\n"])
+
+        # Ignore test cases that don't actually point to a doctest
+        # in the source code
+        tests = Enum.filter(test_module.tests, &doctest_at_line?(lines, &1.tags.doctest_line))
 
-          test_module.tests
+        if tests != [] do
+          tests
           |> Enum.sort_by(& &1.tags.doctest_line)
           |> Enum.each(fn test ->
             report_doctest_running(test)
@@ -38,6 +66,16 @@ defmodule Livebook.Runtime.Evaluator.Doctests do
     :ok
   end
 
+  defp doctest_at_line?(lines, line_number) do
+    line = Enum.fetch!(lines, line_number - 1)
+
+    case String.trim_leading(line) do
+      "iex>" <> _ -> true
+      "iex(" <> _ -> true
+      _ -> false
+    end
+  end
+
   defp report_doctest_running(test) do
     send_doctest_report(%{
       line: test.tags.doctest_line,
@@ -122,9 +160,10 @@ defmodule Livebook.Runtime.Evaluator.Doctests do
     end
   end
 
-  defp define_test_module(modules) do
+  defp define_test_module(doctests_specs) do
     id =
-      modules
+      doctests_specs
+      |> Enum.map(& &1.module)
       |> Enum.sort()
       |> Enum.map_join("-", fn module ->
         module
@@ -140,8 +179,8 @@ defmodule Livebook.Runtime.Evaluator.Doctests do
       defmodule name do
         use ExUnit.Case, register: false
 
-        for module <- modules do
-          doctest module
+        for doctests_spec <- doctests_specs do
+          doctest doctests_spec.module, except: doctests_spec.except
         end
       end
 

test/livebook/runtime/evaluator_test.exs

@@ -154,7 +154,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
       assert_receive {:runtime_evaluation_response, :code_1, {:error, message, :other},
                       metadata()}
 
-      assert clean_message(message) == """
+      assert """
              ** (FunctionClauseError) no function clause matching in List.first/2
 
                  The following arguments were given to List.first/2:
@@ -170,9 +170,7 @@ defmodule Livebook.Runtime.EvaluatorTest do
                      def first([], default)
                      def first([head | _], _default)
 
-                 (elixir 1.15.0-rc.1) lib/list.ex:293: List.first/2
-                 file.ex:1: (file)
-             """
+             """ <> _ = clean_message(message)
     end
 
     test "returns additional metadata when there is a syntax error", %{evaluator: evaluator} do
@@ -630,6 +628,66 @@ defmodule Livebook.Runtime.EvaluatorTest do
                         status: :failed
                       }}
     end
+
+    test "does not run generated doctests", %{evaluator: evaluator} do
+      code = ~S'''
+      defmodule Livebook.Runtime.EvaluatorTest.DoctestsGeneratedBase do
+        defmacro __using__(_) do
+          quote do
+            @doc """
+
+                iex> 1
+                2
+
+            """
+            def foo, do: :ok
+          end
+        end
+      end
+
+      defmodule Livebook.Runtime.EvaluatorTest.DoctestsGenerated do
+        use Livebook.Runtime.EvaluatorTest.DoctestsGeneratedBase
+      end
+      '''
+
+      Evaluator.evaluate_code(evaluator, :elixir, code, :code_1, [])
+
+      assert_receive {:runtime_evaluation_response, :code_1, _, metadata()}
+      refute_received {:runtime_doctest_report, :code_1, %{}}
+
+      # Here the generated doctest line matches another iex> prompt
+      # in the module, but we expect the :erl_anno check to filter
+      # it out
+
+      code = ~S'''
+      defmodule Livebook.Runtime.EvaluatorTest.DoctestsGeneratedBase do
+        defmacro __using__(_) do
+          quote do
+            @doc """
+
+                iex> 1
+                2
+
+            """
+            def foo, do: :ok
+          end
+        end
+      end
+
+      defmodule Livebook.Runtime.EvaluatorTest.DoctestsGenerated do
+        use Livebook.Runtime.EvaluatorTest.DoctestsGeneratedBase
+        @string """
+            iex> 1
+            2
+        """
+      end
+      '''
+
+      Evaluator.evaluate_code(evaluator, :elixir, code, :code_1, [])
+
+      assert_receive {:runtime_evaluation_response, :code_1, _, metadata()}
+      refute_received {:runtime_doctest_report, :code_1, %{}}
+    end
   end
 
   describe "evaluate_code/6 identifier tracking" do