Add common check: solution identical to exemplar (#157)

.github/workflows/elixir_test.yml

@@ -11,6 +11,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
+        with:
+          submodules: true
+
       - name: Install Dependencies
         run: |
           mix local.rebar --force

.gitignore

@@ -32,6 +32,8 @@ ex_mentor-*.tar
 test_results.json
 analysis.json
 
+.mix/
+
 tmp/
 
 /priv/plts/*.plt

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "elixir"]
+	path = elixir
+	url = https://github.com/exercism/elixir.git

config/test.exs

@@ -1,3 +1,3 @@
 use Mix.Config
 
-config :logger, level: :error
+config :logger, level: :warn

lib/elixir_analyzer.ex

@@ -43,8 +43,8 @@ defmodule ElixirAnalyzer do
   * `:output_file`, - specifies the name of the output_file, defaults to
     `@output_file` (`analysis.json`)
 
-  * `:exercise_config` - specifies the path to the JSON exercise configuration,
-    defaults to `@exercise_config` (`./config/exercise_data.json`)
+  * `:exercise_config` - specifies the path to the exercise configuration,
+    defaults to `@exercise_config` (`./config/config.exs`)
 
   * `:write_results` - boolean flag if an analysis should output the results to
     JSON file, defaults to `true`
@@ -110,11 +110,12 @@ defmodule ElixirAnalyzer do
     try do
       Logger.debug("Getting the exercise config")
       exercise_config = params.exercise_config[params.exercise]
-      {code_path, code_file, analysis_module} = do_init(params, exercise_config)
+      {code_path, code_file, exemplar_path, analysis_module} = do_init(params, exercise_config)
 
       Logger.debug("Initialization successful",
         path: params.path,
         code_path: code_path,
+        exemplar_path: exemplar_path,
         analysis_module: analysis_module
       )
 
@@ -131,6 +132,7 @@ defmodule ElixirAnalyzer do
         submission
         | code_path: code_path,
           code_file: code_file,
+          exemplar_path: exemplar_path,
           analysis_module: analysis_module
       }
     rescue
@@ -166,7 +168,14 @@ defmodule ElixirAnalyzer do
     code_path = Path.dirname(full_code_path)
     code_file = Path.basename(full_code_path)
 
-    {code_path, code_file, exercise_config[:analyzer_module] || ElixirAnalyzer.TestSuite.Default}
+    exemplar_path =
+      case meta_config["files"]["exemplar"] do
+        [path | _] -> Path.join(params.path, path)
+        _ -> nil
+      end
+
+    {code_path, code_file, exemplar_path,
+     exercise_config[:analyzer_module] || ElixirAnalyzer.TestSuite.Default}
   end
 
   # Else, use passed in params to analyze
@@ -181,7 +190,9 @@ defmodule ElixirAnalyzer do
   # Check
   # - check if the file exists
   # - read in the code
-  # - compile
+  # - check if there is an exemplar
+  # - read in the exemplar
+  # - parse the exemplar into an AST
   defp check(%Submission{halted: true} = submission, _params) do
     Logger.warning("Check not performed, halted previously")
     submission
@@ -192,7 +203,19 @@ defmodule ElixirAnalyzer do
          :ok <- Logger.info("Attempting to read code file", code_file_path: path_to_code),
          {:code_read, {:ok, code_str}} <- {:code_read, File.read(path_to_code)},
          :ok <- Logger.info("Code file read successfully"),
-         {:code_str, submission} <- {:code_str, %{submission | code: code_str}} do
+         submission <- %{submission | code: code_str},
+         :ok <- Logger.info("Check if exemplar exists", exemplar_path: submission.exemplar_path),
+         {:exemplar_exists, submission, exemplar_path} when not is_nil(exemplar_path) <-
+           {:exemplar_exists, submission, submission.exemplar_path},
+         :ok <-
+           Logger.info("Exemplar file exists, attempting to read", exemplar_path: exemplar_path),
+         {:exemplar_read, submission, {:ok, exemplar_code}} <-
+           {:exemplar_read, submission, File.read(exemplar_path)},
+         :ok <- Logger.info("Exemplar file read successfully, attempting to parse"),
+         {:exemplar_ast, submission, {:ok, exemplar_code}} <-
+           {:exemplar_ast, submission, Code.string_to_quoted(exemplar_code)},
+         :ok <- Logger.info("Exemplar file parsed successfully"),
+         submission <- %{submission | exemplar_code: exemplar_code} do
       submission
     else
       {:code_read, {:error, reason}} ->
@@ -211,6 +234,24 @@ defmodule ElixirAnalyzer do
           },
           type: :essential
         })
+
+      {:exemplar_exists, submission, nil} ->
+        Logger.info("There is no exemplar file for this exercise")
+        submission
+
+      {:exemplar_read, submission, {:error, reason}} ->
+        Logger.warning("Exemplar file not found. Reason: #{reason}",
+          exemplar_path: submission.exemplar_path
+        )
+
+        submission
+
+      {:exemplar_ast, submission, {:error, reason}} ->
+        Logger.warning("Exemplar file could not be parsed. Reason: #{inspect(reason)}",
+          exemplar_code: submission.exemplar_code
+        )
+
+        submission
     end
   end
 
@@ -226,7 +267,7 @@ defmodule ElixirAnalyzer do
 
     submission =
       submission
-      |> submission.analysis_module.analyze(submission.code)
+      |> submission.analysis_module.analyze(submission.code, submission.exemplar_code)
       |> Submission.set_analyzed(true)
 
     Logger.info("Analyzing code complete")

lib/elixir_analyzer/constants.ex

@@ -26,6 +26,7 @@ defmodule ElixirAnalyzer.Constants do
     solution_debug_functions: "elixir.solution.debug_functions",
     solution_last_line_assignment: "elixir.solution.last_line_assignment",
     solution_compiler_warnings: "elixir.solution.compiler_warnings",
+    solution_same_as_exemplar: "elixir.solution.same_as_exemplar",
 
     # Concept exercises
 

lib/elixir_analyzer/exercise_test.ex

@@ -19,7 +19,7 @@ defmodule ElixirAnalyzer.ExerciseTest do
 
       import unquote(__MODULE__)
       @before_compile unquote(__MODULE__)
-      @dialyzer no_match: {:do_analyze, 3}
+      @dialyzer no_match: {:do_analyze, 4}
     end
   end
 
@@ -43,24 +43,24 @@ defmodule ElixirAnalyzer.ExerciseTest do
     assert_call_tests = Enum.map(assert_call_data, &AssertCallCompiler.compile(&1, code_ast))
 
     quote do
-      @spec analyze(Submission.t(), String.t()) :: Submission.t()
-      def analyze(%Submission{} = submission, code_as_string) when is_binary(code_as_string) do
+      @spec analyze(Submission.t(), String.t(), nil | Macro.t()) :: Submission.t()
+      def analyze(%Submission{} = submission, code_as_string, exemplar_ast) do
         case Code.string_to_quoted(code_as_string) do
           {:ok, code_ast} ->
-            do_analyze(submission, code_ast, code_as_string)
+            do_analyze(submission, code_ast, code_as_string, exemplar_ast)
 
           {:error, e} ->
             append_analysis_failure(submission, e)
         end
       end
 
-      defp do_analyze(%Submission{} = submission, code_ast, code_as_string)
+      defp do_analyze(%Submission{} = submission, code_ast, code_as_string, exemplar_ast)
            when is_binary(code_as_string) do
         results =
           Enum.concat([
             unquote(feature_tests),
             unquote(assert_call_tests),
-            CommonChecks.run(code_ast, code_as_string)
+            CommonChecks.run(code_ast, code_as_string, exemplar_ast)
           ])
           |> filter_suppressed_results()
 

lib/elixir_analyzer/exercise_test/common_checks.ex

@@ -9,6 +9,7 @@ defmodule ElixirAnalyzer.ExerciseTest.CommonChecks do
     ModuleAttributeNames,
     ModulePascalCase,
     CompilerWarnings,
+    ExemplarComparison,
     Indentation
   }
 
@@ -23,14 +24,15 @@ defmodule ElixirAnalyzer.ExerciseTest.CommonChecks do
     end
   end
 
-  @spec run(Macro.t(), String.t()) :: [{:pass | :fail | :skip, %Comment{}}]
-  def run(code_ast, code_as_string) when is_binary(code_as_string) do
+  @spec run(Macro.t(), String.t(), nil | Macro.t()) :: [{:pass | :fail | :skip, %Comment{}}]
+  def run(code_ast, code_as_string, exemplar_ast) when is_binary(code_as_string) do
     [
       FunctionNames.run(code_ast),
       VariableNames.run(code_ast),
       ModuleAttributeNames.run(code_ast),
       ModulePascalCase.run(code_ast),
       CompilerWarnings.run(code_ast),
+      ExemplarComparison.run(code_ast, exemplar_ast),
       Indentation.run(code_ast, code_as_string)
     ]
     |> List.flatten()

lib/elixir_analyzer/exercise_test/common_checks/exemplar_comparaison.ex

@@ -0,0 +1,27 @@
+defmodule ElixirAnalyzer.ExerciseTest.CommonChecks.ExemplarComparison do
+  @moduledoc """
+  Compares the solution to the exemplar solution for concept exercises.
+  Ignores practice exercises.
+  """
+
+  alias ElixirAnalyzer.Constants
+  alias ElixirAnalyzer.Comment
+
+  @spec run(Macro.t(), nil | Macro.t()) :: [{:pass | :fail | :skip, %Comment{}}]
+  def run(_ast, nil), do: []
+
+  def run(code_ast, exemplar_ast) do
+    if Macro.to_string(code_ast) == Macro.to_string(exemplar_ast) do
+      [
+        {:pass,
+         %Comment{
+           type: :celebratory,
+           name: Constants.solution_same_as_exemplar(),
+           comment: Constants.solution_same_as_exemplar()
+         }}
+      ]
+    else
+      []
+    end
+  end
+end

lib/elixir_analyzer/submission.ex

@@ -29,6 +29,8 @@ defmodule ElixirAnalyzer.Submission do
             path: nil,
             code_path: nil,
             code_file: nil,
+            exemplar_path: nil,
+            exemplar_code: nil,
             code: nil,
             analysis_module: nil
 
@@ -40,6 +42,8 @@ defmodule ElixirAnalyzer.Submission do
           path: String.t(),
           code_path: String.t(),
           code_file: String.t(),
+          exemplar_path: String.t() | nil,
+          exemplar_code: Macro.t() | nil,
           code: String.t(),
           analysis_module: atom()
         }

test/elixir_analyzer/test_suite/bird_count_test.exs

@@ -3,7 +3,7 @@ defmodule ElixirAnalyzer.ExerciseTest.BirdCountTest do
     exercise_test_module: ElixirAnalyzer.TestSuite.BirdCount
 
   test_exercise_analysis "example solution",
-    comments: [] do
+    comments: [ElixirAnalyzer.Constants.solution_same_as_exemplar()] do
     defmodule BirdCount do
       def today([]), do: nil
       def today([head | _]), do: head

test/elixir_analyzer/test_suite/boutique_suggestions_test.exs

@@ -2,8 +2,8 @@ defmodule ElixirAnalyzer.ExerciseTest.BoutiqueSuggestionsTest do
   use ElixirAnalyzer.ExerciseTestCase,
     exercise_test_module: ElixirAnalyzer.TestSuite.BoutiqueSuggestions
 
-  test_exercise_analysis "correct solutions",
-    comments: [] do
+  test_exercise_analysis "example solution",
+    comments: [ElixirAnalyzer.Constants.solution_same_as_exemplar()] do
     [
       defmodule BoutiqueSuggestions do
         def get_combinations(tops, bottoms, options \\ []) do
@@ -18,7 +18,13 @@ defmodule ElixirAnalyzer.ExerciseTest.BoutiqueSuggestionsTest do
             {top, bottom}
           end
         end
-      end,
+      end
+    ]
+  end
+
+  test_exercise_analysis "correct solutions",
+    comments: [] do
+    [
       defmodule BoutiqueSuggestions do
         def get_combinations(tops, bottoms, options \\ []) do
           maximum_price = Keyword.get(options, :maximum_price, 100.00)

test/elixir_analyzer/test_suite/chesboard_test.exs

@@ -3,7 +3,7 @@ defmodule ElixirAnalyzer.ExerciseTest.ChessboardTest do
     exercise_test_module: ElixirAnalyzer.TestSuite.Chessboard
 
   test_exercise_analysis "example solution",
-    comments: [] do
+    comments: [ElixirAnalyzer.Constants.solution_same_as_exemplar()] do
     defmodule Chessboard do
       def rank_range do
         1..8

test/elixir_analyzer/test_suite/file_sniffer_test.exs

@@ -3,7 +3,7 @@ defmodule ElixirAnalyzer.ExerciseTest.FileSnifferTest do
     exercise_test_module: ElixirAnalyzer.TestSuite.FileSniffer
 
   test_exercise_analysis "example solution",
-    comments: [] do
+    comments: [ElixirAnalyzer.Constants.solution_same_as_exemplar()] do
     [
       defmodule FileSniffer do
         def type_from_extension("bmp"), do: "image/bmp"
@@ -33,7 +33,13 @@ defmodule ElixirAnalyzer.ExerciseTest.FileSnifferTest do
             {:error, "Warning, file format and file extension do not match."}
           end
         end
-      end,
+      end
+    ]
+  end
+
+  test_exercise_analysis "other solutions",
+    comments: [] do
+    [
       defmodule FileSniffer do
         def type_from_binary(<<?B, ?M, _::binary>>), do: "image/bmp"
 

test/elixir_analyzer/test_suite/freelancer_rates_test.exs

@@ -3,7 +3,7 @@ defmodule ElixirAnalyzer.ExerciseTest.FreelancerRatesTest do
     exercise_test_module: ElixirAnalyzer.TestSuite.FreelancerRates
 
   test_exercise_analysis "example solution",
-    comments: [] do
+    comments: [ElixirAnalyzer.Constants.solution_same_as_exemplar()] do
     [
       defmodule FreelancerRates do
         def daily_rate(hourly_rate) do
@@ -26,7 +26,13 @@ defmodule ElixirAnalyzer.ExerciseTest.FreelancerRatesTest do
           days_in_budget = budget / daily_rate_after_discount
           Float.floor(days_in_budget, 1)
         end
-      end,
+      end
+    ]
+  end
+
+  test_exercise_analysis "other solutions",
+    comments: [] do
+    [
       defmodule FreelancerRates do
         def daily_rate(hourly_rate) do
           hourly_rate * 8.0

test/elixir_analyzer/test_suite/german_sysadmin_test.exs

@@ -3,7 +3,7 @@ defmodule ElixirAnalyzer.ExerciseTest.GermanSysadminTest do
     exercise_test_module: ElixirAnalyzer.TestSuite.GermanSysadmin
 
   test_exercise_analysis "example solution",
-    comments: [] do
+    comments: [ElixirAnalyzer.Constants.solution_same_as_exemplar()] do
     defmodule Username do
       def sanitize('') do
         ''

test/elixir_analyzer/test_suite/guessing_game_test.exs

@@ -3,7 +3,7 @@ defmodule ElixirAnalyzer.ExerciseTest.GuessingGameTest do
     exercise_test_module: ElixirAnalyzer.TestSuite.GuessingGame
 
   test_exercise_analysis "example solution",
-    comments: [] do
+    comments: [ElixirAnalyzer.Constants.solution_same_as_exemplar()] do
     defmodule GuessingGame do
       def compare(secret_number, guess \\ :no_guess)
 

test/elixir_analyzer/test_suite/high_score_test.exs

@@ -3,7 +3,7 @@ defmodule ElixirAnalyzer.ExerciseTest.HighScoreTest do
     exercise_test_module: ElixirAnalyzer.TestSuite.HighScore
 
   test_exercise_analysis "example solution",
-    comments: [] do
+    comments: [ElixirAnalyzer.Constants.solution_same_as_exemplar()] do
     [
       defmodule HighScore do
         @initial_score 0
@@ -29,7 +29,13 @@ defmodule ElixirAnalyzer.ExerciseTest.HighScoreTest do
         def get_players(scores) do
           Map.keys(scores)
         end
-      end,
+      end
+    ]
+  end
+
+  test_exercise_analysis "other solutions",
+    comments: [] do
+    [
       defmodule HighScore do
         @initial_score 0