Benchmark and Analysis Tooling (#13)

* added extra valgrind test packages

* added some docs for call,cache-grind and memtest

* adding some comments and docs

* fixed minor detail in profiling derivation

* updating flake

* fixed pipeline

* merged master

CONTRIBUTING.md

@@ -13,17 +13,17 @@ at this [guide](https://github.com/lf-lang/lingua-franca/wiki/Regression-Tests).
 
 ## Building and Testing with Nix
 
-**Listing Compilers**
+**List Compilers**
 ```
     $ nix run .#packages.x86_64-linux.list-compilers
 ```
 
-**Listing all available Tests**
+**List all available Tests**
 ```
     $  nix run .#packages.x86_64-linux.list-tests
 ```
 
-**Listing Benchmarks**
+**List Benchmarks**
 ```
     $ nix run .#packages.x86_64-linux.list-benchmarks
 ```
@@ -48,8 +48,7 @@ This will build and run every tests.
     $ nix run .#packages.x86_64-linux.all-benchmarks
 ```
 
-
-**Locally integration testing**
+**Local integration testing**
 
 Lets assume you have the following folder structure:
  - reactor-cpp/
@@ -60,6 +59,37 @@ Lets assume you have the following folder structure:
     $ nix run .#packages.x86_64-linux.all-tests --override-input reactor-cpp "./." lingua-franca-src "../lingua-franca/build/your_lfc_build.tar.gz"
 ```
 
+Building lfc with nix is a work in progress until then you have to sadly build lfc yourself and specify it this way.
+
+
+**Running all Benchmarks and collect results**
+
+```
+    $ nix build .\#packages.x86_64-linux.make-benchmark
+```
+
+This will generate a data/result.csv with all the measurements of all benchmarks.
+
+**Analysing Benchmarks and Tests for Memory Leaks**
+
+This will run valgrind on that test and create $out/data/Test-memtest.out which contains the requested debug information.
+```
+    nix build .\#packages.x86_64-linux.MLeaks-FullyConnected-gcc-wrapper-10-3-0
+```
+
+**Cachegrind**
+Analyse your benchmark for cache misses.
+```
+    nix build .\#packages.x86_64-linux.cachegrind-SleepingBarber-gcc-wrapper-10-3-0
+```
+
+**Callgrind**
+Profile and analyse your benchmarks call chain.
+```
+    nix build .\#packages.x86_64-linux.callgrind-SleepingBarber-gcc-wrapper-10-3-0
+```
+
+
 
 ### Benchmarking
 If youre changes are performance critically it is adviced to run the test from [here](https://github.com/lf-lang/lingua-franca/wiki/Running-Benchmarks)

flake.nix

@@ -4,6 +4,7 @@
   inputs = {
     utils.url = "github:numtide/flake-utils";
     nixpkgs.url = "github:NixOs/nixpkgs/nixos-unstable";
+    lf-benchmark-runner.url = "github:revol-xut/lf-benchmark-runner";
 
     # input for the reactor-cpp
     reactor-cpp = {
@@ -13,7 +14,7 @@
 
     # source for the lingu franca compiler
     lingua-franca-src = {
-      url = "https://github.com/lf-lang/lingua-franca/releases/download/v0.1.0-beta/lfc_0.1.0-beta.tar.gz";
+      url = "https://github.com/lf-lang/lingua-franca/releases/download/nightly/lfc_nightly_20220517-050749.tar.gz";
       flake = false;
     };
 
@@ -30,7 +31,7 @@
     };
   };
 
-  outputs = inputs@{ self, utils, nixpkgs, reactor-cpp, lingua-franca-src, lingua-franca-tests, lingua-franca-benchmarks, ... }:
+  outputs = { utils, nixpkgs, lf-benchmark-runner, reactor-cpp, lingua-franca-src, lingua-franca-tests, lingua-franca-benchmarks, ... }:
     utils.lib.eachDefaultSystem (system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
@@ -43,12 +44,13 @@
           reactor-cpp-src = reactor-cpp;
           lingua-franca-src = lingua-franca-src;
           lingua-franca-benchmarks = lingua-franca-benchmarks;
+          lf-benchmark-runner = lf-benchmark-runner.packages."${system}".lf-benchmark-runner;
         };
-        customPackages = pkgs.lib.mergeAttrs allTests allBenchmarks;
+        customPackages = pkgs.lib.mergeAttrs allBenchmarks allTests;
       in
       rec {
-        checks = allTests;
-        packages = pkgs.lib.mergeAttrs pkgs customPackages;
+        checks = packages;
+        packages = customPackages;
       }
     );
 }

nix/benchmark.nix

@@ -2,9 +2,11 @@
 , pkgs
 , lib
 , stdenv
+, lf-benchmark-runner
 , reactor-cpp-src
 , lingua-franca-src
 , lingua-franca-benchmarks
+, valgrind
 }:
 let
 
@@ -72,11 +74,62 @@ let
       ${pkgs.coreutils}/bin/cp ${run_all}/bin/* $out/bin/
     '' + install_command;
   };
+
+  # runs our custom benchmark data extractor on the specified benchmark
+  benchmark_command = (benchmark: "${lf-benchmark-runner}/bin/lf-benchmark-runner --target lf-cpp --binary ${benchmark}/bin/${benchmark.name} --file ./result.csv");
+
+  # compiles a giant script to run every benchmark and collect their results into on csv file
+  benchmark_commands = lib.strings.concatStringsSep "\n" (builtins.map benchmark_command list_of_derivations);
+
+  # derivation defintion for running and collecting data from benchmarks
+  make-benchmark = mkDerivation {
+    src = ./.;
+    name = "make-benchmark";
+
+    buildPhase = benchmark_commands;
+
+    installPhase = ''
+      mkdir -p $out/data/
+      cp result.csv $out/data/
+    '';
+  };
+
+  # derivation for call and cachegrind. measuring a given package
+  profiler = (package: valgrind_check:
+    {
+      name = "${valgrind_check}-${package.name}";
+      value = mkDerivation {
+        name = "${valgrind_check}-${package.name}";
+        src = ./.;
+        nativeBuildInputs = [ valgrind ];
+
+        buildPhase = ''
+          ${valgrind}/bin/valgrind --tool=${valgrind_check} ${package}/bin/${package.name} 
+        '';
+        installPhase = ''
+          mkdir -p $out/data
+          cp ${valgrind_check}.out.* $out/data/${package.name}-${valgrind_check}.out
+        '';
+      };
+    }
+  );
+
+  extract_derivations = (list: lib.attrValues (lib.listToAttrs list));
+  attribute_set_derivations = (library.double_map benchmarks library.compilers library.buildDerivation);
+
+
+  attribute_set_cachegrind = (builtins.map (x: profiler x "cachegrind") (extract_derivations attribute_set_derivations));
+  attribute_set_callgrind = (builtins.map (x: profiler x "callgrind") (extract_derivations attribute_set_derivations));
+  attribute_set_memory = (builtins.map library.memtest (extract_derivations attribute_set_derivations));
 in
-lib.listToAttrs ((library.double_map benchmarks library.compilers library.buildDerivation) ++
-  [
-    { name = "all-benchmarks"; value = all-benchmarks; }
-    { name = "list-benchmarks"; value = list-benchmarks; }
-    { name = "list-compilers"; value = library.list-compilers; }
-  ])
+lib.listToAttrs (attribute_set_derivations
+  ++ attribute_set_cachegrind
+  ++ attribute_set_callgrind
+  ++ attribute_set_memory
+  ++ [
+  { name = "all-benchmarks"; value = all-benchmarks; }
+  { name = "list-benchmarks"; value = list-benchmarks; }
+  { name = "list-compilers"; value = library.list-compilers; }
+  { name = "make-benchmark"; value = make-benchmark; }
+])
 

nix/lfc.nix

@@ -2,7 +2,7 @@
 , config
 , lib
 , mkDerivation
-, jdk11_headless
+, jdk17_headless
 , lingua-franca-src
 }:
 let
@@ -19,14 +19,14 @@ mkDerivation {
 
   src = lingua-franca-src;
 
-  buildInputs = [ jdk11_headless ];
+  buildInputs = [ jdk17_headless ];
 
-  _JAVA_HOME = "${jdk11_headless}/";
+  _JAVA_HOME = "${jdk17_headless}/";
 
   postPatch = ''
     substituteInPlace bin/lfc \
       --replace 'base=`dirname $(dirname ''${abs_path})`' "base='$out'" \
-      --replace "run_lfc_with_args" "${jdk11_headless}/bin/java -jar $out/lib/jars/${extracted_name}"
+      --replace "run_lfc_with_args" "${jdk17_headless}/bin/java -jar $out/lib/jars/${extracted_name}"
   '';
 
   buildPhase = ''