From 09f5a93972530771dc715ff2ac2f78b92a6b6f48 Mon Sep 17 00:00:00 2001 From: Tassilo Tanneberger Date: Tue, 17 May 2022 14:49:58 +0200 Subject: [PATCH] Benchmark and Analysis Tooling (#13) * added extra valgrind test packages * added some docs for call,cache-grind and memtest * adding some comments and docs * fixed minor detail in profiling derivation * updating flake * fixed pipeline * merged master --- CONTRIBUTING.md | 40 ++++++++++++++--- flake.lock | 107 +++++++++++++++++++++++++++++++++++++------- flake.nix | 12 ++--- nix/benchmark.nix | 65 ++++++++++++++++++++++++--- nix/lfc.nix | 8 ++-- nix/library.nix | 26 ++++++++++- nix/reactor-cpp.nix | 7 +-- nix/test.nix | 29 +++++++----- 8 files changed, 242 insertions(+), 52 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e9fea2a..7ec51fc4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,17 +13,17 @@ at this [guide](https://github.com/lf-lang/lingua-franca/wiki/Regression-Tests). ## Building and Testing with Nix -**Listing Compilers** +**List Compilers** ``` $ nix run .#packages.x86_64-linux.list-compilers ``` -**Listing all available Tests** +**List all available Tests** ``` $ nix run .#packages.x86_64-linux.list-tests ``` -**Listing Benchmarks** +**List Benchmarks** ``` $ nix run .#packages.x86_64-linux.list-benchmarks ``` @@ -48,8 +48,7 @@ This will build and run every tests. $ nix run .#packages.x86_64-linux.all-benchmarks ``` - -**Locally integration testing** +**Local integration testing** Lets assume you have the following folder structure: - reactor-cpp/ @@ -60,6 +59,37 @@ Lets assume you have the following folder structure: $ nix run .#packages.x86_64-linux.all-tests --override-input reactor-cpp "./." lingua-franca-src "../lingua-franca/build/your_lfc_build.tar.gz" ``` +Building lfc with nix is a work in progress until then you have to sadly build lfc yourself and specify it this way. + + +**Running all Benchmarks and collect results** + +``` + $ nix build .\#packages.x86_64-linux.make-benchmark +``` + +This will generate a data/result.csv with all the measurements of all benchmarks. + +**Analysing Benchmarks and Tests for Memory Leaks** + +This will run valgrind on that test and create $out/data/Test-memtest.out which contains the requested debug information. +``` + nix build .\#packages.x86_64-linux.MLeaks-FullyConnected-gcc-wrapper-10-3-0 +``` + +**Cachegrind** +Analyse your benchmark for cache misses. +``` + nix build .\#packages.x86_64-linux.cachegrind-SleepingBarber-gcc-wrapper-10-3-0 +``` + +**Callgrind** +Profile and analyse your benchmarks call chain. +``` + nix build .\#packages.x86_64-linux.callgrind-SleepingBarber-gcc-wrapper-10-3-0 +``` + + ### Benchmarking If youre changes are performance critically it is adviced to run the test from [here](https://github.com/lf-lang/lingua-franca/wiki/Running-Benchmarks) diff --git a/flake.lock b/flake.lock index 892303cb..1b999758 100644 --- a/flake.lock +++ b/flake.lock @@ -1,13 +1,33 @@ { "nodes": { + "lf-benchmark-runner": { + "inputs": { + "naersk": "naersk", + "nixpkgs": "nixpkgs", + "utils": "utils" + }, + "locked": { + "lastModified": 1650028179, + "narHash": "sha256-p8jQBiVwk0GQQjsK4gS1wk76bwPqr6J5Jn1UDlAWxjc=", + "owner": "revol-xut", + "repo": "lf-benchmark-runner", + "rev": "7317490b9e4407c90b9bf51cd56d00b726a20aaf", + "type": "github" + }, + "original": { + "owner": "revol-xut", + "repo": "lf-benchmark-runner", + "type": "github" + } + }, "lingua-franca-benchmarks": { "flake": false, "locked": { - "lastModified": 1649148080, - "narHash": "sha256-Gr4krm7EYOzue2WPjV7KrGlCCRKpMyQS9ZsD4P8foQo=", + "lastModified": 1652281421, + "narHash": "sha256-n2IQbbFBG6TncL8fTxkGgNdRNhoecG49yC+Aohpob40=", "owner": "lf-lang", "repo": "benchmarks-lingua-franca", - "rev": "69d8758fa636314523201547419fe4e1feea46b1", + "rev": "285a3fe33f126e3d73b917ad774d76a1be8b5301", "type": "github" }, "original": { @@ -19,23 +39,23 @@ "lingua-franca-src": { "flake": false, "locked": { - "narHash": "sha256-mP+MQSTSUeqELGR6KJJCg943p6uw+XSn6/3tmhnCc4s=", + "narHash": "sha256-Zyl79dbIR2oTcS1icEoyII8Jk9JEPfMtT40Dw4DDd2A=", "type": "tarball", - "url": "https://github.com/lf-lang/lingua-franca/releases/download/v0.1.0-beta/lfc_0.1.0-beta.tar.gz" + "url": "https://github.com/lf-lang/lingua-franca/releases/download/nightly/lfc_nightly_20220517-050749.tar.gz" }, "original": { "type": "tarball", - "url": "https://github.com/lf-lang/lingua-franca/releases/download/v0.1.0-beta/lfc_0.1.0-beta.tar.gz" + "url": "https://github.com/lf-lang/lingua-franca/releases/download/nightly/lfc_nightly_20220517-050749.tar.gz" } }, "lingua-franca-tests": { "flake": false, "locked": { - "lastModified": 1649178626, - "narHash": "sha256-EHLjal7wjF/Au+8jemyciQ5q8rMgiHHOT0QzwyToNfo=", + "lastModified": 1652757344, + "narHash": "sha256-m+ZE5bpguTMmaHzT1T1br+qBLuon6QUNqWuhEelD7ks=", "owner": "lf-lang", "repo": "lingua-franca", - "rev": "e4639438ed2ec3ecb0d1346791188d96a2594c1f", + "rev": "90b09afe305db2ca66a8f30eb1a1779ce27ebded", "type": "github" }, "original": { @@ -44,13 +64,50 @@ "type": "github" } }, + "naersk": { + "inputs": { + "nixpkgs": [ + "lf-benchmark-runner", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1649096192, + "narHash": "sha256-7O8e+eZEYeU+ET98u/zW5epuoN/xYx9G+CIh4DjZVzY=", + "owner": "nix-community", + "repo": "naersk", + "rev": "d626f73332a8f587b613b0afe7293dd0777be07d", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "naersk", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1649225869, - "narHash": "sha256-u1zLtPmQzhT9mNXyM8Ey9pk7orDrIKdwooeGDEXm5xM=", + "lastModified": 1649368816, + "narHash": "sha256-lVzCpg2xfTUrfcankjlym/mrh/7F/gpWQ7CYQM6BcPY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "29abf698b384258b540e39a86b53ea980495ac4c", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-21.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1652659998, + "narHash": "sha256-FqNrXC1EE6U2RACwXBlsAvg1lqQGLYpuYb6+W3DL9vA=", "owner": "NixOs", "repo": "nixpkgs", - "rev": "b6966d911da89e5a7301aaef8b4f0a44c77e103c", + "rev": "1d7db1b9e4cf1ee075a9f52e5c36f7b9f4207502", "type": "github" }, "original": { @@ -63,11 +120,11 @@ "reactor-cpp": { "flake": false, "locked": { - "lastModified": 1649143902, - "narHash": "sha256-5kZfT7yzJXTcHPwBcsMZXQMvC3LT81IAhex0tjbXknk=", + "lastModified": 1650459345, + "narHash": "sha256-onjwP1FEJYBq9p8gqm+Jh7Kmsmpb0ocBT6AIK4VyNAs=", "owner": "lf-lang", "repo": "reactor-cpp", - "rev": "aef18b98047039fc7875daa8fceda370bff17a17", + "rev": "46a618c01e494b7b476707c30dd6067ad66759d6", "type": "github" }, "original": { @@ -78,12 +135,13 @@ }, "root": { "inputs": { + "lf-benchmark-runner": "lf-benchmark-runner", "lingua-franca-benchmarks": "lingua-franca-benchmarks", "lingua-franca-src": "lingua-franca-src", "lingua-franca-tests": "lingua-franca-tests", - "nixpkgs": "nixpkgs", + "nixpkgs": "nixpkgs_2", "reactor-cpp": "reactor-cpp", - "utils": "utils" + "utils": "utils_2" } }, "utils": { @@ -100,6 +158,21 @@ "repo": "flake-utils", "type": "github" } + }, + "utils_2": { + "locked": { + "lastModified": 1652776076, + "narHash": "sha256-gzTw/v1vj4dOVbpBSJX4J0DwUR6LIyXo7/SuuTJp1kM=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "04c1b180862888302ddfb2e3ad9eaa63afc60cf8", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 52fd2403..3c7530d4 100644 --- a/flake.nix +++ b/flake.nix @@ -4,6 +4,7 @@ inputs = { utils.url = "github:numtide/flake-utils"; nixpkgs.url = "github:NixOs/nixpkgs/nixos-unstable"; + lf-benchmark-runner.url = "github:revol-xut/lf-benchmark-runner"; # input for the reactor-cpp reactor-cpp = { @@ -13,7 +14,7 @@ # source for the lingu franca compiler lingua-franca-src = { - url = "https://github.com/lf-lang/lingua-franca/releases/download/v0.1.0-beta/lfc_0.1.0-beta.tar.gz"; + url = "https://github.com/lf-lang/lingua-franca/releases/download/nightly/lfc_nightly_20220517-050749.tar.gz"; flake = false; }; @@ -30,7 +31,7 @@ }; }; - outputs = inputs@{ self, utils, nixpkgs, reactor-cpp, lingua-franca-src, lingua-franca-tests, lingua-franca-benchmarks, ... }: + outputs = { utils, nixpkgs, lf-benchmark-runner, reactor-cpp, lingua-franca-src, lingua-franca-tests, lingua-franca-benchmarks, ... }: utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; @@ -43,12 +44,13 @@ reactor-cpp-src = reactor-cpp; lingua-franca-src = lingua-franca-src; lingua-franca-benchmarks = lingua-franca-benchmarks; + lf-benchmark-runner = lf-benchmark-runner.packages."${system}".lf-benchmark-runner; }; - customPackages = pkgs.lib.mergeAttrs allTests allBenchmarks; + customPackages = pkgs.lib.mergeAttrs allBenchmarks allTests; in rec { - checks = allTests; - packages = pkgs.lib.mergeAttrs pkgs customPackages; + checks = packages; + packages = customPackages; } ); } diff --git a/nix/benchmark.nix b/nix/benchmark.nix index 803c2244..781106a3 100644 --- a/nix/benchmark.nix +++ b/nix/benchmark.nix @@ -2,9 +2,11 @@ , pkgs , lib , stdenv +, lf-benchmark-runner , reactor-cpp-src , lingua-franca-src , lingua-franca-benchmarks +, valgrind }: let @@ -72,11 +74,62 @@ let ${pkgs.coreutils}/bin/cp ${run_all}/bin/* $out/bin/ '' + install_command; }; + + # runs our custom benchmark data extractor on the specified benchmark + benchmark_command = (benchmark: "${lf-benchmark-runner}/bin/lf-benchmark-runner --target lf-cpp --binary ${benchmark}/bin/${benchmark.name} --file ./result.csv"); + + # compiles a giant script to run every benchmark and collect their results into on csv file + benchmark_commands = lib.strings.concatStringsSep "\n" (builtins.map benchmark_command list_of_derivations); + + # derivation defintion for running and collecting data from benchmarks + make-benchmark = mkDerivation { + src = ./.; + name = "make-benchmark"; + + buildPhase = benchmark_commands; + + installPhase = '' + mkdir -p $out/data/ + cp result.csv $out/data/ + ''; + }; + + # derivation for call and cachegrind. measuring a given package + profiler = (package: valgrind_check: + { + name = "${valgrind_check}-${package.name}"; + value = mkDerivation { + name = "${valgrind_check}-${package.name}"; + src = ./.; + nativeBuildInputs = [ valgrind ]; + + buildPhase = '' + ${valgrind}/bin/valgrind --tool=${valgrind_check} ${package}/bin/${package.name} + ''; + installPhase = '' + mkdir -p $out/data + cp ${valgrind_check}.out.* $out/data/${package.name}-${valgrind_check}.out + ''; + }; + } + ); + + extract_derivations = (list: lib.attrValues (lib.listToAttrs list)); + attribute_set_derivations = (library.double_map benchmarks library.compilers library.buildDerivation); + + + attribute_set_cachegrind = (builtins.map (x: profiler x "cachegrind") (extract_derivations attribute_set_derivations)); + attribute_set_callgrind = (builtins.map (x: profiler x "callgrind") (extract_derivations attribute_set_derivations)); + attribute_set_memory = (builtins.map library.memtest (extract_derivations attribute_set_derivations)); in -lib.listToAttrs ((library.double_map benchmarks library.compilers library.buildDerivation) ++ - [ - { name = "all-benchmarks"; value = all-benchmarks; } - { name = "list-benchmarks"; value = list-benchmarks; } - { name = "list-compilers"; value = library.list-compilers; } - ]) +lib.listToAttrs (attribute_set_derivations + ++ attribute_set_cachegrind + ++ attribute_set_callgrind + ++ attribute_set_memory + ++ [ + { name = "all-benchmarks"; value = all-benchmarks; } + { name = "list-benchmarks"; value = list-benchmarks; } + { name = "list-compilers"; value = library.list-compilers; } + { name = "make-benchmark"; value = make-benchmark; } +]) diff --git a/nix/lfc.nix b/nix/lfc.nix index c4c11be3..c1b30c12 100644 --- a/nix/lfc.nix +++ b/nix/lfc.nix @@ -2,7 +2,7 @@ , config , lib , mkDerivation -, jdk11_headless +, jdk17_headless , lingua-franca-src }: let @@ -19,14 +19,14 @@ mkDerivation { src = lingua-franca-src; - buildInputs = [ jdk11_headless ]; + buildInputs = [ jdk17_headless ]; - _JAVA_HOME = "${jdk11_headless}/"; + _JAVA_HOME = "${jdk17_headless}/"; postPatch = '' substituteInPlace bin/lfc \ --replace 'base=`dirname $(dirname ''${abs_path})`' "base='$out'" \ - --replace "run_lfc_with_args" "${jdk11_headless}/bin/java -jar $out/lib/jars/${extracted_name}" + --replace "run_lfc_with_args" "${jdk17_headless}/bin/java -jar $out/lib/jars/${extracted_name}" ''; buildPhase = '' diff --git a/nix/library.nix b/nix/library.nix index 9572b986..bb2b62c9 100644 --- a/nix/library.nix +++ b/nix/library.nix @@ -1,6 +1,7 @@ { fetchFromGitHub , lingua-franca-src , reactor-cpp-src +, valgrind , pkgs , lib , stdenv @@ -91,7 +92,6 @@ let }); in { - compilers = compilers; buildDerivation = buildDerivation; mkDerivation = mkDerivation; @@ -122,4 +122,28 @@ in # creates the copy command for every derivation create_install_command = (list_of_derivations: (lib.strings.concatStringsSep "\n" (builtins.map (x: "cp -r ${x}/bin/* $out/bin/") list_of_derivations))); + # checks the given package for memory leaks and exports a the result + memtest = (package: + { + name = "MLeaks-${package.name}"; + value = mkDerivation { + name = "MLeaks-${package.name}"; + src = ./.; + nativeBuildInputs = [ valgrind ]; + + buildPhase = '' + ${valgrind}/bin/valgrind --leak-check=full \ + --show-leak-kinds=all \ + --track-origins=yes \ + --verbose \ + --log-file=valgrind-out.txt \ + ${package}/bin/${package.name} + ''; + installPhase = '' + mkdir -p $out/data + cp valgrind-out.txt $out/data/${package.name}-memtest.out + ''; + }; + } + ); } diff --git a/nix/reactor-cpp.nix b/nix/reactor-cpp.nix index fc04f438..4805cd65 100644 --- a/nix/reactor-cpp.nix +++ b/nix/reactor-cpp.nix @@ -1,9 +1,10 @@ -{ pkgs, mkDerivation, cmake, gcc, reactor-cpp-src, debug}: +{ pkgs, mkDerivation, cmake, gcc, reactor-cpp-src, debug }: let buildMode = if debug then "Debug" else "Release"; -in mkDerivation { +in +mkDerivation { name = "cpp-lingua-franca-runtime"; src = reactor-cpp-src; @@ -12,7 +13,7 @@ in mkDerivation { configurePhase = '' echo "Configuration" ''; - + #TODO: remove debug build here buildPhase = '' mkdir -p build && cd build diff --git a/nix/test.nix b/nix/test.nix index 10dcc107..4e94d6e3 100644 --- a/nix/test.nix +++ b/nix/test.nix @@ -14,11 +14,6 @@ let reactor-cpp-src = reactor-cpp-src; }; - borked_tests = [ - "StructAsState.lf" - "StructAsType.lf" - ]; - # list of special derivations which cannot be run keep_alive = [ "Keepalive.lf" @@ -68,6 +63,7 @@ let ''; }; + # copies all the binaries install_command = (library.create_install_command (library.create_derivations tests)); # package that triggers a build of every test file @@ -80,11 +76,22 @@ let ${pkgs.coreutils}/bin/cp ${run_all}/bin/* $out/bin/ '' + install_command; }; + + # function that takes a list of attributes [ { name = "a"; value = "b";}] and transforms it into + # a list with all the values which are derivations in that case: e.g. ["b"] + extract_derivations = (list: lib.attrValues (lib.listToAttrs list)); + + # takes the cartisean product of packages and compilers + attribute_set_derivations = (library.double_map tests library.compilers library.buildDerivation); + + # creates for every package a memtest version for debug purposes + attribute_set_memory = (builtins.map library.memtest (extract_derivations attribute_set_derivations)); in -lib.listToAttrs ((library.double_map tests library.compilers library.buildDerivation) ++ - [ - { name = "all-tests"; value = all-tests; } - { name = "list-tests"; value = list-tests; } - { name = "list-compilers"; value = library.list-compilers; } - ]) +lib.listToAttrs (attribute_set_derivations + ++ attribute_set_memory + ++ [ + { name = "all-tests"; value = all-tests; } + { name = "list-tests"; value = list-tests; } + { name = "list-compilers"; value = library.list-compilers; } +])