Fix performance regression detection in CI #1248 (#1259)

Co-authored-by: toseven <[email protected]>
Co-authored-by: Dominik Maier <[email protected]>

.github/workflows/build_and_test.yml

@@ -225,6 +225,32 @@ jobs:
       if: runner.os == 'macOS' # use bash v4
       run: /usr/local/bin/bash -c 'RUN_ON_CI=1 ./scripts/test_all_fuzzers.sh'
 
+  executions-check:
+    strategy:
+      matrix:
+          os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: stable
+    - uses: lyricwulf/abc@v1
+      with: 
+        linux: llvm llvm-dev clang 
+        macos: llvm bash coreutils
+    - uses: actions/checkout@v3
+      with:
+        submodules: true # recursively checkout submodules
+        fetch-depth: 0
+    - uses: Swatinem/rust-cache@v2
+    - name: Build and run libfuzzer_libpng (Linux)
+      if: runner.os == 'Linux'
+      run: ./scripts/executions-check.sh 
+    - name: Build and run libfuzzer_libpng (macOS) 
+      if: runner.os == 'macOS'
+      run: /usr/local/bin/bash -c './scripts/executions-check.sh'
+
   nostd-build:
     runs-on: ubuntu-latest
     steps:

scripts/executions-check.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+install_libpng() {
+	cd ./fuzzers/libfuzzer_libpng && wget https://deac-fra.dl.sourceforge.net/project/libpng/libpng16/1.6.37/libpng-1.6.37.tar.xz 
+	tar -xvf libpng-1.6.37.tar.xz || echo "Failed to download libpng" 
+	cd libpng-1.6.37 && ./configure --enable-shared=no --with-pic=yes --enable-hardware-optimizations=yes && cd ..
+}
+
+build_libpng(){
+	cargo build --release || echo "ERROR: Failed to build libfuzzer_libpng" 
+
+	cd libpng-1.6.37 && make CC="$(pwd)/../target/release/libafl_cc" CXX="$(pwd)/../target/release/ libafl_cxx" -j "$(nproc)" && cd ..
+}
+
+git_checkout(){
+	git reset --hard HEAD^
+}
+
+build_run_fuzzer(){
+	./target/release/libafl_cxx ./harness.cc libpng-1.6.37/.libs/libpng16.a -I libpng-1.6.37/ -o fuzzer_libpng -lz -lm || exit 2
+
+	./fuzzer_libpng > log.txt &
+
+    # wait that fuzzer_libpng become the broker 
+    sleep 1
+
+	timeout 5m ./fuzzer_libpng > /dev/null 2>&1 &
+
+	while true; do
+		if grep -q "Broker" log.txt ; then
+			pkill -9 "fuzzer_libpng"
+			executions=$(grep  -m 1 "Broker" log.txt  | awk '{print $14}')
+			rm -rf ./libafl_unix_shmem_server
+			echo "${executions%,}"
+			break
+   		fi
+	done
+}
+
+main(){
+	install_libpng
+
+	build_libpng
+	echo "start to run the new fuzzer"
+	new_executions=$(build_run_fuzzer)
+
+	git_checkout
+
+	build_libpng
+	echo "start to run the last fuzzer"
+	last_executions=$(build_run_fuzzer)	
+
+	echo "the execution count of the new fuzzer is $new_executions"
+	echo "the execution count of the last fuzzer is $last_executions"
+}
+
+main