From d7921ee1ded350738339aa382131f3534808a985 Mon Sep 17 00:00:00 2001 From: Jonathan Bell Date: Wed, 7 Aug 2024 14:32:38 -0400 Subject: [PATCH 1/2] Fix ant --- scripts/experiments/get_coverage.sh | 2 +- scripts/experiments/reproWithJacoco.sh | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/experiments/get_coverage.sh b/scripts/experiments/get_coverage.sh index d54e772e9..d81f152c1 100644 --- a/scripts/experiments/get_coverage.sh +++ b/scripts/experiments/get_coverage.sh @@ -46,7 +46,7 @@ export expandedCP=$(join_by ":" $deps) if [ -d "$2" ]; then rm -Rf $2; fi -export jacocoJars="$ROOT_DIR/examples/target/dependency/org.jacoco.report-0.8.10.jar:$ROOT_DIR/examples/target/dependency/org.jacoco.core-0.8.10.jar" +export jacocoJars="$ROOT_DIR/examples/target/dependency/org.jacoco.report-0.8.7.jar:$ROOT_DIR/examples/target/dependency/org.jacoco.core-0.8.7.jar" export cmd="java -cp $JACOCO_UTIL_JAR:$jacocoJars -DJACOCO_SOURCES=$JACOCO_SOURCES fun.jvm.jacoco.reachability.entry.TolerantJacocoReportBuilder $1 $ROOT_DIR/examples/target/test-classes:$expandedCP $2 $3" COVERAGE_JSON=`$cmd | tail -n 1` diff --git a/scripts/experiments/reproWithJacoco.sh b/scripts/experiments/reproWithJacoco.sh index aa89cfb96..eb76fb517 100644 --- a/scripts/experiments/reproWithJacoco.sh +++ b/scripts/experiments/reproWithJacoco.sh @@ -18,6 +18,11 @@ if [ "$1" = "-i" ]; then shift 1 fi +JACOCO_SOURCES=$ROOT_DIR/examples/target/dependency-sources +if [ ! -d $JACOCO_SOURCES ]; then + (cd $ROOT_DIR/examples && mvn -q dependency:unpack-dependencies -Dclassifier=sources -DincludeArtifactIds=maven-model,closure-compiler,rhino,ant,bcel -DoutputDirectory=target/dependency-sources) +fi + class="$1" method="$2" JACOCO_JAR=$ROOT_DIR/target/jacocoagent.jar @@ -26,7 +31,19 @@ if [ ! -f $JACOCO_JAR ]; then (cd $ROOT_DIR/target && unzip org.jacoco.agent-0.8.7.jar) fi +if [ -f $ROOT_DIR/examples/target/dependency/org.jacoco.report-0.8.10.jar ]; then + rm $ROOT_DIR/examples/target/dependency/org.jacoco.report-0.8.10.jar + mvn -q dependency:get -Dartifact=org.jacoco:org.jacoco.report:0.8.7 + mvn -q dependency:copy -Dartifact=org.jacoco:org.jacoco.report:0.8.7 -DoutputDirectory=$ROOT_DIR/examples/target/dependency/ +fi +if [ -f $ROOT_DIR/examples/target/dependency/org.jacoco.core-0.8.10.jar ]; then + rm $ROOT_DIR/examples/target/dependency/org.jacoco.core-0.8.10.jar + mvn -q dependency:get -Dartifact=org.jacoco:org.jacoco.core:0.8.7 + mvn -q dependency:copy -Dartifact=org.jacoco:org.jacoco.core:0.8.7 -DoutputDirectory=$ROOT_DIR/examples/target/dependency/ +fi echo $JACOCO_JAR +# Chocopy contains a copy of ant, so we need to exclude it when processing coverage for ant... +rm -f $ROOT_DIR/examples/target/dependency/chocopy* export CLASSPATH="$ROOT_DIR/examples/target/classes/:$ROOT_DIR/examples/target/test-classes/:$ROOT_DIR/examples/target/dependency/*" export JVM_OPTS="-javaagent:$JACOCO_JAR=destfile=$3,includes=$4" From b40c0a23236136f072ff2f0ff71c04b01a9c8b1f Mon Sep 17 00:00:00 2001 From: Jonathan Bell Date: Thu, 8 Aug 2024 10:13:42 -0400 Subject: [PATCH 2/2] Update workflow to compute jacoco results per-run --- .github/workflows/evaluation-workflow.yml | 38 +++++++++++++---------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/evaluation-workflow.yml b/.github/workflows/evaluation-workflow.yml index ec8fc563b..7105b979d 100644 --- a/.github/workflows/evaluation-workflow.yml +++ b/.github/workflows/evaluation-workflow.yml @@ -60,13 +60,13 @@ jobs: matrix-analyze: ${{ steps.set-matrix.outputs.matrix-analyze }} base-url: ${{ steps.set-matrix.outputs.base-url }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set matrix for evaluation id: set-matrix run: | - echo "::set-output name=matrix-run::$( python3 scripts/experiments/generate-experiment-matrix.py ${{ inputs.trials }} )" - echo "::set-output name=matrix-analyze::$( python3 scripts/experiments/generate-experiment-matrix.py )" - echo "::set-output name=base-url::https://ci.in.ripley.cloud/logs/public/${{ github.repository }}/${{ github.sha }}/$(./scripts/experiments/urlencode.sh "${{ github.workflow }}")/${{ github.run_id }}/${{ github.run_attempt }}/site/" + echo "matrix-run=$( python3 scripts/experiments/generate-experiment-matrix.py ${{ inputs.trials }} )" >> $GITHUB_OUTPUT + echo "matrix-analyze=$( python3 scripts/experiments/generate-experiment-matrix.py )" >> $GITHUB_OUTPUT + echo "base-url=https://ci.in.ripley.cloud/logs/public/${{ github.repository }}/${{ github.sha }}/$(./scripts/experiments/urlencode.sh "${{ github.workflow }}")/${{ github.run_id }}/${{ github.run_attempt }}/site/" >> $GITHUB_OUTPUT run-fuzzer: runs-on: ${{ inputs.runs_on }} timeout-minutes: 1560 @@ -79,7 +79,7 @@ jobs: strategy: matrix: ${{fromJson(needs.build-matrix.outputs.matrix-run)}} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Telegraf config uses: DamianReeves/write-file-action@v1.0 with: @@ -226,7 +226,7 @@ jobs: DURATION: ${{ inputs.duration }} JAVA_HOME: ${{ inputs.java_home }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/download-artifact@v2 - id: repro name: Repro @@ -242,19 +242,25 @@ jobs: coveragePackages=$(jq -r ".coveragePackages" <<< $value) echo $id echo $coveragePackages + find . -maxdepth 2 -type f -name "${id}_failures_*" -exec mv {} . \; + find . -maxdepth 1 -type f -name "${id}_failures_*.tgz" -exec tar xzf {} \; find . -maxdepth 2 -type f -name "${id}_corpus_*" -exec mv {} . \; find . -maxdepth 1 -type f -name "${id}_corpus_*.tgz" -exec tar xzf {} \; find . -maxdepth 1 -name "*${id}_*.tgz" -exec rm -rf {} \; rm -f jacoco.exec - - bash scripts/experiments/reproWithJacoco.sh $class $method jacoco.exec "$coveragePackages" "${id}_corpus_*" - bash scripts/experiments/get_coverage.sh jacoco.exec "$coveragePackages" jacoco-$id - JACOCO_SUMMARY+="\"${id}\": $(cat jacoco_summary.json)," - mv jacoco_summary.json "$RESULTS_DIR/artifacts/${id}_jacoco_summary.json" - cp -r jacoco-$id "$RESULTS_DIR/" + for i in $(seq 0 $((TRIALS-1))) + do + echo "Processing $TARGET trial $i" + # run the script + bash scripts/experiments/reproWithJacoco.sh $class $method jacoco.exec "$coveragePackages" "${id}_corpus_$i" "${id}_failures_$i" + bash scripts/experiments/get_coverage.sh jacoco.exec "$coveragePackages" jacoco-$id-$i + JACOCO_SUMMARY+="\"${id}-${i}\": $(cat jacoco_summary.json)," + mv jacoco_summary.json "$RESULTS_DIR/artifacts/${id}_${i}_jacoco_summary.json" + cp -r jacoco-$id-$i "$RESULTS_DIR/" + done done JACOCO_SUMMARY+='"site_url":"$BASE_URL"}' - echo "::set-output name=jacoco-summary::$JACOCO_SUMMARY" + echo "jacoco-summary=$JACOCO_SUMMARY" >> $GITHUB_OUTPUT mv "$RESULTS_DIR"/jacoco* "$RESULTS_DIR/artifacts/" build-site: runs-on: self-hosted @@ -263,10 +269,10 @@ jobs: RENV_PATHS_ROOT: /ci-logs/renv PROFILE_HEAP: ${{ inputs.PROFILE_HEAP }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: repository: 'jon-bell/fuzzing-build-site-action' - ref: 'main' + ref: 'v2' - name: Fetch workflow runs to compare to id: fetch-compare uses: jon-bell/list-workflow-runs-by-branch-action@main @@ -274,7 +280,7 @@ jobs: include_branches: ${{ inputs.report_on_branches}} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build site - uses: jon-bell/fuzzing-build-site-action@main + uses: jon-bell/fuzzing-build-site-action@v2 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} comparisons: ${{ steps.fetch-compare.outputs.workflow_runs }} \ No newline at end of file