Skip to content

Commit

Permalink
Relative Benchmarking
Browse files Browse the repository at this point in the history
  • Loading branch information
epompeii committed Dec 15, 2023
1 parent 7989b07 commit 37a2bc0
Show file tree
Hide file tree
Showing 3 changed files with 132 additions and 32 deletions.
86 changes: 86 additions & 0 deletions .github/workflows/benches.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
on:
push:
branches: does_not_exist
pull_request_target:

name: Benchmarks

jobs:
benchmarks:
if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
backend: ["postgres", "sqlite", "mysql"]
env:
BENCHER_PROJECT: diesel
BENCHER_ADAPTER: rust_criterion
steps:
- name: Checkout sources
uses: actions/checkout@v3

- name: Install postgres (Linux)
if: matrix.backend == 'postgres'
run: |
sudo apt-get update
sudo apt-get install -y libpq-dev postgresql
echo "host all all 127.0.0.1/32 md5" > sudo tee -a /etc/postgresql/10/main/pg_hba.conf
sudo service postgresql restart && sleep 3
sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
sudo service postgresql restart && sleep 3
echo 'DATABASE_URL=postgres://postgres:postgres@localhost/' >> $GITHUB_ENV
- name: Install sqlite (Linux)
if: matrix.backend == 'sqlite'
run: |
sudo apt-get update
sudo apt-get install -y libsqlite3-dev
echo 'DATABASE_URL=/tmp/test.db' >> $GITHUB_ENV
- name: Install mysql (Linux)
if: matrix.backend == 'mysql'
run: |
sudo systemctl start mysql.service
sudo apt-get update
sudo apt-get -y install libmysqlclient-dev
mysql -e "create database diesel_test; create database diesel_unit_test; grant all on \`diesel_%\`.* to 'root'@'localhost';" -uroot -proot
echo 'DATABASE_URL=mysql://root:root@localhost/diesel_test' >> $GITHUB_ENV
- name: Install rust toolchain
uses: dtolnay/rust-toolchain@stable

- name: Install Bencher CLI
uses: bencherdev/bencher@main

- name: Benchmark master
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
env:
BENCHER_BRANCH: master
BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }}
run: |
bencher run \
--token '${{ secrets.BENCHER_API_TOKEN }}' \
'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}"'
- name: Checkout PR sources
if: github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
persist-credentials: false

- name: Benchmark PR ${{ github.event.pull_request.number }}
if: github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
env:
BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }}
run: |
bencher run \
--if-branch '${{ github.event.pull_request.head.ref }}' \
--else-if-branch '${{ github.event.pull_request.base.ref }}' \
--else-if-branch master \
--err \
--github-actions '${{ secrets.GITHUB_TOKEN }}' \
--token '${{ secrets.BENCHER_API_TOKEN }}' \
'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}"'
29 changes: 21 additions & 8 deletions .github/workflows/run_benches.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
on:
push:
branches: master
pull_request:
types: [labeled, opened, reopened, synchronize]

name: Run and Cache Benchmarks

jobs:
run_benchmarks:
if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks'))
if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
runs-on: ubuntu-latest
strategy:
fail-fast: false
Expand Down Expand Up @@ -48,14 +46,29 @@ jobs:
- name: Install rust toolchain
uses: dtolnay/rust-toolchain@stable

- name: Benchmark ${{ matrix.backend }}
run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > ${{ matrix.backend }}.txt
- name: Benchmark PR ${{ matrix.backend }}
run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > pr_${{ matrix.backend }}.txt

- name: Upload ${{ matrix.backend }} Benchmark Results
- name: Checkout base branch
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.base.sha }}
repository: ${{ github.event.pull_request.base.repo.full_name }}

- name: Benchmark base ${{ matrix.backend }}
run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > base_${{ matrix.backend }}.txt

- name: Upload PR ${{ matrix.backend }} Benchmark Results
uses: actions/upload-artifact@v3
with:
name: pr_${{ matrix.backend }}.txt
path: ./pr_${{ matrix.backend }}.txt

- name: Upload base ${{ matrix.backend }} Benchmark Results
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.backend }}.txt
path: ./${{ matrix.backend }}.txt
name: base_${{ matrix.backend }}.txt
path: ./base_${{ matrix.backend }}.txt

- name: Upload GitHub Event
uses: actions/upload-artifact@v3
Expand Down
49 changes: 25 additions & 24 deletions .github/workflows/track_benches.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ jobs:
BENCHER_PROJECT: diesel
BENCHER_ADAPTER: rust_criterion
BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }}
BENCHMARK_RESULTS: ${{ matrix.backend }}.txt
PR_BENCHMARK_RESULTS: pr_${{ matrix.backend }}.txt
BASE_BENCHMARK_RESULTS: base_${{ matrix.backend }}.txt
GITHUB_EVENT: event.json
steps:
- name: Download Benchmark Results
Expand Down Expand Up @@ -46,11 +47,13 @@ jobs:
let fs = require('fs');
fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data));
}
await downloadArtifact(process.env.BENCHMARK_RESULTS);
await downloadArtifact(process.env.PR_BENCHMARK_RESULTS);
await downloadArtifact(process.env.BASE_BENCHMARK_RESULTS);
await downloadArtifact(process.env.GITHUB_EVENT);
- name: Unzip Benchmark Results
run: |
unzip $BENCHMARK_RESULTS.zip
unzip $PR_BENCHMARK_RESULTS.zip
unzip $BASE_BENCHMARK_RESULTS.zip
unzip $GITHUB_EVENT.zip
- name: Export GitHub Event Data
uses: actions/github-script@v6
Expand All @@ -59,32 +62,30 @@ jobs:
let fs = require('fs');
let githubEvent = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT, {encoding: 'utf8'}));
console.log(githubEvent);
if (githubEvent.ref) {
core.exportVariable("EVENT_NAME", "push");
core.exportVariable("PUSH_REF", githubEvent.ref);
} else {
core.exportVariable("EVENT_NAME", "pull_request");
core.exportVariable("PR_HEAD", githubEvent.pull_request.head.ref);
core.exportVariable("PR_BASE", githubEvent.pull_request.base.ref);
core.exportVariable("PR_DEFAULT", githubEvent.pull_request.base.repo.default_branch);
core.exportVariable("PR_NUMBER", githubEvent.number);
}
core.exportVariable("PR_HEAD", `${githubEvent.pull_request.head.ref}-${githubEvent.pull_request.head.sha.slice(0, 8)}`);
core.exportVariable("PR_NUMBER", githubEvent.number);
- uses: bencherdev/bencher@main
- name: Track Benchmarks
- name: Track base Benchmarks
run: |
(${{ env.EVENT_NAME == 'push' && env.PUSH_REF == 'refs/heads/master' }} && \
bencher run \
--branch "master" \
--if-branch '${{ env.PR_HEAD }}' \
--else-branch
--token "${{ secrets.BENCHER_API_TOKEN }}" \
--err \
--file "$BENCHMARK_RESULTS") || \
(${{ env.EVENT_NAME == 'pull_request'}} && \
--file "$BASE_BENCHMARK_RESULTS"
- name: Create PR threshold
run: |
bencher threshold create \
--branch '${{ env.PR_HEAD }}' \
--testbed "$BENCHER_TESTBED" \
--measure latency \
--test t \
--upper-boundary 0.98
- name: Track PR Benchmarks
run: |
bencher run \
--if-branch '${{ env.PR_HEAD }}' \
--else-if-branch '${{ env.PR_BASE }}' \
--else-if-branch '${{ env.PR_DEFAULT }}' \
--branch '${{ env.PR_HEAD }}' \
--token "${{ secrets.BENCHER_API_TOKEN }}" \
--ci-number '${{ env.PR_NUMBER }}' \
--github-actions "${{ secrets.GITHUB_TOKEN }}" \
--token "${{ secrets.BENCHER_API_TOKEN }}" \
--err \
--file "$BENCHMARK_RESULTS")
--file "$PR_BENCHMARK_RESULTS"

0 comments on commit 37a2bc0

Please sign in to comment.