chore: add historical performance benchmarking (#1432)

.github/workflows/performance.yaml

@@ -6,13 +6,13 @@ on:
       - main
 
 jobs:
-  hyperfine:
+  latest-vs-next:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
           cache: 'npm'
       - name: Install Dependencies
         run: npm ci
@@ -38,4 +38,41 @@ jobs:
         uses: thollander/actions-comment-pull-request@v2
         with:
           filePath: benchmark_check.md
-          comment_tag: hyperfine-comparison
+          comment_tag: latest-vs-next-comparison
+
+  historical-versions:
+    # Run only on release branch (changeset-release/main):
+    if: github.head_ref == 'changeset-release/main'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: 'npm'
+      - name: Install Dependencies
+        run: npm ci
+      - name: Install External
+        run: npm i -g hyperfine
+      - name: Prepare
+        run: |
+          npm run compile
+          npm run pack:prepare
+          cd benchmark/
+          npm i
+          npm run make-test
+      - name: Run Benchmark
+        run: |
+          cd benchmark/
+          npm test
+          cat benchmark_check.md
+        env:
+          CI: true
+          REDOCLY_TELEMETRY: off
+
+      - name: Comment PR
+        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          filePath: benchmark/benchmark_check.md
+          comment_tag: historical-versions-comparison

.github/workflows/release.yaml

@@ -199,7 +199,7 @@ jobs:
 
       - name: Send to a Slack channel
         id: slack
-        uses: slackapi/slack-github-action@v1.23.0
+        uses: slackapi/slack-github-action@v1.25.0
         with:
           channel-id: C019K52TC0L #releases
           payload-file-path: './output/release-message.json'

.github/workflows/tests.yaml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
           cache: 'npm'
       - name: Install dependencies
         run: npm ci
@@ -35,7 +35,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
           cache: 'npm'
       - name: Install dependencies
         run: npm ci
@@ -56,7 +56,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
           cache: 'npm'
       - name: Coverage Report
         uses: artiomtr/[email protected]
@@ -72,7 +72,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
           cache: 'npm'
       - name: Install dependencies
         run: npm ci
@@ -86,7 +86,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
           cache: 'npm'
       - name: Install dependencies
         run: npm ci

benchmark/.gitignore

@@ -0,0 +1,6 @@
+api-definitions
+node_modules
+package-lock.json
+test-command.txt
+benchmark_check.md
+benchmark_check.json

benchmark/README.md

@@ -0,0 +1,4 @@
+# Historical versions benchmark
+
+This directory is for automated benchmark testing of the existing versions of **Redocly CLI**.
+After releasing a new version, please add its reference to the `dependencies` section of the [package.json](./package.json) file.

benchmark/make-test-command.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -eo pipefail # Fail on script errors
+
+# Clone the repo with test openapi files:
+git clone https://github.com/Rebilly/api-definitions.git
+
+# Store the command into a text file:
+echo hyperfine -i --warmup 3 $(cat package.json | jq '.dependencies' | jq 'keys' | jq 'map("'\''node node_modules/" + . + "/bin/cli.js lint api-definitions/openapi/openapi.yaml'\''")' | jq 'join(" ")' | xargs) --export-markdown benchmark_check.md --export-json benchmark_check.json > test-command.txt
+
+# Put the command in the test section of the package.json:
+cat package.json | jq ".scripts.test = \"$(cat test-command.txt)\"" > package.json

benchmark/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "benchmark",
+  "version": "1.0.0",
+  "description": "Test benchmark for Redocly CLI",
+  "scripts": {
+    "make-test": "bash make-test-command.sh"
+  },
+  "dependencies": {
+    "cli-100": "npm:@redocly/[email protected]",
+    "cli-110": "npm:@redocly/[email protected]",
+    "cli-120": "npm:@redocly/[email protected]",
+    "cli-130": "npm:@redocly/[email protected]",
+    "cli-140": "npm:@redocly/[email protected]",
+    "cli-150": "npm:@redocly/[email protected]",
+    "cli-160": "npm:@redocly/[email protected]",
+    "cli-170": "npm:@redocly/[email protected]",
+    "cli-180": "npm:@redocly/[email protected]",
+    "cli-190": "npm:@redocly/[email protected]",
+    "cli-next": "file:../redocly-cli.tgz"
+  }
+}