Added benchmark luau support (#123)

* added benchmark luau support * added unit tests for luau benchmarks output * updated README.md with the new tool
benchmark-action · May 11, 2022 · f6baee1 · f6baee1
1 parent 4cd86fc
commit f6baee1
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 1 deletion.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -19,6 +19,7 @@
       },
       "plugins": ["@typescript-eslint", "prettier"],
       "rules": {
+        "@typescript-eslint/no-unused-vars": ["warn", { "varsIgnorePattern": "^_" }],
         "@typescript-eslint/naming-convention": "error",
         "@typescript-eslint/no-floating-promises": "error",
         "@typescript-eslint/no-unnecessary-type-arguments": "error",

diff --git a/README.md b/README.md
@@ -24,6 +24,7 @@ This action currently supports the following tools:
 - [Catch2][catch2] for C++ projects
 - [BenchmarkTools.jl][] for Julia packages
 - [Benchmark.Net][benchmarkdotnet] for .Net projects
+- [benchmarkluau](https://github.com/Roblox/luau/tree/master/bench) for Luau projects
 - Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter'
 
 Multiple languages in the same repository are supported for polyglot projects.
@@ -47,6 +48,7 @@ definitions are in [.github/workflows/](./.github/workflows) directory. Live wor
 | C++ (Catch2) | [![C++ Catch2 Example Workflow][catch2-badge]][catch2-workflow-example]                 | [examples/catch2](./examples/catch2)           |
 | Julia | [![Julia Example][julia-badge]][julia-workflow-example]                 | [examples/julia](./examples/julia)           |
 | .Net         | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) |
+| Luau         | Coming soon | Coming soon |
 
 All benchmark charts from above workflows are gathered in GitHub pages:
 
@@ -330,6 +332,7 @@ and store it to file. Then specify the file path to `output-file-path` input.
 - [catch2 for C++ projects](./examples/cpp/README.md)
 - [BenchmarkTools.jl for Julia projects](./examples/julia/README.md)
 - [Benchmark.Net for .Net projects](./examples/benchmarkdotnet/README.md)
+- [benchmarkluau for Luau projects](#) - Examples for this are still a work in progress.
 
 These examples are run in workflows of this repository as described in the 'Examples' section above.
 
@@ -351,7 +354,7 @@ Name of the benchmark. This value must be identical across all benchmarks in you
 - Default: N/A
 
 Tool for running benchmark. The value must be one of `"cargo"`, `"go"`, `"benchmarkjs"`, `"pytest"`,
-`"googlecpp"`, `"catch2"`, `"julia"`, `"benchmarkdotnet"`, `"customBiggerIsBetter"`, `"customSmallerIsBetter"`.
+`"googlecpp"`, `"catch2"`, `"julia"`, `"benchmarkdotnet"`,`"benchmarkluau"`, `"customBiggerIsBetter"`, `"customSmallerIsBetter"`.
 
 #### `output-file-path` (Required)
 

diff --git a/src/config.ts b/src/config.ts
@@ -7,6 +7,7 @@ export type ToolType =
     | 'cargo'
     | 'go'
     | 'benchmarkjs'
+    | 'benchmarkluau'
     | 'pytest'
     | 'googlecpp'
     | 'catch2'

diff --git a/src/default_index_html.ts b/src/default_index_html.ts
@@ -114,6 +114,7 @@ export const DEFAULT_INDEX_HTML = String.raw`<!DOCTYPE html>
           cargo: '#dea584',
           go: '#00add8',
           benchmarkjs: '#f1e05a',
+          benchmarkluau: '#000080',
           pytest: '#3572a5',
           googlecpp: '#f34b7d',
           catch2: '#f34b7d',

diff --git a/src/extract.ts b/src/extract.ts
@@ -593,6 +593,27 @@ function extractCustomBenchmarkResult(output: string): BenchmarkResult[] {
     }
 }
 
+function extractLuauBenchmarkResult(output: string): BenchmarkResult[] {
+    const lines = output.split(/\n/);
+    const results: BenchmarkResult[] = [];
+
+    output;
+    for (const line of lines) {
+        if (!line.startsWith('SUCCESS')) continue;
+        const [_0, name, _2, valueStr, _4, range, _6, extra] = line.split(/\s+/);
+
+        results.push({
+            name: name,
+            value: parseFloat(valueStr),
+            unit: valueStr.replace(/.[0-9]+/g, ''),
+            range: `±${range}`,
+            extra: extra,
+        });
+    }
+
+    return results;
+}
+
 export async function extractResult(config: Config): Promise<Benchmark> {
     const output = await fs.readFile(config.outputFilePath, 'utf8');
     const { tool, githubToken } = config;
@@ -629,6 +650,9 @@ export async function extractResult(config: Config): Promise<Benchmark> {
         case 'customSmallerIsBetter':
             benches = extractCustomBenchmarkResult(output);
             break;
+        case 'benchmarkluau':
+            benches = extractLuauBenchmarkResult(output);
+            break;
         default:
             throw new Error(`FATAL: Unexpected tool: '${tool}'`);
     }

diff --git a/src/write.ts b/src/write.ts
@@ -64,6 +64,8 @@ function biggerIsBetter(tool: ToolType): boolean {
             return false;
         case 'benchmarkjs':
             return true;
+        case 'benchmarkluau':
+            return false;
         case 'pytest':
             return true;
         case 'googlecpp':

diff --git a/test/data/extract/benchmarkluau_output.txt b/test/data/extract/benchmarkluau_output.txt
@@ -0,0 +1,12 @@
+SUCCESS: base64                                  :   15.041ms +/-  0.636% on luau
+SUCCESS: chess                                   :   69.560ms +/-  0.212% on luau
+SUCCESS: life                                    :   85.089ms +/-  0.187% on luau
+
+==================================================RESULTS==================================================
+Test                   | Min         | Average     | StdDev%    | Driver 
+-----------------------+-------------+-------------+------------+--------
+base64                 |   14.831ms  |   15.041ms  |    0.636%  | luau   
+chess                  |   69.106ms  |   69.560ms  |    0.212%  | luau   
+life                   |   84.696ms  |   85.089ms  |    0.187%  | luau     
+Total                  |  168.630ms  |  169.690ms  |       ---  | luau   
+---
diff --git a/test/extract.spec.ts b/test/extract.spec.ts
@@ -227,6 +227,32 @@ describe('extractResult()', function () {
                 },
             ],
         },
+        {
+            tool: 'benchmarkluau',
+            expected: [
+                {
+                    name: 'base64',
+                    range: '±0.636%',
+                    unit: 'ms',
+                    value: 15.041,
+                    extra: 'luau',
+                },
+                {
+                    name: 'chess',
+                    range: '±0.212%',
+                    unit: 'ms',
+                    value: 69.56,
+                    extra: 'luau',
+                },
+                {
+                    name: 'life',
+                    range: '±0.187%',
+                    unit: 'ms',
+                    value: 85.089,
+                    extra: 'luau',
+                },
+            ],
+        },
         {
             tool: 'pytest',
             expected: [