CI: Ensure immutability of test cases

.github/workflows/ci.yml

@@ -15,6 +15,37 @@ jobs:
 
       - name: All required files are present
         run: bin/check_required_files_present
+
+  immutability:
+    name: No test has been mutated
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: 'new'
+
+      # Pull Requests
+      - name: Checkout the target branch (PRs)
+        uses: actions/checkout@v2
+        if: github.event_name == 'pull_request'
+        with:
+          ref: '${{ github.base_ref }}'
+          path: 'old'
+
+      # Pushes
+      - name: Checkout the previous commit (Pushes)
+        uses: actions/checkout@v2
+        if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+        with:
+          ref: '${{ github.sha }}~'
+          path: 'old'
+
+      - name: Check that no test has been mutated
+        run: |
+          for oldf in old/exercises/*/canonical-data.json; do
+            newf=${oldf//old/new}
+            ./new/bin/check-immutability.py "$oldf" "$newf"
+          done
 
   schema-validation:
     name: Schema validation

bin/check-immutability.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python 
+
+import json
+import subprocess
+import sys
+
+oldf = sys.argv[1]
+newf = sys.argv[2]
+
+immutable_keys = ('property', 'input', 'expected')
+
+# Use jq to flatten the test data, and parse it
+old = json.loads(subprocess.run([f"jq -r '[.. | objects | select(.uuid != null)]' {oldf}"], stdout=subprocess.PIPE, shell=True).stdout.decode('utf-8'))
+new = json.loads(subprocess.run([f"jq -r '[.. | objects | select(.uuid != null)]' {newf}"], stdout=subprocess.PIPE, shell=True).stdout.decode('utf-8'))
+
+# Convert new to dict uuid => case
+new = {c['uuid']: c for c in new}
+
+fails = set()
+
+# Iterate through old cases as only those could potentially be mutated
+for case in old:
+    for k in immutable_keys:
+        if case[k] != new[case['uuid']][k]:
+            fails.add(case['uuid'])
+            break
+
+if len(fails) == 0:
+    exit(0)
+
+print('The following tests contain illegal mutations:')
+for f in fails:
+    print(f" - {f} ({new[f]['description']})")
+exit(1)