CI: Ensure immutability of test cases

.github/workflows/ci.yml

@@ -15,6 +15,37 @@ jobs:
 
       - name: All required files are present
         run: bin/check_required_files_present
+
+  immutability:
+    name: No test has been mutated
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: 'new'
+
+      # Pull Requests
+      - name: Checkout the target branch (PRs)
+        uses: actions/checkout@v2
+        if: github.event_name == 'pull_request'
+        with:
+          ref: '${{ github.base_ref }}'
+          path: 'old'
+
+      # Pushes
+      - name: Checkout the previous commit (Pushes)
+        uses: actions/checkout@v2
+        if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+        with:
+          ref: '${{ github.sha }}~'
+          path: 'old'
+
+      - name: Check that no test has been mutated
+        run: |
+          for oldf in old/exercises/*/canonical-data.json; do
+            newf=${oldf//old/new}
+            ./new/bin/check-immutability.py "$oldf" "$newf"
+          done
 
   schema-validation:
     name: Schema validation

bin/check-immutability.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python 
+
+import json
+import subprocess
+import sys
+
+oldf = sys.argv[1]
+newf = sys.argv[2]
+
+immutable_keys = ('property', 'input', 'expected')
+
+# Use jq to flatten the test data, and parse it
+old = json.loads(subprocess.run([f"jq -r '[.. | objects | select(.uuid != null)]' {oldf}"], stdout=subprocess.PIPE, shell=True).stdout.decode('utf-8'))
+new = json.loads(subprocess.run([f"jq -r '[.. | objects | select(.uuid != null)]' {newf}"], stdout=subprocess.PIPE, shell=True).stdout.decode('utf-8'))
+
+# Convert new to dict uuid => case
+new = {c['uuid']: c for c in new}
+
+fails = set()
+deleted = set()
+
+# Iterate through old cases as only those could potentially be mutated
+for case in old:
+    uuid = case['uuid']
+
+    # Check if the case has been deleted
+    if uuid not in new:
+        deleted.add(uuid)
+        continue
+
+    # Check that scenarios are only updated additively
+    if 'scenarios' in case and not set(case['scenarios']).issubset(set(new[uuid]['scenarios'])):
+        fails.add(uuid)
+        continue
+    # Check for changes to immutable keys
+    for k in immutable_keys:
+        if case[k] != new[uuid][k]:
+            fails.add(uuid)
+            break
+
+if len(fails) == 0 and len(deleted) == 0:
+    exit(0)
+
+if len(fails) > 0:
+    print('The following tests contain illegal mutations:')
+    for f in fails:
+        print(f" - {f} ({new[f]['description']})")
+
+if len(deleted) > 0:
+    print('The following tests have been deleted illegally:')
+    for d in deleted:
+        print(f" - {d}")
+
+exit(1)