Report meaningful pipeline status

.github/workflows/nrn-modeldb-ci.yaml

@@ -162,7 +162,14 @@ jobs:
           python -m pip install $NEURON_V1
         fi
         nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"`
-        runmodels --workdir=$nrn_ver $MODELS_TO_RUN
+        ps uxf # debug
+        runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN
+        # Filter out the gout data before generating HTML reports. The HTML
+        # diff uses the original gout files on disk anyway. Compress the large
+        # JSON file including gout data for inclusion in the artifacts
+        mv ${nrn_ver}.json ${nrn_ver}-full.json
+        jq -r 'del(.[].gout)' ${nrn_ver}-full.json > ${nrn_ver}.json
+        xz ${nrn_ver}-full.json
         report2html ${nrn_ver}.json
         if [[ -d "${DROP_DIR_V1}" ]]; then
           python -m pip uninstall --yes neuron-nightly==${nrn_ver}
@@ -182,7 +189,14 @@ jobs:
           python -m pip install $NEURON_V2
         fi
         nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"`
-        runmodels --workdir=$nrn_ver $MODELS_TO_RUN
+        ps uxf # debug
+        runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN
+        # Filter out the gout data before generating HTML reports. The HTML
+        # diff uses the original gout files on disk anyway. Compress the large
+        # JSON file including gout data for inclusion in the artifacts
+        mv ${nrn_ver}.json ${nrn_ver}-full.json
+        jq -r 'del(.[].gout)' ${nrn_ver}-full.json > ${nrn_ver}.json
+        xz ${nrn_ver}-full.json
         report2html ${nrn_ver}.json
         if [[ -d "${DROP_DIR_V2}" ]]; then
           python -m pip uninstall --yes neuron-nightly==${nrn_ver}
@@ -197,8 +211,9 @@ jobs:
         diffreports2html ${nrn1_ver}.json ${nrn2_ver}.json
 
     - uses: actions/upload-artifact@v3
+      if: ${{ always() }}
       with:
         name: ${{ env.nrn1_ver }}-vs-${{ env.nrn2_ver }}
         path: |
-          ./*.json
+          ./*.json.xz
           ./*.html

modeldb/commands.py

@@ -222,7 +222,7 @@ def diffreports2html(args=None):
     report_filename = os.path.join(Path(json_report1).resolve().parent, report_title + '.html')
     runtime_report_title = 'Runtimes ' + report_title
     runtime_report_filename = os.path.join(Path(json_report1).resolve().parent, "runtimes-" + report_title + '.html')
-    diff_dict, gout_dict,runtime_dict, v1, v2 = diff_reports(json_report1, json_report2)
+    diff_dict, gout_dict, runtime_dict, stats_dict, v1, v2 = diff_reports(json_report1, json_report2)
 
     print('Writing {} ...'.format(report_filename))
     with open(report_filename, 'w') as fh:
@@ -242,3 +242,53 @@ def diffreports2html(args=None):
             v2=v2),
     )
     print('Done.')
+    # Return a useful status code
+    code = 0
+    if len(diff_dict) > 1:
+        assert "0" in diff_dict  # summary info; not a real diff
+        print("FAILURE: stdout diffs in {}".format(set(diff_dict.keys()) - {"0"}))
+        code = 1
+    if len(gout_dict) > 1:
+        assert "0" in gout_dict  # summary info; not a real diff
+        print("FAILURE: gout diffs in {}".format(set(diff_dict.keys()) - {"0"}))
+        code = 1
+    total_failures = sum(
+        version_stats["Failed models"]["Count"] for version_stats in stats_dict.values()
+    )
+    if total_failures > 0:
+        print(
+            "FAILURE: there were {} failed model builds across {} versions of NEURON".format(
+                total_failures, len(stats_dict)
+            )
+        )
+        code = 1
+    total_runtime_failures = sum(
+        version_stats["Failed runs"]["Count"] for version_stats in stats_dict.values()
+    )
+    if total_runtime_failures > 0:
+        print(
+            "FAILURE: there were {} failed model runs across {} versions of NEURON".format(
+                total_runtime_failures, len(stats_dict)
+            )
+        )
+        code = 1
+    # These are not expected to be different between the two NEURON versions tested
+    assert (
+        len(
+            {
+                version_stats["Skipped runs"]["Count"]
+                for version_stats in stats_dict.values()
+            }
+        )
+        == 1
+    )
+    assert (
+        len(
+            {
+                version_stats["Total nof models run"]
+                for version_stats in stats_dict.values()
+            }
+        )
+        == 1
+    )
+    return code

modeldb/report.py

@@ -61,8 +61,11 @@ def diff_reports(report1_json, report2_json):
         hd = difflib.HtmlDiff()
         v1 = data_a["0"]["NEURON version"]
         v2 = data_b["0"]["NEURON version"]
-        diff_dict["0"] = hd.make_table(json.dumps(data_a["0"], indent='\t').split('\n'),
-                                             json.dumps(data_b["0"], indent='\t').split('\n')).replace("\n", "")
+        diff_dict["0"] = hd.make_table(
+            json.dumps(data_a["0"], indent="\t").split("\n"),
+            json.dumps(data_b["0"], indent="\t").split("\n"),
+        ).replace("\n", "")
+        stats_dict = {v1: data_a["0"]["Stats"], v2: data_b["0"]["Stats"]}
         for k in data_a.keys():
             if int(k) == 0:
                 continue  # skip info key
@@ -103,11 +106,11 @@ def _speedup(a, b):
                 # gout may be missing in one of the paths. `diff -N` treats non-existent files as empty.
                 if os.path.isfile(gout_a_file) or os.path.isfile(gout_b_file):
                     diff_out = subprocess.getoutput(
-                        "diff -uN {} {} | head -n 30".format(
+                        "diff -uN --speed-large-files {} {} | head -n 30".format(
                             shlex.quote(gout_a_file), shlex.quote(gout_b_file)
                         )
                     )
                     if diff_out:
                         gout_dict[k] = highlight(diff_out, DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True))
 
-    return diff_dict, gout_dict, runtime_dict, v1, v2
+    return diff_dict, gout_dict, runtime_dict, stats_dict, v1, v2