From 0f4d4bd2411be8020e22e1eedf007ae4bb91882c Mon Sep 17 00:00:00 2001
From: Olli Lupton <oliver.lupton@epfl.ch>
Date: Wed, 1 Mar 2023 10:53:58 +0100
Subject: [PATCH] Run with --gout, compress .json, report status.

---
 .github/workflows/nrn-modeldb-ci.yaml | 11 ++++--
 modeldb/commands.py                   | 52 ++++++++++++++++++++++++++-
 modeldb/report.py                     |  9 +++--
 3 files changed, 65 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/nrn-modeldb-ci.yaml b/.github/workflows/nrn-modeldb-ci.yaml
index c5797eb..d1ef16c 100644
--- a/.github/workflows/nrn-modeldb-ci.yaml
+++ b/.github/workflows/nrn-modeldb-ci.yaml
@@ -162,7 +162,7 @@ jobs:
           python -m pip install $NEURON_V1
         fi
         nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"`
-        runmodels --workdir=$nrn_ver $MODELS_TO_RUN
+        runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN
         report2html ${nrn_ver}.json
         if [[ -d "${DROP_DIR_V1}" ]]; then
           python -m pip uninstall --yes neuron-nightly==${nrn_ver}
@@ -182,7 +182,7 @@ jobs:
           python -m pip install $NEURON_V2
         fi
         nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"`
-        runmodels --workdir=$nrn_ver $MODELS_TO_RUN
+        runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN
         report2html ${nrn_ver}.json
         if [[ -d "${DROP_DIR_V2}" ]]; then
           python -m pip uninstall --yes neuron-nightly==${nrn_ver}
@@ -194,11 +194,16 @@ jobs:
     - name: diffreports2html ${{ env.nrn1_ver }}.json <-> ${{ env.nrn2_ver }}.json
       if: env.NEURON_V1 != env.NEURON_V2
       run: |
+        # JSON files containing gout can be large. Do this first, so the
+        # compressed JSON is uploaded even if there's a diff. Pass --keep so
+        # that diffreports2html can use the uncompressed files.
+        xz --keep ${nrn1_ver}.json ${nrn2_ver}.json
         diffreports2html ${nrn1_ver}.json ${nrn2_ver}.json
 
     - uses: actions/upload-artifact@v3
+      if: ${{ always() }}
       with:
         name: ${{ env.nrn1_ver }}-vs-${{ env.nrn2_ver }}
         path: |
-          ./*.json
+          ./*.json.xz
           ./*.html
diff --git a/modeldb/commands.py b/modeldb/commands.py
index 1e68747..ca3fa3a 100644
--- a/modeldb/commands.py
+++ b/modeldb/commands.py
@@ -222,7 +222,7 @@ def diffreports2html(args=None):
     report_filename = os.path.join(Path(json_report1).resolve().parent, report_title + '.html')
     runtime_report_title = 'Runtimes ' + report_title
     runtime_report_filename = os.path.join(Path(json_report1).resolve().parent, "runtimes-" + report_title + '.html')
-    diff_dict, gout_dict,runtime_dict, v1, v2 = diff_reports(json_report1, json_report2)
+    diff_dict, gout_dict, runtime_dict, stats_dict, v1, v2 = diff_reports(json_report1, json_report2)
 
     print('Writing {} ...'.format(report_filename))
     with open(report_filename, 'w') as fh:
@@ -242,3 +242,53 @@ def diffreports2html(args=None):
             v2=v2),
     )
     print('Done.')
+    # Return a useful status code
+    code = 0
+    if len(diff_dict) > 1:
+        assert "0" in diff_dict  # summary info; not a real diff
+        print("FAILURE: stdout diffs in {}".format(set(diff_dict.keys()) - {"0"}))
+        code = 1
+    if len(gout_dict) > 1:
+        assert "0" in gout_dict  # summary info; not a real diff
+        print("FAILURE: gout diffs in {}".format(set(diff_dict.keys()) - {"0"}))
+        code = 1
+    total_failures = sum(
+        version_stats["Failed models"]["Count"] for version_stats in stats_dict.values()
+    )
+    if total_failures > 0:
+        print(
+            "FAILURE: there were {} failed model builds across {} versions of NEURON".format(
+                total_failures, len(stats_dict)
+            )
+        )
+        code = 1
+    total_runtime_failures = sum(
+        version_stats["Failed runs"]["Count"] for version_stats in stats_dict.values()
+    )
+    if total_runtime_failures > 0:
+        print(
+            "FAILURE: there were {} failed model runs across {} versions of NEURON".format(
+                total_runtime_failures, len(stats_dict)
+            )
+        )
+        code = 1
+    # These are not expected to be different between the two NEURON versions tested
+    assert (
+        len(
+            {
+                version_stats["Skipped runs"]["Count"]
+                for version_stats in stats_dict.values()
+            }
+        )
+        == 1
+    )
+    assert (
+        len(
+            {
+                version_stats["Total nof models run"]
+                for version_stats in stats_dict.values()
+            }
+        )
+        == 1
+    )
+    return code
diff --git a/modeldb/report.py b/modeldb/report.py
index 5bce904..6f03049 100644
--- a/modeldb/report.py
+++ b/modeldb/report.py
@@ -61,8 +61,11 @@ def diff_reports(report1_json, report2_json):
         hd = difflib.HtmlDiff()
         v1 = data_a["0"]["NEURON version"]
         v2 = data_b["0"]["NEURON version"]
-        diff_dict["0"] = hd.make_table(json.dumps(data_a["0"], indent='\t').split('\n'),
-                                             json.dumps(data_b["0"], indent='\t').split('\n')).replace("\n", "")
+        diff_dict["0"] = hd.make_table(
+            json.dumps(data_a["0"], indent="\t").split("\n"),
+            json.dumps(data_b["0"], indent="\t").split("\n"),
+        ).replace("\n", "")
+        stats_dict = {v1: data_a["0"]["Stats"], v2: data_b["0"]["Stats"]}
         for k in data_a.keys():
             if int(k) == 0:
                 continue  # skip info key
@@ -110,4 +113,4 @@ def _speedup(a, b):
                     if diff_out:
                         gout_dict[k] = highlight(diff_out, DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True))
 
-    return diff_dict, gout_dict, runtime_dict, v1, v2
+    return diff_dict, gout_dict, runtime_dict, stats_dict, v1, v2