From 0f4d4bd2411be8020e22e1eedf007ae4bb91882c Mon Sep 17 00:00:00 2001 From: Olli Lupton Date: Wed, 1 Mar 2023 10:53:58 +0100 Subject: [PATCH 1/7] Run with --gout, compress .json, report status. --- .github/workflows/nrn-modeldb-ci.yaml | 11 ++++-- modeldb/commands.py | 52 ++++++++++++++++++++++++++- modeldb/report.py | 9 +++-- 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nrn-modeldb-ci.yaml b/.github/workflows/nrn-modeldb-ci.yaml index c5797eb..d1ef16c 100644 --- a/.github/workflows/nrn-modeldb-ci.yaml +++ b/.github/workflows/nrn-modeldb-ci.yaml @@ -162,7 +162,7 @@ jobs: python -m pip install $NEURON_V1 fi nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"` - runmodels --workdir=$nrn_ver $MODELS_TO_RUN + runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN report2html ${nrn_ver}.json if [[ -d "${DROP_DIR_V1}" ]]; then python -m pip uninstall --yes neuron-nightly==${nrn_ver} @@ -182,7 +182,7 @@ jobs: python -m pip install $NEURON_V2 fi nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"` - runmodels --workdir=$nrn_ver $MODELS_TO_RUN + runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN report2html ${nrn_ver}.json if [[ -d "${DROP_DIR_V2}" ]]; then python -m pip uninstall --yes neuron-nightly==${nrn_ver} @@ -194,11 +194,16 @@ jobs: - name: diffreports2html ${{ env.nrn1_ver }}.json <-> ${{ env.nrn2_ver }}.json if: env.NEURON_V1 != env.NEURON_V2 run: | + # JSON files containing gout can be large. Do this first, so the + # compressed JSON is uploaded even if there's a diff. Pass --keep so + # that diffreports2html can use the uncompressed files. + xz --keep ${nrn1_ver}.json ${nrn2_ver}.json diffreports2html ${nrn1_ver}.json ${nrn2_ver}.json - uses: actions/upload-artifact@v3 + if: ${{ always() }} with: name: ${{ env.nrn1_ver }}-vs-${{ env.nrn2_ver }} path: | - ./*.json + ./*.json.xz ./*.html diff --git a/modeldb/commands.py b/modeldb/commands.py index 1e68747..ca3fa3a 100644 --- a/modeldb/commands.py +++ b/modeldb/commands.py @@ -222,7 +222,7 @@ def diffreports2html(args=None): report_filename = os.path.join(Path(json_report1).resolve().parent, report_title + '.html') runtime_report_title = 'Runtimes ' + report_title runtime_report_filename = os.path.join(Path(json_report1).resolve().parent, "runtimes-" + report_title + '.html') - diff_dict, gout_dict,runtime_dict, v1, v2 = diff_reports(json_report1, json_report2) + diff_dict, gout_dict, runtime_dict, stats_dict, v1, v2 = diff_reports(json_report1, json_report2) print('Writing {} ...'.format(report_filename)) with open(report_filename, 'w') as fh: @@ -242,3 +242,53 @@ def diffreports2html(args=None): v2=v2), ) print('Done.') + # Return a useful status code + code = 0 + if len(diff_dict) > 1: + assert "0" in diff_dict # summary info; not a real diff + print("FAILURE: stdout diffs in {}".format(set(diff_dict.keys()) - {"0"})) + code = 1 + if len(gout_dict) > 1: + assert "0" in gout_dict # summary info; not a real diff + print("FAILURE: gout diffs in {}".format(set(diff_dict.keys()) - {"0"})) + code = 1 + total_failures = sum( + version_stats["Failed models"]["Count"] for version_stats in stats_dict.values() + ) + if total_failures > 0: + print( + "FAILURE: there were {} failed model builds across {} versions of NEURON".format( + total_failures, len(stats_dict) + ) + ) + code = 1 + total_runtime_failures = sum( + version_stats["Failed runs"]["Count"] for version_stats in stats_dict.values() + ) + if total_runtime_failures > 0: + print( + "FAILURE: there were {} failed model runs across {} versions of NEURON".format( + total_runtime_failures, len(stats_dict) + ) + ) + code = 1 + # These are not expected to be different between the two NEURON versions tested + assert ( + len( + { + version_stats["Skipped runs"]["Count"] + for version_stats in stats_dict.values() + } + ) + == 1 + ) + assert ( + len( + { + version_stats["Total nof models run"] + for version_stats in stats_dict.values() + } + ) + == 1 + ) + return code diff --git a/modeldb/report.py b/modeldb/report.py index 5bce904..6f03049 100644 --- a/modeldb/report.py +++ b/modeldb/report.py @@ -61,8 +61,11 @@ def diff_reports(report1_json, report2_json): hd = difflib.HtmlDiff() v1 = data_a["0"]["NEURON version"] v2 = data_b["0"]["NEURON version"] - diff_dict["0"] = hd.make_table(json.dumps(data_a["0"], indent='\t').split('\n'), - json.dumps(data_b["0"], indent='\t').split('\n')).replace("\n", "") + diff_dict["0"] = hd.make_table( + json.dumps(data_a["0"], indent="\t").split("\n"), + json.dumps(data_b["0"], indent="\t").split("\n"), + ).replace("\n", "") + stats_dict = {v1: data_a["0"]["Stats"], v2: data_b["0"]["Stats"]} for k in data_a.keys(): if int(k) == 0: continue # skip info key @@ -110,4 +113,4 @@ def _speedup(a, b): if diff_out: gout_dict[k] = highlight(diff_out, DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True)) - return diff_dict, gout_dict, runtime_dict, v1, v2 + return diff_dict, gout_dict, runtime_dict, stats_dict, v1, v2 From acd4474ca1e00f9b0e0e255bce4208fca0798dcf Mon Sep 17 00:00:00 2001 From: Olli Lupton Date: Wed, 1 Mar 2023 16:10:14 +0100 Subject: [PATCH 2/7] Try and make JSON handling lighter by filtering out the gout data. --- .github/workflows/nrn-modeldb-ci.yaml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nrn-modeldb-ci.yaml b/.github/workflows/nrn-modeldb-ci.yaml index d1ef16c..b034316 100644 --- a/.github/workflows/nrn-modeldb-ci.yaml +++ b/.github/workflows/nrn-modeldb-ci.yaml @@ -163,7 +163,12 @@ jobs: fi nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"` runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN - report2html ${nrn_ver}.json + # Filter out the gout data before generating HTML reports. The HTML + # diff uses the original gout files on disk anyway. Compress the large + # JSON file including gout data for inclusion in the artifacts + jq --stream 'fromstream(inputs | select(.[0][1] != "gout"))' ${nrn_ver}.json < ${nrn_ver}-nogout.json + xz ${nrn_ver}.json + report2html ${nrn_ver}-nogout.json if [[ -d "${DROP_DIR_V1}" ]]; then python -m pip uninstall --yes neuron-nightly==${nrn_ver} else @@ -183,7 +188,12 @@ jobs: fi nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"` runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN - report2html ${nrn_ver}.json + # Filter out the gout data before generating HTML reports. The HTML + # diff uses the original gout files on disk anyway. Compress the large + # JSON file including gout data for inclusion in the artifacts + jq --stream 'fromstream(inputs | select(.[0][1] != "gout"))' ${nrn_ver}.json < ${nrn_ver}-nogout.json + xz ${nrn_ver}.json + report2html ${nrn_ver}-nogout.json if [[ -d "${DROP_DIR_V2}" ]]; then python -m pip uninstall --yes neuron-nightly==${nrn_ver} else @@ -194,11 +204,7 @@ jobs: - name: diffreports2html ${{ env.nrn1_ver }}.json <-> ${{ env.nrn2_ver }}.json if: env.NEURON_V1 != env.NEURON_V2 run: | - # JSON files containing gout can be large. Do this first, so the - # compressed JSON is uploaded even if there's a diff. Pass --keep so - # that diffreports2html can use the uncompressed files. - xz --keep ${nrn1_ver}.json ${nrn2_ver}.json - diffreports2html ${nrn1_ver}.json ${nrn2_ver}.json + diffreports2html ${nrn1_ver}-nogout.json ${nrn2_ver}-nogout.json - uses: actions/upload-artifact@v3 if: ${{ always() }} From 25abc86d9bac8e13af6e4ac14d312cc0ba42fb07 Mon Sep 17 00:00:00 2001 From: Olli Lupton Date: Wed, 1 Mar 2023 16:14:26 +0100 Subject: [PATCH 3/7] add --speed-large-files --- modeldb/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modeldb/report.py b/modeldb/report.py index 6f03049..e06364b 100644 --- a/modeldb/report.py +++ b/modeldb/report.py @@ -106,7 +106,7 @@ def _speedup(a, b): # gout may be missing in one of the paths. `diff -N` treats non-existent files as empty. if os.path.isfile(gout_a_file) or os.path.isfile(gout_b_file): diff_out = subprocess.getoutput( - "diff -uN {} {} | head -n 30".format( + "diff -uN --speed-large-files {} {} | head -n 30".format( shlex.quote(gout_a_file), shlex.quote(gout_b_file) ) ) From 3dd4eb4a7abc78eae92c8dcb0f6e4fe21ca6e4ca Mon Sep 17 00:00:00 2001 From: Olli Lupton Date: Wed, 1 Mar 2023 16:42:57 +0100 Subject: [PATCH 4/7] fixup --- .github/workflows/nrn-modeldb-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nrn-modeldb-ci.yaml b/.github/workflows/nrn-modeldb-ci.yaml index b034316..7f954b9 100644 --- a/.github/workflows/nrn-modeldb-ci.yaml +++ b/.github/workflows/nrn-modeldb-ci.yaml @@ -166,7 +166,7 @@ jobs: # Filter out the gout data before generating HTML reports. The HTML # diff uses the original gout files on disk anyway. Compress the large # JSON file including gout data for inclusion in the artifacts - jq --stream 'fromstream(inputs | select(.[0][1] != "gout"))' ${nrn_ver}.json < ${nrn_ver}-nogout.json + jq --stream 'fromstream(inputs | select(.[0][1] != "gout"))' ${nrn_ver}.json > ${nrn_ver}-nogout.json xz ${nrn_ver}.json report2html ${nrn_ver}-nogout.json if [[ -d "${DROP_DIR_V1}" ]]; then @@ -191,7 +191,7 @@ jobs: # Filter out the gout data before generating HTML reports. The HTML # diff uses the original gout files on disk anyway. Compress the large # JSON file including gout data for inclusion in the artifacts - jq --stream 'fromstream(inputs | select(.[0][1] != "gout"))' ${nrn_ver}.json < ${nrn_ver}-nogout.json + jq --stream 'fromstream(inputs | select(.[0][1] != "gout"))' ${nrn_ver}.json > ${nrn_ver}-nogout.json xz ${nrn_ver}.json report2html ${nrn_ver}-nogout.json if [[ -d "${DROP_DIR_V2}" ]]; then From 45b6f450d58dedc038b7c0a26310abb87a937f3a Mon Sep 17 00:00:00 2001 From: Olli Lupton Date: Wed, 1 Mar 2023 17:15:53 +0100 Subject: [PATCH 5/7] fixup --- .github/workflows/nrn-modeldb-ci.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nrn-modeldb-ci.yaml b/.github/workflows/nrn-modeldb-ci.yaml index 7f954b9..93e88e6 100644 --- a/.github/workflows/nrn-modeldb-ci.yaml +++ b/.github/workflows/nrn-modeldb-ci.yaml @@ -162,11 +162,12 @@ jobs: python -m pip install $NEURON_V1 fi nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"` + ps uxf # debug runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN # Filter out the gout data before generating HTML reports. The HTML # diff uses the original gout files on disk anyway. Compress the large # JSON file including gout data for inclusion in the artifacts - jq --stream 'fromstream(inputs | select(.[0][1] != "gout"))' ${nrn_ver}.json > ${nrn_ver}-nogout.json + jq -r 'del(.[].gout)' ${nrn_ver}.json > ${nrn_ver}-nogout.json xz ${nrn_ver}.json report2html ${nrn_ver}-nogout.json if [[ -d "${DROP_DIR_V1}" ]]; then @@ -187,11 +188,12 @@ jobs: python -m pip install $NEURON_V2 fi nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"` + ps uxf # debug runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN # Filter out the gout data before generating HTML reports. The HTML # diff uses the original gout files on disk anyway. Compress the large # JSON file including gout data for inclusion in the artifacts - jq --stream 'fromstream(inputs | select(.[0][1] != "gout"))' ${nrn_ver}.json > ${nrn_ver}-nogout.json + jq -r 'del(.[].gout)' ${nrn_ver}.json > ${nrn_ver}-nogout.json xz ${nrn_ver}.json report2html ${nrn_ver}-nogout.json if [[ -d "${DROP_DIR_V2}" ]]; then From 40cb2c958518544be7e1e937f066b5cfcdf239f4 Mon Sep 17 00:00:00 2001 From: Olli Lupton Date: Wed, 1 Mar 2023 17:38:24 +0100 Subject: [PATCH 6/7] remove nogout from names --- .github/workflows/nrn-modeldb-ci.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nrn-modeldb-ci.yaml b/.github/workflows/nrn-modeldb-ci.yaml index 93e88e6..61e5afe 100644 --- a/.github/workflows/nrn-modeldb-ci.yaml +++ b/.github/workflows/nrn-modeldb-ci.yaml @@ -167,9 +167,10 @@ jobs: # Filter out the gout data before generating HTML reports. The HTML # diff uses the original gout files on disk anyway. Compress the large # JSON file including gout data for inclusion in the artifacts - jq -r 'del(.[].gout)' ${nrn_ver}.json > ${nrn_ver}-nogout.json - xz ${nrn_ver}.json - report2html ${nrn_ver}-nogout.json + mv ${nrn_ver}.json ${nrn_ver}-full.json + jq -r 'del(.[].gout)' ${nrn_ver}-full.json > ${nrn_ver}.json + xz ${nrn_ver}-full.json + report2html ${nrn_ver}.json if [[ -d "${DROP_DIR_V1}" ]]; then python -m pip uninstall --yes neuron-nightly==${nrn_ver} else @@ -193,9 +194,10 @@ jobs: # Filter out the gout data before generating HTML reports. The HTML # diff uses the original gout files on disk anyway. Compress the large # JSON file including gout data for inclusion in the artifacts - jq -r 'del(.[].gout)' ${nrn_ver}.json > ${nrn_ver}-nogout.json - xz ${nrn_ver}.json - report2html ${nrn_ver}-nogout.json + mv ${nrn_ver}.json ${nrn_ver}-full.json + jq -r 'del(.[].gout)' ${nrn_ver}-full.json > ${nrn_ver}.json + xz ${nrn_ver}-full.json + report2html ${nrn_ver}.json if [[ -d "${DROP_DIR_V2}" ]]; then python -m pip uninstall --yes neuron-nightly==${nrn_ver} else From 9b86efba5069f4fd4f7ae3a426c41d2b60676ab6 Mon Sep 17 00:00:00 2001 From: Olli Lupton Date: Wed, 1 Mar 2023 18:53:21 +0100 Subject: [PATCH 7/7] fixup! --- .github/workflows/nrn-modeldb-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nrn-modeldb-ci.yaml b/.github/workflows/nrn-modeldb-ci.yaml index 61e5afe..27de7c1 100644 --- a/.github/workflows/nrn-modeldb-ci.yaml +++ b/.github/workflows/nrn-modeldb-ci.yaml @@ -208,7 +208,7 @@ jobs: - name: diffreports2html ${{ env.nrn1_ver }}.json <-> ${{ env.nrn2_ver }}.json if: env.NEURON_V1 != env.NEURON_V2 run: | - diffreports2html ${nrn1_ver}-nogout.json ${nrn2_ver}-nogout.json + diffreports2html ${nrn1_ver}.json ${nrn2_ver}.json - uses: actions/upload-artifact@v3 if: ${{ always() }}