diff --git a/.github/workflows/nrn-modeldb-ci.yaml b/.github/workflows/nrn-modeldb-ci.yaml index c5797eb..27de7c1 100644 --- a/.github/workflows/nrn-modeldb-ci.yaml +++ b/.github/workflows/nrn-modeldb-ci.yaml @@ -162,7 +162,14 @@ jobs: python -m pip install $NEURON_V1 fi nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"` - runmodels --workdir=$nrn_ver $MODELS_TO_RUN + ps uxf # debug + runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN + # Filter out the gout data before generating HTML reports. The HTML + # diff uses the original gout files on disk anyway. Compress the large + # JSON file including gout data for inclusion in the artifacts + mv ${nrn_ver}.json ${nrn_ver}-full.json + jq -r 'del(.[].gout)' ${nrn_ver}-full.json > ${nrn_ver}.json + xz ${nrn_ver}-full.json report2html ${nrn_ver}.json if [[ -d "${DROP_DIR_V1}" ]]; then python -m pip uninstall --yes neuron-nightly==${nrn_ver} @@ -182,7 +189,14 @@ jobs: python -m pip install $NEURON_V2 fi nrn_ver=`python -c "from neuron import __version__ as nrn_ver; print(nrn_ver)"` - runmodels --workdir=$nrn_ver $MODELS_TO_RUN + ps uxf # debug + runmodels --gout --workdir=$nrn_ver $MODELS_TO_RUN + # Filter out the gout data before generating HTML reports. The HTML + # diff uses the original gout files on disk anyway. Compress the large + # JSON file including gout data for inclusion in the artifacts + mv ${nrn_ver}.json ${nrn_ver}-full.json + jq -r 'del(.[].gout)' ${nrn_ver}-full.json > ${nrn_ver}.json + xz ${nrn_ver}-full.json report2html ${nrn_ver}.json if [[ -d "${DROP_DIR_V2}" ]]; then python -m pip uninstall --yes neuron-nightly==${nrn_ver} @@ -197,8 +211,9 @@ jobs: diffreports2html ${nrn1_ver}.json ${nrn2_ver}.json - uses: actions/upload-artifact@v3 + if: ${{ always() }} with: name: ${{ env.nrn1_ver }}-vs-${{ env.nrn2_ver }} path: | - ./*.json + ./*.json.xz ./*.html diff --git a/modeldb/commands.py b/modeldb/commands.py index 1e68747..ca3fa3a 100644 --- a/modeldb/commands.py +++ b/modeldb/commands.py @@ -222,7 +222,7 @@ def diffreports2html(args=None): report_filename = os.path.join(Path(json_report1).resolve().parent, report_title + '.html') runtime_report_title = 'Runtimes ' + report_title runtime_report_filename = os.path.join(Path(json_report1).resolve().parent, "runtimes-" + report_title + '.html') - diff_dict, gout_dict,runtime_dict, v1, v2 = diff_reports(json_report1, json_report2) + diff_dict, gout_dict, runtime_dict, stats_dict, v1, v2 = diff_reports(json_report1, json_report2) print('Writing {} ...'.format(report_filename)) with open(report_filename, 'w') as fh: @@ -242,3 +242,53 @@ def diffreports2html(args=None): v2=v2), ) print('Done.') + # Return a useful status code + code = 0 + if len(diff_dict) > 1: + assert "0" in diff_dict # summary info; not a real diff + print("FAILURE: stdout diffs in {}".format(set(diff_dict.keys()) - {"0"})) + code = 1 + if len(gout_dict) > 1: + assert "0" in gout_dict # summary info; not a real diff + print("FAILURE: gout diffs in {}".format(set(diff_dict.keys()) - {"0"})) + code = 1 + total_failures = sum( + version_stats["Failed models"]["Count"] for version_stats in stats_dict.values() + ) + if total_failures > 0: + print( + "FAILURE: there were {} failed model builds across {} versions of NEURON".format( + total_failures, len(stats_dict) + ) + ) + code = 1 + total_runtime_failures = sum( + version_stats["Failed runs"]["Count"] for version_stats in stats_dict.values() + ) + if total_runtime_failures > 0: + print( + "FAILURE: there were {} failed model runs across {} versions of NEURON".format( + total_runtime_failures, len(stats_dict) + ) + ) + code = 1 + # These are not expected to be different between the two NEURON versions tested + assert ( + len( + { + version_stats["Skipped runs"]["Count"] + for version_stats in stats_dict.values() + } + ) + == 1 + ) + assert ( + len( + { + version_stats["Total nof models run"] + for version_stats in stats_dict.values() + } + ) + == 1 + ) + return code diff --git a/modeldb/report.py b/modeldb/report.py index 5bce904..e06364b 100644 --- a/modeldb/report.py +++ b/modeldb/report.py @@ -61,8 +61,11 @@ def diff_reports(report1_json, report2_json): hd = difflib.HtmlDiff() v1 = data_a["0"]["NEURON version"] v2 = data_b["0"]["NEURON version"] - diff_dict["0"] = hd.make_table(json.dumps(data_a["0"], indent='\t').split('\n'), - json.dumps(data_b["0"], indent='\t').split('\n')).replace("\n", "") + diff_dict["0"] = hd.make_table( + json.dumps(data_a["0"], indent="\t").split("\n"), + json.dumps(data_b["0"], indent="\t").split("\n"), + ).replace("\n", "") + stats_dict = {v1: data_a["0"]["Stats"], v2: data_b["0"]["Stats"]} for k in data_a.keys(): if int(k) == 0: continue # skip info key @@ -103,11 +106,11 @@ def _speedup(a, b): # gout may be missing in one of the paths. `diff -N` treats non-existent files as empty. if os.path.isfile(gout_a_file) or os.path.isfile(gout_b_file): diff_out = subprocess.getoutput( - "diff -uN {} {} | head -n 30".format( + "diff -uN --speed-large-files {} {} | head -n 30".format( shlex.quote(gout_a_file), shlex.quote(gout_b_file) ) ) if diff_out: gout_dict[k] = highlight(diff_out, DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True)) - return diff_dict, gout_dict, runtime_dict, v1, v2 + return diff_dict, gout_dict, runtime_dict, stats_dict, v1, v2