diff --git a/modeldb/commands.py b/modeldb/commands.py index e35f1e2..3f7f3f6 100644 --- a/modeldb/commands.py +++ b/modeldb/commands.py @@ -214,11 +214,14 @@ def diffreports2html(args=None): file_loader = FileSystemLoader(os.path.join(Path(__file__).parent.resolve(), 'templates')) env = Environment(loader=file_loader) template = env.get_template('diffreport.html') + runtime_template = env.get_template('runtimes.html') report_title = '{}-vs-{}'.format(os.path.splitext(json_report1)[0], os.path.splitext(json_report2)[0]) report_filename = os.path.join(Path(json_report1).resolve().parent, report_title + '.html') - diff_dict, gout_dict = diff_reports(json_report1, json_report2) + runtime_report_title = 'Runtimes ' + report_title + runtime_report_filename = os.path.join(Path(json_report1).resolve().parent, "runtimes-" + report_title + '.html') + diff_dict, gout_dict,runtime_dict, v1, v2 = diff_reports(json_report1, json_report2) print('Writing {} ...'.format(report_filename)) with open(report_filename, 'w') as fh: @@ -228,3 +231,13 @@ def diffreports2html(args=None): gout_dict=gout_dict), ) print('Done.') + print('Writing {} ...'.format(runtime_report_filename)) + with open(runtime_report_filename, 'w') as fh: + fh.write(runtime_template.render( + title="{}".format(runtime_report_title), + runtime_dict=runtime_dict, + stats={"stats":diff_dict['0']}, + v1=v1, + v2=v2), + ) + print('Done.') \ No newline at end of file diff --git a/modeldb/report.py b/modeldb/report.py index 52ad563..c61ef3d 100644 --- a/modeldb/report.py +++ b/modeldb/report.py @@ -49,12 +49,15 @@ def curate_run_data(run_data, model=None): def diff_reports(report1_json, report2_json): diff_dict = {} gout_dict = {} + runtime_dict = {} with open(report1_json, 'r+') as f, open(report2_json, 'r+') as f2: data_a = json.load(f) data_b = json.load(f2) hd = difflib.HtmlDiff() + v1 = data_a["0"]["NEURON version"] + v2 = data_b["0"]["NEURON version"] diff_dict["0"] = hd.make_table(json.dumps(data_a["0"], indent='\t').split('\n'), json.dumps(data_b["0"], indent='\t').split('\n')).replace("\n", "") for k in data_a.keys(): @@ -68,18 +71,33 @@ def diff_reports(report1_json, report2_json): tofile=data_b[k]["run_info"]["start_dir"]) diff_dict[k] = highlight('\n'.join(ud), DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True)) - # List of keys that make gout comparison pointless - skip_gout_keys = {"do_not_run", "moderr", "nrn_run_err"} - if skip_gout_keys.isdisjoint(data_a[k]) and skip_gout_keys.isdisjoint(data_b[k]): + def _speedup(a, b): + dict = {} + dict["v1"] = a + dict["v2"] = b + # compute slowdown/speedup relative to runtime_b (negative means slowdown) + dict["speedup"] = (float(b) - float(a)) / float(b) * 100 + return dict + + # List of keys that make gout comparison and speedup comparison pointless + skip_keys = {"do_not_run", "moderr", "nrn_run_err"} + if skip_keys.isdisjoint(data_a[k]) and skip_keys.isdisjoint(data_b[k]): + # compare runtimes and compute slowdown or speedup + runtime_dict[k] = {} + runtime_dict[k]["total"] = _speedup(data_a[k]["run_time"], data_b[k]["run_time"]) + for runkey in ("model", "nrnivmodl"): + if runkey in data_a[k]["run_times"] and runkey in data_b[k]["run_times"]: + runtime_dict[k][runkey] = _speedup(data_a[k]["run_times"][runkey], data_b[k]["run_times"][runkey]) + + # compare gout gout_a_file = os.path.join(data_a[k]["run_info"]["start_dir"], "gout") gout_b_file = os.path.join(data_b[k]["run_info"]["start_dir"], "gout") - # gout may be missing in one of the paths. `diff -N` treats non-existent files as empty. if os.path.isfile(gout_a_file) or os.path.isfile(gout_b_file): diff_out = subprocess.getoutput("diff -uN {} {} | head -n 30".format(gout_a_file, gout_b_file)) if diff_out: gout_dict[k] = highlight(diff_out, DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True)) - return diff_dict, gout_dict + return diff_dict, gout_dict, runtime_dict, v1, v2 diff --git a/modeldb/templates/runtimes.html b/modeldb/templates/runtimes.html new file mode 100644 index 0000000..7c93089 --- /dev/null +++ b/modeldb/templates/runtimes.html @@ -0,0 +1,120 @@ + + + +
+