Skip to content

Commit

Permalink
Add runtimes report for nrnivmodl & model runtimes (#64)
Browse files Browse the repository at this point in the history
  • Loading branch information
alexsavulescu authored Dec 7, 2022
1 parent baa9ad0 commit 9c828d3
Show file tree
Hide file tree
Showing 3 changed files with 157 additions and 6 deletions.
15 changes: 14 additions & 1 deletion modeldb/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,11 +214,14 @@ def diffreports2html(args=None):
file_loader = FileSystemLoader(os.path.join(Path(__file__).parent.resolve(), 'templates'))
env = Environment(loader=file_loader)
template = env.get_template('diffreport.html')
runtime_template = env.get_template('runtimes.html')

report_title = '{}-vs-{}'.format(os.path.splitext(json_report1)[0],
os.path.splitext(json_report2)[0])
report_filename = os.path.join(Path(json_report1).resolve().parent, report_title + '.html')
diff_dict, gout_dict = diff_reports(json_report1, json_report2)
runtime_report_title = 'Runtimes ' + report_title
runtime_report_filename = os.path.join(Path(json_report1).resolve().parent, "runtimes-" + report_title + '.html')
diff_dict, gout_dict,runtime_dict, v1, v2 = diff_reports(json_report1, json_report2)

print('Writing {} ...'.format(report_filename))
with open(report_filename, 'w') as fh:
Expand All @@ -228,3 +231,13 @@ def diffreports2html(args=None):
gout_dict=gout_dict),
)
print('Done.')
print('Writing {} ...'.format(runtime_report_filename))
with open(runtime_report_filename, 'w') as fh:
fh.write(runtime_template.render(
title="{}".format(runtime_report_title),
runtime_dict=runtime_dict,
stats={"stats":diff_dict['0']},
v1=v1,
v2=v2),
)
print('Done.')
28 changes: 23 additions & 5 deletions modeldb/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,15 @@ def curate_run_data(run_data, model=None):
def diff_reports(report1_json, report2_json):
diff_dict = {}
gout_dict = {}
runtime_dict = {}

with open(report1_json, 'r+') as f, open(report2_json, 'r+') as f2:
data_a = json.load(f)
data_b = json.load(f2)

hd = difflib.HtmlDiff()
v1 = data_a["0"]["NEURON version"]
v2 = data_b["0"]["NEURON version"]
diff_dict["0"] = hd.make_table(json.dumps(data_a["0"], indent='\t').split('\n'),
json.dumps(data_b["0"], indent='\t').split('\n')).replace("\n", "")
for k in data_a.keys():
Expand All @@ -68,18 +71,33 @@ def diff_reports(report1_json, report2_json):
tofile=data_b[k]["run_info"]["start_dir"])
diff_dict[k] = highlight('\n'.join(ud), DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True))

# List of keys that make gout comparison pointless
skip_gout_keys = {"do_not_run", "moderr", "nrn_run_err"}
if skip_gout_keys.isdisjoint(data_a[k]) and skip_gout_keys.isdisjoint(data_b[k]):
def _speedup(a, b):
dict = {}
dict["v1"] = a
dict["v2"] = b
# compute slowdown/speedup relative to runtime_b (negative means slowdown)
dict["speedup"] = (float(b) - float(a)) / float(b) * 100
return dict

# List of keys that make gout comparison and speedup comparison pointless
skip_keys = {"do_not_run", "moderr", "nrn_run_err"}
if skip_keys.isdisjoint(data_a[k]) and skip_keys.isdisjoint(data_b[k]):
# compare runtimes and compute slowdown or speedup
runtime_dict[k] = {}
runtime_dict[k]["total"] = _speedup(data_a[k]["run_time"], data_b[k]["run_time"])
for runkey in ("model", "nrnivmodl"):
if runkey in data_a[k]["run_times"] and runkey in data_b[k]["run_times"]:
runtime_dict[k][runkey] = _speedup(data_a[k]["run_times"][runkey], data_b[k]["run_times"][runkey])

# compare gout
gout_a_file = os.path.join(data_a[k]["run_info"]["start_dir"], "gout")
gout_b_file = os.path.join(data_b[k]["run_info"]["start_dir"], "gout")

# gout may be missing in one of the paths. `diff -N` treats non-existent files as empty.
if os.path.isfile(gout_a_file) or os.path.isfile(gout_b_file):
diff_out = subprocess.getoutput("diff -uN {} {} | head -n 30".format(gout_a_file, gout_b_file))
if diff_out:
gout_dict[k] = highlight(diff_out, DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True))

return diff_dict, gout_dict
return diff_dict, gout_dict, runtime_dict, v1, v2


120 changes: 120 additions & 0 deletions modeldb/templates/runtimes.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@

<!doctype html>
<html>
<head>
<title>{{ title }}</title>
<style type="text/css">
table.diff {font-family:Courier; border:medium;}
.diff_header {background-color:#e0e0e0}
td.diff_header {text-align:right}
.diff_next {background-color:#c0c0c0}
.diff_add {background-color:#aaffaa}
.diff_chg {background-color:#ffff77}
.diff_sub {background-color:#ffaaaa}
</style>
</head>
<body>
<details open="true">
<summary>{{ title }} : Run Info</summary>
<div id="run_info"></div>
</details>

<br/>
<label for="fSearch">Search entire report</label>
<input type="text" id="fSearch" name="fSearch">
<div id="example-table"></div>

<script src="https:////ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
<link href="https://unpkg.com/[email protected]/dist/css/tabulator.min.css" rel="stylesheet">
<script type="text/javascript" src="https://unpkg.com/[email protected]/dist/js/tabulator.min.js"></script>

<script>
$(document).ready ( function() {
// report = JSON.parse(JSON.stringify(report));
var report_data = [];
runtime_dict = {{ runtime_dict }}
v1 = "{{ v1 }}"
v2 = "{{ v2 }}"

function htmlDecode(input) {
return input
.replace(/\\/g, "")
.replace(/^"(.*)"$/, '$1');
}
stats = {{ stats }}
document.getElementById("run_info").innerHTML=htmlDecode(JSON.stringify(stats["stats"]));

for(var i in runtime_dict){
for (var key in runtime_dict[i]) {
if (key == "total") {
continue
}
run_dict = {}

run_dict["type"] = key
run_dict["v1"] = runtime_dict[i][key]["v1"];
run_dict["v2"] = runtime_dict[i][key]["v2"];
run_dict["speedup"] = runtime_dict[i][key]["speedup"];
run_dict["id"]=i
report_data.push(run_dict)
}
}
const input = document.getElementById("fSearch");
input.addEventListener("keyup", function() {
table.setFilter(matchAny, { value: input.value });
if (input.value == " ") {
table.clearFilter()
}
});
function matchAny(data, filterParams) {
var match = false;
const regex = RegExp(filterParams.value, 'i');

for (var key in data) {
if (regex.test(data[key]) == true) {
match = true;
}
}
return match;
}

var table = new Tabulator("#example-table", {
data:report_data, //load row data from array
height:"100%",
layout:"fitDataFill", //fit columns to width of table
addRowPos:"top", //when adding a new row, add it to the top of the table
history:true, //allow undo and redo actions on the table
pagination:"local", //paginate the data
paginationSize:100, //allow 7 rows per page of data
paginationCounter:"rows", //display count of paginated rows in footer
movableColumns:true, //allow column order to be changed
resizableRows:true, //allow row order to be changed
initialSort:[ //set the initial sort order of the data
{column:"speedup", dir:"asc",
},
],
columns:[ //define the table columns
{title:"Acc. No.", field:"id"},
{title:"runtime type", field:"type",
headerFilterPlaceholder: " select runtime type (nrnivmodl, model) ...",
headerFilter: 'select',
headerFilterFunc:"=",
headerFilterParams: {multiselect:false, values: ["nrnivmodl", "model"]},
//size to fit the headerFilterPlaceholder
width: 320,
},
{title:v1+" (seconds)", field:"v1", formatter:"money", formatterParams:{precision:2}, sorter:"number"},
{title:v2+" (seconds)", field:"v2", formatter:"money", formatterParams:{precision:2}, sorter:"number"},
{title:"speedup (%)", field:"speedup", formatter:"money", formatterParams:{precision:2}, sorter:"number"},
{title: "hidden", field: "hidden", visible: false},
],
groupBy:"id",
sortBy:"speedup",

});

}
)
</script>
</body>
</html>

0 comments on commit 9c828d3

Please sign in to comment.