diff --git a/modeldb/commands.py b/modeldb/commands.py index 37ea3fc..1e68747 100644 --- a/modeldb/commands.py +++ b/modeldb/commands.py @@ -71,7 +71,7 @@ def runmodels(args=None): if virtual: from pyvirtualdisplay import Display - with Display(manage_global_env=False) as _: + with Display(manage_global_env=True, visible=False) as _: mrm.run_models(model_list) else: mrm.run_models(model_list) diff --git a/modeldb/modeldb-run.yaml b/modeldb/modeldb-run.yaml index 27e68f4..de34678 100644 --- a/modeldb/modeldb-run.yaml +++ b/modeldb/modeldb-run.yaml @@ -380,6 +380,12 @@ run: - load_file("testnet.hoc") - verify_graph_() + script: + # there are two problems with the code in this model: + # - it tries to access /proc/uptime, which is not portable + # - if that works (i.e. on Linux) it runs with a different seed every time, + # which is unhelpful in a CI context + - sed -i'.bak' -e 's#ropen(#// ropen(#g;s#rseed = fscan()#rseed=424242#g' testnet.hoc 52034: run: - run_short() @@ -625,24 +631,34 @@ run: - xopen("show_start.hoc") - verify_graph_() + script: + # avoid printing out times + - sed -i'.bak' -e 's/^startsw()$/{startsw()}/g' ga_setup.hoc 97860: run: - run() - verify_graph_() 97868: - curate_patterns: - - pattern: '^dyld[\d+]: symbol not found in flat namespace' - repl: 'dyld[%pid%]: symbol not found in flat namespace' - comment: //do not run - run: null + comment: 'Depends on MySQL, do not run' + skip: true 97874: model_dir: modeldb run: - verify_graph_() 97917: + curate_patterns: + - pattern: 'SetupTime: [0-9.\-e]+' + repl: 'SetupTime: %setup_time%' + # There is a different (clashing) model under pardentategyrus, don't test + # that one at all for now + model_dir: nrntraub/mod run: - traub_launch() - verify_graph_() + # Stop mosinit from running nrnivmodl on the fly; use the special produced + # by nrn-modeldb-ci from the model_dir set above + script: + - sed -i'.bak' -e 's/mkdll_("nrntraub", "mod", s.s)) {/1) {\n\t\tchdir("nrntraub")/g' mosinit.hoc 98005: run: - run() @@ -1086,6 +1102,14 @@ 223962: skip: true comment: takes too long, need to see how to reduce time +229276: + model_dir: mechanisms + script: + - cp template.hoc template.hoc.iconv.bak + - cp createsimulation.hoc createsimulation.hoc.iconv.bak + - iconv -f LATIN1 -t UTF-8 template.hoc.iconv.bak > template.hoc + - iconv -f LATIN1 -t UTF-8 createsimulation.hoc.iconv.bak > createsimulation.hoc + - sed -i'.bak' -e 's#^tstop=\([0-9a-z+]*\)#tstop=((\1)/500)#g' createsimulation.hoc 244262: script: - if [[ ! -f Iintra.dat ]]; then unzip Iintra.dat.zip; fi @@ -1101,6 +1125,18 @@ - sed -i'.bak' -e 's/tstop = /tstop = 50\/\//' Fig8_tuft_NMDA_spike.hoc - sed -i'.bak' -e 's/simul_iter=/simul_iter=2 \/\//' Fig8_tuft_NMDA_spike.hoc comment: usual simulation time is ~ 10 minutes (tstop = 120 and simul_iter=10) +267067: + # This model ID seems to have two ~different models inside it. Arbitrarily choose one. + model_dir: 'Na12 Analysis/mechanisms' + run: + - run() + - verify_graph_() + script: + - (cd "Na12 Analysis" && mv mosinit.hoc mosinit.hoc.bak && ln -s runModel.hoc mosinit.hoc) + - sed -i'.bak' -e 's#\./mosinit\.hoc#./mosinit.hoc.bak#g' 'Na12 Analysis/runModel.hoc' + - sed -i'.bak' -e 's#^tstop=30000#tstop=100#g' 'Na12 Analysis/constants.hoc' +267384: + model_dir: 'BBP_TTPC_EXAMPLE' 206267: run: - run() @@ -1152,5 +1188,5 @@ script: - 2to3 -w . - sed -i'.bak' -e 's/plt.show()/plt.show(block=False);plt.pause(1);plt.close();quit()/' clay_mohit.py - - +251881: + model_dir: mechanisms diff --git a/modeldb/modeldb.py b/modeldb/modeldb.py index 1c2a009..e72fb73 100644 --- a/modeldb/modeldb.py +++ b/modeldb/modeldb.py @@ -39,15 +39,20 @@ def download_model(arg_tuple): # we downloaded from the ModelDB API just above with a version from # GitHub github = model_run_info["github"] + organisation = "ModelDBRepository" + suffix = "" # default branch if github == "default": - suffix = "" + pass elif github.startswith("pull/"): pr_number = int(github[5:]) suffix = "/pull/{}/head".format(pr_number) + elif github.startswith('/'): + # /org implies that we use the default branch from org/model_id + organisation = github[1:] else: raise Exception("Invalid value for github key: {}".format(github)) - github_url = "https://api.github.com/repos/ModelDBRepository/{model_id}/zipball{suffix}".format( - model_id=model_id, suffix=suffix + github_url = "https://api.github.com/repos/{organisation}/{model_id}/zipball{suffix}".format( + model_id=model_id, organisation=organisation, suffix=suffix ) # Replace the local file `model_zip_uri` with the zip file we # downloaded from `github_url` diff --git a/modeldb/modelrun.py b/modeldb/modelrun.py index 83b4cf3..f71bb44 100644 --- a/modeldb/modelrun.py +++ b/modeldb/modelrun.py @@ -99,9 +99,12 @@ def append_log(model, model_sink, text): model_sink.extend(curate_log_string(model, text).split('\n')) -def run_commands(model, cmds, work_dir=None): +def run_commands(model, cmds, env={}, work_dir=None): + full_env = os.environ + full_env.update(env) out, _ = subprocess.Popen( cmds, + env=full_env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, @@ -134,7 +137,21 @@ def clean_model_dir(model): def compile_mods(model, mods): - run_commands(model, ["nrnivmodl"] + mods, work_dir=model.run_info["start_dir"]) + # Unfortunately nrnivmodl doesn't have an option to steer how much build + # parallellism it tries to do, it just hardcodes `make -j 4`. Because we + # parallelise over models, at a higher level, we want to remove this + # internal parallelism from nrnivmodl. In the CI we install NEURON using + # pip from precompiled wheels, and the real nrnivmodl is hidden behind + # an extra layer of wrappers. This makes it inconvenient to change the + # hardcoded value. Instead, we try to achieve the same effect using Make's + # environment variables. --max-load 0.0 should ban >1 job being launched if + # the system load is larger than zero. + run_commands( + model, + ["nrnivmodl"] + mods, + env={"MAKEFLAGS": " --max-load 0.0"}, + work_dir=model.run_info["start_dir"], + ) def build_driver_hoc(model):