diff --git a/cosmosis/main.py b/cosmosis/main.py index f92e9218..1fd46276 100755 --- a/cosmosis/main.py +++ b/cosmosis/main.py @@ -163,6 +163,7 @@ def setup_output(sampler_class, sampler_number, ini, pool, number_samplers, samp def run_cosmosis(args, pool=None, ini=None, pipeline=None, values=None): + no_subprocesses = os.environ.get("COSMOSIS_NO_SUBPROCESS", "") not in ["", "0"] # In case we need to hand-hold a naive demo-10 user. # Load configuration. @@ -173,12 +174,16 @@ def run_cosmosis(args, pool=None, ini=None, pipeline=None, values=None): pre_script = ini.get(RUNTIME_INI_SECTION, "pre_script", fallback="") post_script = ini.get(RUNTIME_INI_SECTION, "post_script", fallback="") - if is_root: - # This decodes the exist status - status = os.WEXITSTATUS(os.system(pre_script)) - if status: - raise RuntimeError("The pre-run script {} retuned non-zero status {}".format( - pre_script, status)) + if is_root and pre_script: + if no_subprocesses: + print("Warning: subprocesses not allowed on this system as") + print("COSMOSIS_NO_SUBPROCESS variable was set.") + print("Ignoring pre-script.") + else: + status = os.WEXITSTATUS(os.system(pre_script)) + if status: + raise RuntimeError("The pre-run script {} retuned non-zero status {}".format( + pre_script, status)) if is_root and args.mem: from cosmosis.runtime.memmon import MemoryMonitor @@ -325,10 +330,15 @@ def run_cosmosis(args, pool=None, ini=None, pipeline=None, values=None): # But we still offer it if post_script and is_root: # This decodes the exist status - status = os.WEXITSTATUS(os.system(post_script)) - if status: - sys.stdout.write("WARNING: The post-run script {} failed with error {}".format( - post_script, error)) + if no_subprocesses: + print("Warning: subprocesses not allowed on this system as") + print("COSMOSIS_NO_SUBPROCESS variable was set.") + print("Ignoring post-script.") + else: + status = os.WEXITSTATUS(os.system(post_script)) + if status: + sys.stdout.write("WARNING: The post-run script {} failed with error {}".format( + post_script, error)) return 0 diff --git a/cosmosis/test/test_pipeline.py b/cosmosis/test/test_pipeline.py index b7213f2f..fbec3b2d 100644 --- a/cosmosis/test/test_pipeline.py +++ b/cosmosis/test/test_pipeline.py @@ -8,6 +8,7 @@ import os import tempfile import pstats +import pytest root = os.path.split(os.path.abspath(__file__))[0] @@ -168,6 +169,45 @@ def test_profile(capsys): +def test_script_skip(): + with tempfile.TemporaryDirectory() as dirname: + values_file = f"{dirname}/values.ini" + params_file = f"{dirname}/params.ini" + output_file = f"{dirname}/output.txt" + with open(values_file, "w") as values: + values.write( + "[parameters]\n" + "p1=-3.0 0.0 3.0\n" + "p2=-3.0 0.0 3.0\n") + + params = { + ('runtime', 'root'): os.path.split(os.path.abspath(__file__))[0], + ('runtime', 'pre_script'): "this_executable_does_not_exist", + ('runtime', 'post_script'): "this_executable_does_not_exist", + ('runtime', 'sampler'): "test", + ("pipeline", "debug"): "F", + ("pipeline", "quiet"): "F", + ("pipeline", "modules"): "test1", + ("pipeline", "values"): values_file, + ("test1", "file"): "test_module.py", + ("output", "filename"): output_file, + } + + args = parser.parse_args(["not_a_real_file"]) + ini = Inifile(None, override=params) + + with pytest.raises(RuntimeError): + status = run_cosmosis(args, ini=ini) + + # shopuld work this time + try: + os.environ["COSMOSIS_NO_SUBPROCESS"] = "1" + status = run_cosmosis(args, ini=ini) + finally: + del os.environ["COSMOSIS_NO_SUBPROCESS"] + + + + if __name__ == '__main__': - test_add_param() - test_missing_setup() + test_script_skip() diff --git a/cosmosis/test/test_utils.py b/cosmosis/test/test_utils.py new file mode 100644 index 00000000..50e81e0c --- /dev/null +++ b/cosmosis/test/test_utils.py @@ -0,0 +1,91 @@ +import cosmosis.utils +import tempfile +import os +import contextlib +import subprocess +import pytest +import sys + +dulwich_orig = cosmosis.utils.dulwich + + +@contextlib.contextmanager +def setup_git_repo(): + with tempfile.TemporaryDirectory() as dirname: + repo_dir = f"{dirname}/repo" + repo_subdir = f"{repo_dir}/subdir" + os.mkdir(repo_dir) + os.mkdir(repo_subdir) + + cmd = ["git", "init", "."] + p = subprocess.run(cmd, cwd=repo_dir) + cmd = ["git", "config", "--local", "user.email", "test@user.com"] + p = subprocess.run(cmd, cwd=repo_dir) + cmd = ["git", "config", "--local", "user.name", "Test User"] + p = subprocess.run(cmd, cwd=repo_dir) + + + with open(f"{repo_subdir}/f.txt", "w") as f: + f.write("hello\n") + + cmd = ["git", "add", "subdir/f.txt"] + p = subprocess.run(cmd, cwd=repo_dir) + + cmd = ["git", "commit", "-m", "added_file"] + p = subprocess.run(cmd, cwd=repo_dir) + + cmd = ["git", "log"] + p = subprocess.run(cmd, cwd=repo_dir, capture_output=True, universal_newlines=True) + print('log stdout:', p.stdout) + print('log stderr:', p.stderr) + sha = p.stdout.split("\n")[0].split()[1] + + yield sha, repo_dir, repo_subdir + +@pytest.mark.skipif(sys.version_info < (3, 7), reason="test requires python3.7 or higher") +@pytest.mark.skipif(dulwich_orig is None, reason="dulwich not installed") +def test_dulwich_git_path1(): + with setup_git_repo() as info: + sha, repo_dir, repo_subdir = info + sha2 = cosmosis.utils.get_git_revision_dulwich(repo_dir) + sha3 = cosmosis.utils.get_git_revision_dulwich(repo_subdir) + assert sha == sha2 + assert sha == sha3 + +@pytest.mark.skipif(sys.version_info < (3, 7), reason="test requires python3.7 or higher") +@pytest.mark.skipif(dulwich_orig is None, reason="dulwich not installed") +def test_dulwich_git_path2(): + with setup_git_repo() as info: + sha, repo_dir, repo_subdir = info + sha2 = cosmosis.utils.get_git_revision(repo_dir) + sha3 = cosmosis.utils.get_git_revision(repo_subdir) + assert sha == sha2 + assert sha == sha3 + +@pytest.mark.skipif(sys.version_info < (3, 7), reason="test requires python3.7 or higher") +def test_git_fallback(): + cosmosis.utils.dulwich = None + try: + with setup_git_repo() as info: + sha, repo_dir, repo_subdir = info + sha2 = cosmosis.utils.get_git_revision(repo_dir) + sha3 = cosmosis.utils.get_git_revision(repo_subdir) + assert sha == sha2 + assert sha == sha3 + finally: + cosmosis.utils.dulwich = dulwich_orig + +@pytest.mark.skipif(sys.version_info < (3, 7), reason="test requires python3.7 or higher") +def test_git_nosub(): + os.environ["COSMOSIS_NO_SUBPROCESS"] = "1" + cosmosis.utils.dulwich = None + try: + with setup_git_repo() as info: + sha, repo_dir, repo_subdir = info + sha2 = cosmosis.utils.get_git_revision(repo_dir) + sha3 = cosmosis.utils.get_git_revision(repo_subdir) + assert sha2 == "" + assert sha3 == "" + finally: + cosmosis.utils.dulwich = dulwich_orig + del os.environ["COSMOSIS_NO_SUBPROCESS"] diff --git a/cosmosis/utils.py b/cosmosis/utils.py index ee8ad72f..e2ebabf9 100644 --- a/cosmosis/utils.py +++ b/cosmosis/utils.py @@ -14,7 +14,11 @@ import tempfile import subprocess from functools import wraps - +import pathlib +try: + import dulwich +except: + dulwich = None class EverythingIsNan: @@ -292,12 +296,30 @@ def evaluate_logp_all(self, p_in): """ return self._evaluate(p_in, self.all_priors) +def get_git_revision_dulwich(directory): + import dulwich.repo + path = pathlib.Path(directory) + while not (path / ".git").exists(): + if str(path) == path.root: + # not a git repo + return "" + path = path.parent.absolute() + repo = dulwich.repo.Repo(path) + return repo.head().decode('utf-8') + def get_git_revision(directory): # Turn e.g. $COSMOSIS_SRC_DIR into a proper path directory = os.path.expandvars(directory) if not os.path.isdir(directory): return "" + + if dulwich is not None: + return get_git_revision_dulwich(directory) + + if os.environ.get("COSMOSIS_NO_SUBPROCESS", "") not in ["", "0"]: + return "" + # this git command gives the current commit ID of the # directory it is run from cmd = "git rev-parse HEAD".split() @@ -335,4 +357,4 @@ def import_by_path(name, path): spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - return module \ No newline at end of file + return module diff --git a/setup.py b/setup.py index 1b49d552..09dc9131 100644 --- a/setup.py +++ b/setup.py @@ -183,6 +183,7 @@ def run(self): "emcee", "dynesty", "zeus-mcmc", + "dulwich" ] all_package_files = (datablock_libs + sampler_libs