Merge pull request #50 from joezuntz/git_fallback

Add a way to avoid launching subprocesses

cosmosis/main.py

@@ -163,6 +163,7 @@ def setup_output(sampler_class, sampler_number, ini, pool, number_samplers, samp
 
 
 def run_cosmosis(args, pool=None, ini=None, pipeline=None, values=None):
+    no_subprocesses = os.environ.get("COSMOSIS_NO_SUBPROCESS", "") not in ["", "0"]
     # In case we need to hand-hold a naive demo-10 user.
 
     # Load configuration.
@@ -173,12 +174,16 @@ def run_cosmosis(args, pool=None, ini=None, pipeline=None, values=None):
     pre_script = ini.get(RUNTIME_INI_SECTION, "pre_script", fallback="")
     post_script = ini.get(RUNTIME_INI_SECTION, "post_script", fallback="")
 
-    if is_root:
-        # This decodes the exist status
-        status = os.WEXITSTATUS(os.system(pre_script))
-        if status:
-            raise RuntimeError("The pre-run script {} retuned non-zero status {}".format(
-                pre_script, status))
+    if is_root and pre_script:
+        if no_subprocesses:
+            print("Warning: subprocesses not allowed on this system as")
+            print("COSMOSIS_NO_SUBPROCESS variable was set.")
+            print("Ignoring pre-script.")
+        else:
+            status = os.WEXITSTATUS(os.system(pre_script))
+            if status:
+                raise RuntimeError("The pre-run script {} retuned non-zero status {}".format(
+                    pre_script, status))
 
     if is_root and args.mem:
         from cosmosis.runtime.memmon import MemoryMonitor
@@ -325,10 +330,15 @@ def run_cosmosis(args, pool=None, ini=None, pipeline=None, values=None):
     # But we still offer it
     if post_script and is_root:
         # This decodes the exist status
-        status = os.WEXITSTATUS(os.system(post_script))
-        if status:
-            sys.stdout.write("WARNING: The post-run script {} failed with error {}".format(
-                post_script, error))
+        if no_subprocesses:
+            print("Warning: subprocesses not allowed on this system as")
+            print("COSMOSIS_NO_SUBPROCESS variable was set.")
+            print("Ignoring post-script.")
+        else:
+            status = os.WEXITSTATUS(os.system(post_script))
+            if status:
+                sys.stdout.write("WARNING: The post-run script {} failed with error {}".format(
+                    post_script, error))
 
     return 0
 

cosmosis/test/test_pipeline.py

@@ -8,6 +8,7 @@
 import os
 import tempfile
 import pstats
+import pytest
 
 root = os.path.split(os.path.abspath(__file__))[0]
 
@@ -168,6 +169,45 @@ def test_profile(capsys):
 
 
 
+def test_script_skip():
+    with tempfile.TemporaryDirectory() as dirname:
+        values_file = f"{dirname}/values.ini"
+        params_file = f"{dirname}/params.ini"
+        output_file = f"{dirname}/output.txt"
+        with open(values_file, "w") as values:
+            values.write(
+                "[parameters]\n"
+                "p1=-3.0  0.0  3.0\n"
+                "p2=-3.0  0.0  3.0\n")
+
+        params = {
+            ('runtime', 'root'): os.path.split(os.path.abspath(__file__))[0],
+            ('runtime', 'pre_script'): "this_executable_does_not_exist",
+            ('runtime', 'post_script'): "this_executable_does_not_exist",
+            ('runtime', 'sampler'):  "test",
+            ("pipeline", "debug"): "F",
+            ("pipeline", "quiet"): "F",
+            ("pipeline", "modules"): "test1",
+            ("pipeline", "values"): values_file,
+            ("test1", "file"): "test_module.py",
+            ("output", "filename"): output_file,
+        }
+
+        args = parser.parse_args(["not_a_real_file"])
+        ini = Inifile(None, override=params)
+
+        with pytest.raises(RuntimeError):
+            status = run_cosmosis(args, ini=ini)
+
+        # shopuld work this time
+        try:
+            os.environ["COSMOSIS_NO_SUBPROCESS"] = "1"
+            status = run_cosmosis(args, ini=ini)
+        finally:
+            del os.environ["COSMOSIS_NO_SUBPROCESS"]
+
+
+
+
 if __name__ == '__main__':
-    test_add_param()
-    test_missing_setup()
+    test_script_skip()

cosmosis/test/test_utils.py

@@ -0,0 +1,91 @@
+import cosmosis.utils
+import tempfile
+import os
+import contextlib
+import subprocess
+import pytest
+import sys
+
+dulwich_orig = cosmosis.utils.dulwich
+
+
+@contextlib.contextmanager
+def setup_git_repo():
+    with tempfile.TemporaryDirectory() as dirname:
+        repo_dir = f"{dirname}/repo"
+        repo_subdir = f"{repo_dir}/subdir"
+        os.mkdir(repo_dir)
+        os.mkdir(repo_subdir)
+
+        cmd = ["git", "init", "."]
+        p = subprocess.run(cmd, cwd=repo_dir)
+        cmd = ["git", "config", "--local", "user.email", "[email protected]"]
+        p = subprocess.run(cmd, cwd=repo_dir)
+        cmd = ["git", "config", "--local", "user.name", "Test User"]
+        p = subprocess.run(cmd, cwd=repo_dir)
+
+
+        with open(f"{repo_subdir}/f.txt", "w") as f:
+            f.write("hello\n")
+
+        cmd = ["git", "add", "subdir/f.txt"]
+        p = subprocess.run(cmd, cwd=repo_dir)
+
+        cmd = ["git", "commit", "-m", "added_file"]
+        p = subprocess.run(cmd, cwd=repo_dir)
+
+        cmd = ["git", "log"]
+        p = subprocess.run(cmd, cwd=repo_dir, capture_output=True, universal_newlines=True)
+        print('log stdout:', p.stdout)
+        print('log stderr:', p.stderr)
+        sha = p.stdout.split("\n")[0].split()[1]
+
+        yield sha, repo_dir, repo_subdir
+
+@pytest.mark.skipif(sys.version_info < (3, 7), reason="test requires python3.7 or higher")
+@pytest.mark.skipif(dulwich_orig is None, reason="dulwich not installed")
+def test_dulwich_git_path1():
+    with setup_git_repo() as info:
+        sha, repo_dir, repo_subdir = info
+        sha2 = cosmosis.utils.get_git_revision_dulwich(repo_dir)
+        sha3 = cosmosis.utils.get_git_revision_dulwich(repo_subdir)
+        assert sha == sha2
+        assert sha == sha3
+
+@pytest.mark.skipif(sys.version_info < (3, 7), reason="test requires python3.7 or higher")
+@pytest.mark.skipif(dulwich_orig is None, reason="dulwich not installed")
+def test_dulwich_git_path2():
+    with setup_git_repo() as info:
+        sha, repo_dir, repo_subdir = info
+        sha2 = cosmosis.utils.get_git_revision(repo_dir)
+        sha3 = cosmosis.utils.get_git_revision(repo_subdir)
+        assert sha == sha2
+        assert sha == sha3
+
+@pytest.mark.skipif(sys.version_info < (3, 7), reason="test requires python3.7 or higher")
+def test_git_fallback():
+    cosmosis.utils.dulwich = None
+    try:
+        with setup_git_repo() as info:
+            sha, repo_dir, repo_subdir = info
+            sha2 = cosmosis.utils.get_git_revision(repo_dir)
+            sha3 = cosmosis.utils.get_git_revision(repo_subdir)
+            assert sha == sha2
+            assert sha == sha3
+    finally:
+        cosmosis.utils.dulwich = dulwich_orig
+
+@pytest.mark.skipif(sys.version_info < (3, 7), reason="test requires python3.7 or higher")
+def test_git_nosub():
+    os.environ["COSMOSIS_NO_SUBPROCESS"] = "1"
+    cosmosis.utils.dulwich = None
+    try:
+        with setup_git_repo() as info:
+            sha, repo_dir, repo_subdir = info
+            sha2 = cosmosis.utils.get_git_revision(repo_dir)
+            sha3 = cosmosis.utils.get_git_revision(repo_subdir)
+            assert sha2 == ""
+            assert sha3 == ""
+    finally:
+        cosmosis.utils.dulwich = dulwich_orig
+        del os.environ["COSMOSIS_NO_SUBPROCESS"]

cosmosis/utils.py

@@ -14,7 +14,11 @@
 import tempfile
 import subprocess
 from functools import wraps
-
+import pathlib
+try:
+    import dulwich
+except:
+    dulwich = None
 
 
 class EverythingIsNan:
@@ -292,12 +296,30 @@ def evaluate_logp_all(self, p_in):
         """
         return self._evaluate(p_in, self.all_priors)
 
+def get_git_revision_dulwich(directory):
+    import dulwich.repo
+    path = pathlib.Path(directory)
+    while not (path / ".git").exists():
+        if str(path) == path.root:
+            # not a git repo
+            return ""
+        path = path.parent.absolute()
+    repo = dulwich.repo.Repo(path)
+    return repo.head().decode('utf-8')
+
 
 def get_git_revision(directory):
     # Turn e.g. $COSMOSIS_SRC_DIR into a proper path
     directory = os.path.expandvars(directory)
     if not os.path.isdir(directory):
         return ""
+
+    if dulwich is not None:
+        return get_git_revision_dulwich(directory)
+
+    if os.environ.get("COSMOSIS_NO_SUBPROCESS", "") not in ["", "0"]:
+        return ""
+
     # this git command gives the current commit ID of the
     # directory it is run from
     cmd = "git rev-parse HEAD".split()
@@ -335,4 +357,4 @@ def import_by_path(name, path):
     spec = importlib.util.spec_from_file_location(name, path)
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)
-    return module
+    return module

setup.py

@@ -183,6 +183,7 @@ def run(self):
     "emcee",
     "dynesty",
     "zeus-mcmc",
+    "dulwich"
 ]
 
 all_package_files = (datablock_libs + sampler_libs