Separated conf dict file discovery and loading from experiment._from_…

…dict and added tests for the new functions.

naklar/experiment.py

@@ -47,6 +47,20 @@ def _decorate_function(f, f_code, d, key):
     return d
 
 
+def _find_conf_files(root_dir, dict_filename='conf.pkl'):
+    for root, _, files in os.walk(root_dir, topdown=False):
+        if dict_filename in files:
+            pth = os.path.join(root, dict_filename)
+            with open(pth, 'r') as fh:
+                yield fh
+
+
+def _read_conf_dicts(itr):
+    for fh in itr:
+        d = pickle.load(fh)
+        yield d
+
+
 def _from_dict(root_dir, dict_filename='conf.pkl', primary_keys=['id'],
                autoload=True, decorators={}):
     conf = {}

test/test_experiment.py

@@ -0,0 +1,33 @@
+__author__ = 'mattilyra'
+
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+import pytest
+
+from naklar.experiment import _find_conf_files
+from naklar.experiment import _read_conf_dicts
+
+
+@pytest.fixture
+def experiment_tree(tmpdir):
+    for i in xrange(10):
+        pth = tmpdir.mkdir('exp{}'.format(i)).join('conf.pkl')
+        d = {'a': 1, 'b': 2}
+        pth.write(pickle.dumps(d))
+    return tmpdir
+
+
+def test_conf_discovery(experiment_tree):
+    confs = _find_conf_files(experiment_tree.strpath)
+    assert(len(list(confs)) == 10)
+    for i, fh in enumerate(confs):
+        assert(fh.name.endswith('conf.pkl'))
+
+
+def test_conf_dict_load(experiment_tree):
+    confs = _find_conf_files(experiment_tree.strpath)
+    dicts = _read_conf_dicts(confs)
+    assert(len(list(dicts)) == 10)