Merge branch 'FABLE-3DXRD:master' into master

ImageD11/nbGui/segmenter_gui.py

@@ -1,81 +1,11 @@
-import os
 import numpy as np
 import matplotlib.pyplot as plt
 from ipywidgets import interact, interactive, widgets, fixed, Layout
 from IPython.display import display
 import h5py
 import ImageD11.sinograms.lima_segmenter
 import ImageD11.sparseframe
-
-
-def guess_ESRF_paths():  # This should be in silx somewhere?
-    """ Locates:
-    dataroot     holds raw data       in folders dataroot     + {sample}/{sample}_{dataset}
-    analysisroot holds output results in folders analysisroot + {sample}/{sample}_{dataset}
-    """
-    path_items = os.getcwd().split('/')
-    if 'visitor' in path_items:
-        idx = path_items.index('visitor')
-        experiment, session = path_items[idx + 1], path_items[idx + 3]
-        path = os.path.join("/data", "visitor", experiment, "id11", session)
-        return [os.path.join(path, folder) for folder in
-                ("RAW_DATA", "PROCESSED_DATA")]
-    return "", ""
-
-
-def printsamples(dataroot):
-    samples = sorted([name for name in os.listdir(dataroot)
-                      if os.path.isdir(os.path.join(dataroot, name))])
-    print("Samples:\n\t " + "\n\t".join(sorted(samples)))
-
-
-def printdatasets(dataroot, sample):
-    sroot = os.path.join(dataroot, sample)
-    print("Datsets:\n\t " + "\n\t".join(sorted(
-        [name[len(sample) + 1:] for name in os.listdir(sroot)
-         if os.path.isdir(os.path.join(sroot, name))
-         and name.startswith(sample)])))
-
-
-def chooseframe(dset, scan=None, idx=None, counter="_roi1", fetch_raw_image=False):
-    """
-    Locate a busy frame from the dataset and optionally fetch the raw image.
-
-    Args:
-        dset: Dataset object.
-        scan: Scan to use (default is middle scan).
-        idx: Frame index (None to auto-detect).
-        counter: Counter to use for locating the frame.
-        fetch_raw_image: Whether to fetch the raw image for the frame.
-
-    Returns:
-        scan, idx, raw_image (if fetch_raw_image is True, otherwise None)
-    """
-    if scan is None:
-        scan = dset.scans[len(dset.scans) // 2]
-
-    raw_image = None
-    with h5py.File(dset.masterfile, "r") as hin:
-        ctr = dset.detector + counter
-
-        if idx is None:
-            if scan.find("::") > -1:  # 1.1::[10000:12000]  etc
-                lo, hi = [int(v) for v in scan[:-1].split("[")[1].split(":")]
-                scan = scan.split("::")[0]
-                roi1 = hin[scan]["measurement"][ctr][lo:hi]
-                idx = np.argmax(roi1) + lo
-            else:  # "1.1"
-                roi1 = hin[scan]["measurement"][ctr][:]
-                idx = np.argmax(roi1)
-
-        print("Using frame", idx, "from scan", scan)
-
-        # placing the raw image
-        if fetch_raw_image:
-            raw_image = hin[scan + "/measurement/" + dset.detector][idx]
-
-    return scan, idx, raw_image
-
+from ImageD11.nbGui.segmenter_gui_utils import * 
 
 class SegmenterGui:
     """ UI for a jupyter notebook to set the segmentation parameters

ImageD11/nbGui/segmenter_gui_utils.py

@@ -0,0 +1,72 @@
+import h5py
+import numpy as np
+import os
+
+
+def guess_ESRF_paths():  # This should be in silx somewhere?
+    """ Locates:
+    dataroot     holds raw data       in folders dataroot     + {sample}/{sample}_{dataset}
+    analysisroot holds output results in folders analysisroot + {sample}/{sample}_{dataset}
+    """
+    path_items = os.getcwd().split('/')
+    if 'visitor' in path_items:
+        idx = path_items.index('visitor')
+        experiment, session = path_items[idx + 1], path_items[idx + 3]
+        path = os.path.join("/data", "visitor", experiment, "id11", session)
+        return [os.path.join(path, folder) for folder in
+                ("RAW_DATA", "PROCESSED_DATA")]
+    return "", ""
+
+
+def printsamples(dataroot):
+    samples = sorted([name for name in os.listdir(dataroot)
+                      if os.path.isdir(os.path.join(dataroot, name))])
+    print("Samples:\n\t " + "\n\t".join(sorted(samples)))
+
+
+def printdatasets(dataroot, sample):
+    sroot = os.path.join(dataroot, sample)
+    print("Datsets:\n\t " + "\n\t".join(sorted(
+        [name[len(sample) + 1:] for name in os.listdir(sroot)
+         if os.path.isdir(os.path.join(sroot, name))
+         and name.startswith(sample)])))
+
+
+def chooseframe(dset, scan=None, idx=None, counter="_roi1", fetch_raw_image=False):
+    """
+    Locate a busy frame from the dataset and optionally fetch the raw image.
+
+    Args:
+        dset: Dataset object.
+        scan: Scan to use (default is middle scan).
+        idx: Frame index (None to auto-detect).
+        counter: Counter to use for locating the frame.
+        fetch_raw_image: Whether to fetch the raw image for the frame.
+
+    Returns:
+        scan, idx, raw_image (if fetch_raw_image is True, otherwise None)
+    """
+    if scan is None:
+        scan = dset.scans[len(dset.scans) // 2]
+
+    raw_image = None
+    with h5py.File(dset.masterfile, "r") as hin:
+        ctr = dset.detector + counter
+
+        if idx is None:
+            if scan.find("::") > -1:  # 1.1::[10000:12000]  etc
+                lo, hi = [int(v) for v in scan[:-1].split("[")[1].split(":")]
+                scan = scan.split("::")[0]
+                roi1 = hin[scan]["measurement"][ctr][lo:hi]
+                idx = np.argmax(roi1) + lo
+            else:  # "1.1"
+                roi1 = hin[scan]["measurement"][ctr][:]
+                idx = np.argmax(roi1)
+
+        print("Using frame", idx, "from scan", scan)
+
+        # placing the raw image
+        if fetch_raw_image:
+            raw_image = hin[scan + "/measurement/" + dset.detector][idx]
+
+    return scan, idx, raw_image

ImageD11/sandbox/__init__.py

@@ -0,0 +1,8 @@
+
+"""
+ImageD11.sandbox is for development/testing and not general usage
+
+If there is something in here that you need then please clean it up
+and do 'git mv my_useful_thing.py somewhere_else'
+"""
+

ImageD11/sinograms/properties.py

@@ -37,41 +37,43 @@
 
 import multiprocessing as mp
 try:
-    from multiprocessing import shared_memory, resource_tracker
+    from multiprocessing import shared_memory
 except ImportError:
     import warnings
     warnings.warn('python2: multiprocessing lacks shared memory, please backport it')
     shared_memory = None
-    resource_tracker = None
 
-#####################################################################################
-def remove_shm_from_resource_tracker():
-    """Monkey-patch multiprocessing.resource_tracker so SharedMemory won't be tracked
-    More details at: https://bugs.python.org/issue38119
-    """
-    self = None
-
-    def fix_register(name, rtype):
-        if rtype == "shared_memory":
-            return
-        return resource_tracker._resource_tracker.register(name, rtype)
-
-    if resource_tracker.register is not fix_register:
-        resource_tracker.register = fix_register
-
-    def fix_unregister(name, rtype):
-        if rtype == "shared_memory":
-            return
-        return resource_tracker._resource_tracker.unregister(name, rtype)
 
-    if resource_tracker.unregister is not fix_register:
-        resource_tracker.unregister = fix_register
-    resource_tracker.unregister = fix_unregister
-    if "shared_memory" in resource_tracker._CLEANUP_FUNCS:
-        del resource_tracker._CLEANUP_FUNCS["shared_memory"]
-
-
-######################################################################################
+# The reason for this was not entirely clear. It was probably clean up of an OOM issue.
+if os.environ.get('IMAGED11_PATCH_MP', None) != 'MONKEYPATCH':
+    def remove_shm_from_resource_tracker():
+        pass
+else:
+    #####################################################################################
+    from multiprocessing import resource_tracker
+    import warnings
+    warnings.warn('Applying monkeypatch to multiprocessing shared memory')
+    def remove_shm_from_resource_tracker():
+        """Monkey-patch multiprocessing.resource_tracker so SharedMemory won't be tracked
+        More details at: https://bugs.python.org/issue38119
+        """
+        self = None
+        def fix_register(name, rtype):
+            if rtype == "shared_memory":
+                return
+            return resource_tracker._resource_tracker.register(name, rtype)
+        if resource_tracker.register is not fix_register:
+            resource_tracker.register = fix_register
+        def fix_unregister(name, rtype):
+            if rtype == "shared_memory":
+                return
+            return resource_tracker._resource_tracker.unregister(name, rtype)
+        if resource_tracker.unregister is not fix_register:
+            resource_tracker.unregister = fix_register
+        resource_tracker.unregister = fix_unregister
+        if "shared_memory" in resource_tracker._CLEANUP_FUNCS:
+            del resource_tracker._CLEANUP_FUNCS["shared_memory"]
+    ######################################################################################
 
 
 class shared_numpy_array:

setup.py

@@ -154,6 +154,8 @@ def build_extension(self, ext):
     "xfab>=0.0.4", #
        # comes from xfab : "PyCifRW",
     "matplotlib",  # tkGui
+    'pyopengl==3.1.7 ; python_version < "3"',
+    'pyopengl ; python_version >= "3"',
     "pyopengltk",  # plot3d in tkGui
     "scipy",       #
     'hdf5plugin==1.4.1 ; python_version < "3"',   #  and sys_platform == "win32" ',
@@ -221,6 +223,7 @@ def build_extension(self, ext):
                   "ImageD11.silxGui",
                   "ImageD11.nbGui",
                   "ImageD11.sinograms",
+                  "ImageD11.sandbox",
                  ],
       package_dir = {"ImageD11":"ImageD11"},
       url = "http://github.com/jonwright/ImageD11",