Add a functional Dataset class

.circleci/config.yml

@@ -2,7 +2,7 @@ apt-run: &apt-install
   name: Install apt packages
   command: |
     sudo apt update
-    sudo apt install -y graphviz npm node
+    sudo apt install -y graphviz
 
 deps-run: &cihelpers-install
   name: Install CI Helpers
@@ -23,21 +23,13 @@ pip-run: &pip-install
     . venv/bin/activate
     pip install -q --user $PIP_DEPENDENCIES
 
-report-status: &report-status
-  name: Report CI Status
-  command: |
-    sudo npm install -g commit-status
-    if [[ ! -z $CIRCLE_PR_NUMBER ]]; then
-        commit-status success docs "Your documentation built" $DOCS_URL
-    fi
-
 version: 2
 jobs:
   build:
     environment:
-      - PIP_DEPENDENCIES: "sphinx git+https://bitbucket.org/dkistdc/dkist-sphinx-theme.git sunpy asdf gwcs"
+      - PIP_DEPENDENCIES: "sphinx git+https://bitbucket.org/dkistdc/dkist-sphinx-theme.git sunpy git+https://github.com/spacetelescope/asdf gwcs dask[complete] astropy ndcube"
       - MAIN_CMD: "python setup.py"
-      - SETUP_CMD: "build_docs -W"
+      - SETUP_CMD: "build_docs -w"
     docker:
       - image: circleci/python:3.6
     steps:
@@ -52,5 +44,3 @@ jobs:
       - run:
           name: "Built documentation is available at:"
           command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/docs/_build/html/index.html"; echo $DOCS_URL
-
-      - run: *report-status

.travis.yml

@@ -29,11 +29,11 @@ env:
 
         # List other runtime dependencies for the package that are available as
         # conda packages here.
-        - CONDA_DEPENDENCIES=''
+        - CONDA_DEPENDENCIES='asdf gwcs'
 
         # List other runtime dependencies for the package that are available as
         # pip packages here.
-        - PIP_DEPENDENCIES='pytest-astropy'
+        - PIP_DEPENDENCIES='pytest-astropy git+https://github.com/sunpy/ndcube.git'
 
         # Conda packages for affiliated packages are hosted in channel
         # "astropy" while builds for astropy LTS with recent numpy versions
@@ -61,11 +61,6 @@ matrix:
         - os: linux
           env: SETUP_CMD='test --coverage'
 
-        # Check for sphinx doc build warnings - we do this first because it
-        # may run for a long time
-        - os: linux
-          env: SETUP_CMD='build_docs -w'
-
         # Do a PEP8 test with pycodestyle
         - os: linux
           env: MAIN_CMD='pycodestyle dkist --count' SETUP_CMD=''

ah_bootstrap.py

@@ -19,9 +19,14 @@
 contains an option called ``auto_use`` with a value of ``True``, it will
 automatically call the main function of this module called
 `use_astropy_helpers` (see that function's docstring for full details).
-Otherwise no further action is taken (however,
-``ah_bootstrap.use_astropy_helpers`` may be called manually from within the
-setup.py script).
+Otherwise no further action is taken and by default the system-installed version
+of astropy-helpers will be used (however, ``ah_bootstrap.use_astropy_helpers``
+may be called manually from within the setup.py script).
+
+This behavior can also be controlled using the ``--auto-use`` and
+``--no-auto-use`` command-line flags. For clarity, an alias for
+``--no-auto-use`` is ``--use-system-astropy-helpers``, and we recommend using
+the latter if needed.
 
 Additional options in the ``[ah_boostrap]`` section of setup.cfg have the same
 names as the arguments to `use_astropy_helpers`, and can be used to configure
@@ -47,14 +52,7 @@
     from configparser import ConfigParser, RawConfigParser
 
 
-if sys.version_info[0] < 3:
-    _str_types = (str, unicode)
-    _text_type = unicode
-    PY3 = False
-else:
-    _str_types = (str, bytes)
-    _text_type = str
-    PY3 = True
+_str_types = (str, bytes)
 
 
 # What follows are several import statements meant to deal with install-time
@@ -137,7 +135,6 @@
 
 from setuptools import Distribution
 from setuptools.package_index import PackageIndex
-from setuptools.sandbox import run_setup
 
 from distutils import log
 from distutils.debug import DEBUG
@@ -146,6 +143,7 @@
 # TODO: Maybe enable checking for a specific version of astropy_helpers?
 DIST_NAME = 'astropy-helpers'
 PACKAGE_NAME = 'astropy_helpers'
+UPPER_VERSION_EXCLUSIVE = None
 
 # Defaults for other options
 DOWNLOAD_IF_NEEDED = True
@@ -177,7 +175,7 @@ def __init__(self, path=None, index_url=None, use_git=None, offline=None,
         if not (isinstance(path, _str_types) or path is False):
             raise TypeError('path must be a string or False')
 
-        if PY3 and not isinstance(path, _text_type):
+        if not isinstance(path, str):
             fs_encoding = sys.getfilesystemencoding()
             path = path.decode(fs_encoding)  # path to unicode
 
@@ -287,6 +285,18 @@ def parse_command_line(cls, argv=None):
             config['offline'] = True
             argv.remove('--offline')
 
+        if '--auto-use' in argv:
+            config['auto_use'] = True
+            argv.remove('--auto-use')
+
+        if '--no-auto-use' in argv:
+            config['auto_use'] = False
+            argv.remove('--no-auto-use')
+
+        if '--use-system-astropy-helpers' in argv:
+            config['auto_use'] = False
+            argv.remove('--use-system-astropy-helpers')
+
         return config
 
     def run(self):
@@ -464,9 +474,10 @@ def _directory_import(self):
             # setup.py exists we can generate it
             setup_py = os.path.join(path, 'setup.py')
             if os.path.isfile(setup_py):
-                with _silence():
-                    run_setup(os.path.join(path, 'setup.py'),
-                              ['egg_info'])
+                # We use subprocess instead of run_setup from setuptools to
+                # avoid segmentation faults - see the following for more details:
+                # https://github.com/cython/cython/issues/2104
+                sp.check_output([sys.executable, 'setup.py', 'egg_info'], cwd=path)
 
                 for dist in pkg_resources.find_distributions(path, True):
                     # There should be only one...
@@ -501,16 +512,32 @@ def get_option_dict(self, command_name):
         if version:
             req = '{0}=={1}'.format(DIST_NAME, version)
         else:
-            req = DIST_NAME
+            if UPPER_VERSION_EXCLUSIVE is None:
+                req = DIST_NAME
+            else:
+                req = '{0}<{1}'.format(DIST_NAME, UPPER_VERSION_EXCLUSIVE)
 
         attrs = {'setup_requires': [req]}
 
+        # NOTE: we need to parse the config file (e.g. setup.cfg) to make sure
+        # it honours the options set in the [easy_install] section, and we need
+        # to explicitly fetch the requirement eggs as setup_requires does not
+        # get honored in recent versions of setuptools:
+        # https://github.com/pypa/setuptools/issues/1273
+
         try:
-            if DEBUG:
-                _Distribution(attrs=attrs)
-            else:
-                with _silence():
-                    _Distribution(attrs=attrs)
+
+            context = _verbose if DEBUG else _silence
+            with context():
+                dist = _Distribution(attrs=attrs)
+                try:
+                    dist.parse_config_files(ignore_option_errors=True)
+                    dist.fetch_build_eggs(req)
+                except TypeError:
+                    # On older versions of setuptools, ignore_option_errors
+                    # doesn't exist, and the above two lines are not needed
+                    # so we can just continue
+                    pass
 
             # If the setup_requires succeeded it will have added the new dist to
             # the main working_set
@@ -791,9 +818,9 @@ def run_cmd(cmd):
         stdio_encoding = 'latin1'
 
     # Unlikely to fail at this point but even then let's be flexible
-    if not isinstance(stdout, _text_type):
+    if not isinstance(stdout, str):
         stdout = stdout.decode(stdio_encoding, 'replace')
-    if not isinstance(stderr, _text_type):
+    if not isinstance(stderr, str):
         stderr = stderr.decode(stdio_encoding, 'replace')
 
     return (p.returncode, stdout, stderr)
@@ -846,6 +873,10 @@ def flush(self):
         pass
 
 
+@contextlib.contextmanager
+def _verbose():
+    yield
+
 @contextlib.contextmanager
 def _silence():
     """A context manager that silences sys.stdout and sys.stderr."""

dkist/dataset/dataset.py

@@ -1,48 +1,129 @@
+import copy
 import glob
 import os.path
 from pathlib import Path
 
+import asdf
+import numpy as np
+
+import astropy.units as u
+
 from ndcube.ndcube import NDCubeBase
 
 
-class Dataset(NDCubeBase):
-    """
-    Load a DKIST dataset.
+from dkist.dataset.mixins import DatasetPlotMixin
+from dkist.io import DaskFITSArrayContainer, AstropyFITSLoader
 
-    Parameters
-    ----------
+__all__ = ['Dataset']
+
+
+class Dataset(DatasetPlotMixin, NDCubeBase):
+    """
+    The base class for DKIST datasets.
 
-    directory : `str`
-        The directory holding the dataset.
+    This class is backed by `dask.array.Array` and `gwcs.wcs.WCS` objects.
     """
 
-    def __init__(self, directory):
+    @classmethod
+    def from_directory(cls, directory):
+        """
+        Construct a `~dkist.dataset.Dataset` from a directory containing one
+        asdf file and a collection of FITS files.
+        """
         if not os.path.isdir(directory):
             raise ValueError("directory argument must be a directory")
-        self.base_path = Path(directory)
-        asdf_files = glob.glob(self.base_path / "*.asdf")
+        base_path = Path(directory)
+        asdf_files = glob.glob(str(base_path / "*.asdf"))
 
         if not asdf_files:
             raise ValueError("No asdf file found in directory.")
         elif len(asdf_files) > 1:
             raise NotImplementedError("Multiple asdf files found in this"
                                       " directory. Can't handle this yet.")
 
-        self.asdf_file = asdf_files[0]
+        asdf_file = asdf_files[0]
 
-        # super().__init__(self, data, wcs)
+        with asdf.AsdfFile.open(asdf_file) as ff:
+            # TODO: without this it segfaults on access
+            asdf_tree = copy.deepcopy(ff.tree)
+            pointer_array = np.array(ff.tree['dataset'])
 
-    """
-    Methods to be implemented.
-    """
-    def pixel_to_world(self, quantity_axis_list, origin=0):
-        raise NotImplementedError()
+        array_container = DaskFITSArrayContainer(pointer_array, loader=AstropyFITSLoader,
+                                                 basepath=str(base_path))
+
+        data = array_container.array
+
+        wcs = asdf_tree['gwcs']
 
-    def world_to_pixel(self, quantity_axis_list, origin=0):
+        return cls(data, wcs=wcs)
+
+    def __repr__(self):
+        """
+        Overload the NDData repr because it does not play nice with the dask delayed io.
+        """
+        prefix = self.__class__.__name__ + '('
+        body = str(self.data)
+        return ''.join([prefix, body, ')'])
+
+    def pixel_to_world(self, *quantity_axis_list):
+        """
+        Convert a pixel coordinate to a data (world) coordinate by using
+        `~gwcs.wcs.WCS`.
+
+        Parameters
+        ----------
+        quantity_axis_list : iterable
+            An iterable of `~astropy.units.Quantity` with unit as pixel `pix`.
+            Note that these quantities must be entered as separate arguments, not as one list.
+
+        Returns
+        -------
+        coord : `list`
+            A list of arrays containing the output coordinates.
+        """
+        return tuple(self.wcs(*quantity_axis_list, output='numericals_plus'))
+
+    def world_to_pixel(self, *quantity_axis_list):
+        """
+        Convert a world coordinate to a data (pixel) coordinate by using
+        `~gwcs.wcs.WCS.invert`.
+
+        Parameters
+        ----------
+        quantity_axis_list : iterable
+            A iterable of `~astropy.units.Quantity`.
+            Note that these quantities must be entered as separate arguments, not as one list.
+
+        Returns
+        -------
+        coord : `list`
+            A list of arrays containing the output coordinates.
+        """
+        return tuple(self.wcs.invert(*quantity_axis_list, output="numericals_plus"))
+
+    def world_axis_physical_types(self):
         raise NotImplementedError()
 
+    @property
     def dimensions(self):
-        raise NotImplementedError()
+        """
+        The dimensions of the data as a `~astropy.units.Quantity`.
+        """
+        return u.Quantity(self.data.shape, unit=u.pix)
 
-    def crop_by_coords(self, lower_left_corner, dimension_widths):
-        raise NotImplementedError()
+    def crop_by_coords(self, min_coord_values, interval_widths):
+        # The docstring is defined in NDDataBase
+
+        n_dim = len(self.dimensions)
+        if len(min_coord_values) != len(interval_widths) != n_dim:
+            raise ValueError("min_coord_values and interval_widths must have "
+                             "same number of elements as number of data dimensions.")
+        # Convert coords of lower left corner to pixel units.
+        lower_pixels = self.world_to_pixel(*min_coord_values)
+        upper_pixels = self.world_to_pixel(*[min_coord_values[i] + interval_widths[i]
+                                             for i in range(len(min_coord_values))])
+        # Round pixel values to nearest integer.
+        lower_pixels = [int(np.rint(l.value)) for l in lower_pixels]
+        upper_pixels = [int(np.rint(u.value)) for u in upper_pixels]
+        item = tuple([slice(lower_pixels[i], upper_pixels[i]) for i in range(n_dim)])
+        return self.data[item]