Extract chart studio functionality, optimized imports (#1476)

## Overview
This PR is an important step towards the [version 4 goal](#1420) of removing all of the chart studio (i.e. cloud-service related) functionality from plotly.py, and putting it in a separate optional package.

## chart studio extraction
For the time being, I've done this by creating a new top-level `chart_studio` package next to the top-level `plotly` package.  I've moved all of the cloud-related functionality to the `chart_studio` package, following the same structure as in the current plotly package.  For example, the `plotly.plotly` module was moved to `chart_studio.plotly`.

This PR takes advantage of the `_future_plotly_` system introduced in #1474 to make this refactor backward compatible.

 - By default all of the old entry points are still usable and they are aliased to the `chart_studio` package.
 - If the `extract_chart_studio` future flag is set, then deprecation warnings are raised whenever the `chart_studio` modules/functions are used from their legacy locations under the `plotly` package.
 - If the `remove_deprecations` future flag is set then the chart studio functions are fully removed from the plotly package and are accessible only under `chart_studio`.  When `remove_deprecations`  is set, `plotly` has no dependency on the `chart_studio` package.

## Usage
To remove the chart_studio functionality from the main `plotly` module, use the

```python
from _plotly_future_ import remove_deprecations
```

This will further speed up imports, and will allow for testing code to make sure it will be compatible with the package structure of plotly.py version 4.


## Import optimization
This PR also makes a relatively minor change to the code generation logic for `graph_objs` and `validator` that yields an import time reduction of ~10-20% .  Rather that creating a single file for each datatype and validator class, all of the classes in a `graph_obj` or `validator` module are specified directly in the `__init__.py` file.  This reduces the number of files significantly, which seems to yield a modest but consistent speedup while being 100% backward compatible.

_plotly_future_/__init__.py

@@ -1,3 +1,6 @@
+import warnings
+import functools
+
 _future_flags = set()
 
 
@@ -6,3 +9,49 @@ def _assert_plotly_not_imported():
     if 'plotly' in sys.modules:
         raise ImportError("""\
 The _plotly_future_ module must be imported before the plotly module""")
+
+
+warnings.filterwarnings(
+    'default',
+    '.*?is deprecated, please use chart_studio*',
+    DeprecationWarning
+)
+
+
+def _chart_studio_warning(submodule):
+    if 'extract_chart_studio' in _future_flags:
+        warnings.warn(
+            'The plotly.{submodule} module is deprecated, '
+            'please use chart_studio.{submodule} instead'
+                .format(submodule=submodule),
+            DeprecationWarning,
+            stacklevel=2)
+
+
+def _chart_studio_deprecation(fn):
+
+    fn_name = fn.__name__
+    fn_module = fn.__module__
+    plotly_name = '.'.join(
+        ['plotly'] + fn_module.split('.')[1:] + [fn_name])
+    chart_studio_name = '.'.join(
+        ['chart_studio'] + fn_module.split('.')[1:] + [fn_name])
+
+    msg = """\
+{plotly_name} is deprecated, please use {chart_studio_name}\
+""".format(plotly_name=plotly_name, chart_studio_name=chart_studio_name)
+
+    @functools.wraps(fn)
+    def wrapper(*args, **kwargs):
+        if 'extract_chart_studio' in _future_flags:
+            warnings.warn(
+                msg,
+                DeprecationWarning,
+                stacklevel=2)
+
+        return fn(*args, **kwargs)
+
+    return wrapper
+
+
+__all__ = ['_future_flags', '_chart_studio_warning']

_plotly_future_/extract_chart_studio.py

@@ -0,0 +1,5 @@
+from __future__ import absolute_import
+from _plotly_future_ import _future_flags, _assert_plotly_not_imported
+
+_assert_plotly_not_imported()
+_future_flags.add('extract_chart_studio')

_plotly_future_/remove_deprecations.py

@@ -0,0 +1,5 @@
+from __future__ import absolute_import
+from _plotly_future_ import _future_flags, _assert_plotly_not_imported
+
+_assert_plotly_not_imported()
+_future_flags.add('remove_deprecations')

_plotly_future_/v4.py

@@ -1,2 +1,4 @@
 from __future__ import absolute_import
-from _plotly_future_ import renderer_defaults, template_defaults
+from _plotly_future_ import (
+    renderer_defaults, template_defaults, extract_chart_studio,
+    remove_deprecations)

_plotly_utils/basevalidators.py

@@ -1,33 +1,18 @@
+from __future__ import absolute_import
+
 import base64
 import numbers
 import textwrap
 import uuid
 from importlib import import_module
 import copy
-
 import io
 from copy import deepcopy
-
 import re
-
-# Optional imports
-# ----------------
 import sys
 from six import string_types
 
-np = None
-pd = None
-
-try:
-    np = import_module('numpy')
-
-    try:
-        pd = import_module('pandas')
-    except ImportError:
-        pass
-
-except ImportError:
-    pass
+from _plotly_utils.optional_imports import get_module
 
 
 # back-port of fullmatch from Py3.4+
@@ -50,6 +35,8 @@ def to_scalar_or_list(v):
     # Python native scalar type ('float' in the example above).
     # We explicitly check if is has the 'item' method, which conventionally
     # converts these types to native scalars.
+    np = get_module('numpy')
+    pd = get_module('pandas')
     if np and np.isscalar(v) and hasattr(v, 'item'):
         return v.item()
     if isinstance(v, (list, tuple)):
@@ -86,7 +73,8 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
     np.ndarray
         Numpy array with the 'WRITEABLE' flag set to False
     """
-
+    np = get_module('numpy')
+    pd = get_module('pandas')
     assert np is not None
 
     # ### Process kind ###
@@ -175,7 +163,9 @@ def is_numpy_convertable(v):
 def is_homogeneous_array(v):
     """
     Return whether a value is considered to be a homogeneous array
-    """    
+    """
+    np = get_module('numpy')
+    pd = get_module('pandas')
     if ((np and isinstance(v, np.ndarray) or
         (pd and isinstance(v, (pd.Series, pd.Index))))):
             return True
@@ -616,7 +606,7 @@ def description(self):
     as a plotly.grid_objs.Column object""".format(plotly_name=self.plotly_name))
 
     def validate_coerce(self, v):
-        from plotly.grid_objs import Column
+        from chart_studio.grid_objs import Column
         if v is None:
             # Pass None through
             pass
@@ -704,7 +694,7 @@ def validate_coerce(self, v):
             # Pass None through
             pass
         elif self.array_ok and is_homogeneous_array(v):
-
+            np = get_module('numpy')
             try:
                 v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
             except (ValueError, TypeError, OverflowError):
@@ -825,7 +815,7 @@ def validate_coerce(self, v):
             # Pass None through
             pass
         elif self.array_ok and is_homogeneous_array(v):
-
+            np = get_module('numpy')
             v_array = copy_to_readonly_numpy_array(v,
                                                    kind=('i', 'u'),
                                                    force_numeric=True)
@@ -964,6 +954,8 @@ def validate_coerce(self, v):
                     self.raise_invalid_elements(invalid_els)
 
             if is_homogeneous_array(v):
+                np = get_module('numpy')
+
                 # If not strict, let numpy cast elements to strings
                 v = copy_to_readonly_numpy_array(v, kind='U')
 

_plotly_utils/exceptions.py

@@ -0,0 +1,82 @@
+class PlotlyError(Exception):
+    pass
+
+
+class PlotlyEmptyDataError(PlotlyError):
+    pass
+
+
+class PlotlyGraphObjectError(PlotlyError):
+    def __init__(self, message='', path=(), notes=()):
+        """
+        General graph object error for validation failures.
+
+        :param (str|unicode) message: The error message.
+        :param (iterable) path: A path pointing to the error.
+        :param notes: Add additional notes, but keep default exception message.
+
+        """
+        self.message = message
+        self.plain_message = message  # for backwards compat
+        self.path = list(path)
+        self.notes = notes
+        super(PlotlyGraphObjectError, self).__init__(message)
+
+    def __str__(self):
+        """This is called by Python to present the error message."""
+        format_dict = {
+            'message': self.message,
+            'path': '[' + ']['.join(repr(k) for k in self.path) + ']',
+            'notes': '\n'.join(self.notes)
+        }
+        return ('{message}\n\nPath To Error: {path}\n\n{notes}'
+                .format(**format_dict))
+
+
+class PlotlyDictKeyError(PlotlyGraphObjectError):
+    def __init__(self, obj, path, notes=()):
+        """See PlotlyGraphObjectError.__init__ for param docs."""
+        format_dict = {'attribute': path[-1], 'object_name': obj._name}
+        message = ("'{attribute}' is not allowed in '{object_name}'"
+                   .format(**format_dict))
+        notes = [obj.help(return_help=True)] + list(notes)
+        super(PlotlyDictKeyError, self).__init__(
+            message=message, path=path, notes=notes
+        )
+
+
+class PlotlyDictValueError(PlotlyGraphObjectError):
+    def __init__(self, obj, path, notes=()):
+        """See PlotlyGraphObjectError.__init__ for param docs."""
+        format_dict = {'attribute': path[-1], 'object_name': obj._name}
+        message = ("'{attribute}' has invalid value inside '{object_name}'"
+                   .format(**format_dict))
+        notes = [obj.help(path[-1], return_help=True)] + list(notes)
+        super(PlotlyDictValueError, self).__init__(
+            message=message, notes=notes, path=path
+        )
+
+
+class PlotlyListEntryError(PlotlyGraphObjectError):
+    def __init__(self, obj, path, notes=()):
+        """See PlotlyGraphObjectError.__init__ for param docs."""
+        format_dict = {'index': path[-1], 'object_name': obj._name}
+        message = ("Invalid entry found in '{object_name}' at index, '{index}'"
+                   .format(**format_dict))
+        notes = [obj.help(return_help=True)] + list(notes)
+        super(PlotlyListEntryError, self).__init__(
+            message=message, path=path, notes=notes
+        )
+
+
+class PlotlyDataTypeError(PlotlyGraphObjectError):
+    def __init__(self, obj, path, notes=()):
+        """See PlotlyGraphObjectError.__init__ for param docs."""
+        format_dict = {'index': path[-1], 'object_name': obj._name}
+        message = ("Invalid entry found in '{object_name}' at index, '{index}'"
+                   .format(**format_dict))
+        note = "It's invalid because it doesn't contain a valid 'type' value."
+        notes = [note] + list(notes)
+        super(PlotlyDataTypeError, self).__init__(
+            message=message, path=path, notes=notes
+        )

_plotly_utils/files.py

@@ -0,0 +1,36 @@
+import os
+
+PLOTLY_DIR = os.environ.get("PLOTLY_DIR",
+                            os.path.join(os.path.expanduser("~"), ".plotly"))
+TEST_FILE = os.path.join(PLOTLY_DIR, ".permission_test")
+
+
+def _permissions():
+    try:
+        if not os.path.exists(PLOTLY_DIR):
+            try:
+                os.mkdir(PLOTLY_DIR)
+            except Exception:
+                # in case of race
+                if not os.path.isdir(PLOTLY_DIR):
+                    raise
+        with open(TEST_FILE, 'w') as f:
+            f.write('testing\n')
+        try:
+            os.remove(TEST_FILE)
+        except Exception:
+            pass
+        return True
+    except Exception: # Do not trap KeyboardInterrupt.
+        return False
+
+
+_file_permissions = None
+
+
+def ensure_writable_plotly_dir():
+    # Cache permissions status
+    global _file_permissions
+    if _file_permissions is None:
+        _file_permissions = _permissions()
+    return _file_permissions

_plotly_utils/optional_imports.py

@@ -0,0 +1,31 @@
+"""
+Stand-alone module to provide information about whether optional deps exist.
+
+"""
+from __future__ import absolute_import
+
+from importlib import import_module
+import logging
+
+logger = logging.getLogger(__name__)
+_not_importable = set()
+
+
+def get_module(name):
+    """
+    Return module or None. Absolute import is required.
+
+    :param (str) name: Dot-separated module path. E.g., 'scipy.stats'.
+    :raise: (ImportError) Only when exc_msg is defined.
+    :return: (module|None) If import succeeds, the module will be returned.
+
+    """
+    if name not in _not_importable:
+        try:
+            return import_module(name)
+        except ImportError:
+            _not_importable.add(name)
+        except Exception as e:
+            _not_importable.add(name)
+            msg = "Error importing optional module {}".format(name)
+            logger.exception(msg)