1.7.4

- fix for #63, periods in column names - added json_loader CLI options - updated moving/locking of columns to be persisted to back-end as well as front-end - added the ability to show/hide columns - added column builder popup (#61)
man-group · Feb 19, 2020 · fef2d01 · fef2d01
1 parent 57b1e24
commit fef2d01
Show file tree

Hide file tree

Showing 71 changed files with 2,546 additions and 350 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -19,17 +19,6 @@ defaults: &defaults
         - v1-dep-master-
         # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
         - v1-dep-
-    - run:
-        name: Download & Install latest pandoc
-        command: |
-          # the 'latest' URL redirects to the name of the latest tag.
-          export PANDOC_VERSION=$(curl --connect-timeout 5  --max-time 10  --retry 5  --retry-delay 0  --retry-max-time 40 -L -I "$PANDOC_RELEASES_URL/latest" | sed -ne 's#Location:.*tag/\(.*\)$#\1#p' | tr -d "\n\r")
-          echo $PANDOC_VERSION
-          export PANDOC_FILE=pandoc-$PANDOC_VERSION-1-amd64.deb
-          echo $PANDOC_FILE
-          wget $PANDOC_RELEASES_URL/download/$PANDOC_VERSION/$PANDOC_FILE
-          sudo dpkg -i $PANDOC_FILE
-          rm $PANDOC_FILE
     - restore_cache:
         name: Restore Yarn Package Cache
         keys:
@@ -101,8 +90,28 @@ defaults: &defaults
          python setup.py test
          bash <(curl -s https://codecov.io/bash) -c -F python
          cp -r ./htmlcov /tmp/circleci-test-results
-         python setup.py build_sphinx
-         cp -r ./build /tmp/circleci-artifacts
+    - run:
+        name: Download & Install latest pandoc
+        command: |
+          if [ "${CIRCLE_BRANCH}" == "master" ]; then
+            # the 'latest' URL redirects to the name of the latest tag.
+            export PANDOC_VERSION=$(curl --connect-timeout 5  --max-time 10  --retry 5  --retry-delay 0  --retry-max-time 40 -L -I "$PANDOC_RELEASES_URL/latest" | sed -ne 's#Location:.*tag/\(.*\)$#\1#p' | tr -d "\n\r")
+            echo $PANDOC_VERSION
+            export PANDOC_FILE=pandoc-$PANDOC_VERSION-1-amd64.deb
+            echo $PANDOC_FILE
+            wget $PANDOC_RELEASES_URL/download/$PANDOC_VERSION/$PANDOC_FILE
+            sudo dpkg -i $PANDOC_FILE
+            rm $PANDOC_FILE
+          fi
+    - run:
+        name: Build Sphinx Documentation
+        command: |
+          if [ "${CIRCLE_BRANCH}" == "master" ]; then
+            set -e
+            . ci/bin/activate
+            python setup.py build_sphinx
+            cp -r ./build /tmp/circleci-artifacts
+          fi
     # Build egg
     - run:
         name: Build egg

diff --git a/README.md b/README.md
@@ -30,7 +30,7 @@ D-Tale was the product of a SAS to Python conversion.  What was originally a per
   - [Main Menu Functions](#main-menu-functions)
     - [Describe](#describe), [Filter](#filter), [Charts](#charts), [Correlations](#correlations), [Heat Map](#heat-map), [Instances](#instances), [About](#about), [Resize](#resize), [Shutdown](#shutdown)
   - [Column Menu Functions](#column-menu-functions)
-    - [Move To Front](#move-to-front), [Lock](#lock), [Unlock](#unlock), [Sorting](#sorting), [Formats](#formats), [Histogram](#histogram)
+    - [Moving Columns](#moving-columns), [Hiding Columns](#hiding-columns), [Building Columns](#building-columns), [Lock](#lock), [Unlock](#unlock), [Sorting](#sorting), [Formats](#formats), [Histogram](#histogram)
   - [Menu Functions within a Jupyter Notebook](#menu-functions-within-a-jupyter-notebook)
 - [For Developers](#for-developers)
   - [Cloning](#cloning)
@@ -232,6 +232,8 @@ Apply a simple pandas `query` to your data (link to pandas documentation include
 |--------|:------:|
 |![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Filter_apply.png)|![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Post_filter.png)|
 
+FYI: For python 3 users, there is now support for filtering on column names with special characters in them (EX: 'a.b') :metal:
+
 #### Charts
 Build custom charts based off your data(powered by [plotly/dash](https://github.com/plotly/dash)).
 
@@ -361,8 +363,27 @@ Pretty self-explanatory, kills your D-Tale session (there is also an auto-kill p
 
 ![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Col_menu.png)
 
-#### Move To Front
-Moves your column to the front of the "unlocked" columns
+#### Moving Columns
+
+[![](http://img.youtube.com/vi/We4TH477rRs/0.jpg)](http://www.youtube.com/watch?v=We4TH477rRs "Moving Columns in D-Tale")
+
+All column movements are saved on the server so refreshing your browser won't lose them :ok_hand:
+
+#### Hiding Columns
+
+[![](http://img.youtube.com/vi/ryZT2Lk_YaA/0.jpg)](http://www.youtube.com/watch?v=ryZT2Lk_YaA "Hide/Unhide Columns in D-Tale")
+
+All column movements are saved on the server so refreshing your browser won't lose them :ok_hand:
+
+#### Building Columns
+
+[![](http://img.youtube.com/vi/G6wNS9-lG04/0.jpg)](http://www.youtube.com/watch?v=G6wNS9-lG04 "Build Columns in D-Tale")
+
+This video shows you how to build the following:
+ - Numeric: adding/subtracting two columns or columns with static values
+ - Bins: bucketing values using pandas cut & qcut as well as assigning custom labels
+ - Dates: retrieveing date properties (hour, weekday, month...) as well as conversions (month end)
+
 
 #### Lock
 Adds your column to "locked" columns
@@ -392,9 +413,11 @@ Applies/removes sorting (Ascending/Descending/Clear) to the column selected
 #### Formats
 Apply simple formats to numeric values in your grid
 
-|Editing|Result|
+|Type|Editing|Result|
 |--------|:------:|
-|![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Formatting_apply.png)|![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Post_formatting.png)|
+|Numeric|![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Formatting_apply.png)|![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Post_formatting.png)|
+|Date|![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Formatting_date_apply.png)|![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Post_date_formatting.png)|
+|String|![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Formatting_string_apply.png)|![](https://raw.githubusercontent.com/man-group/dtale/master/docs/images/Post_string_formatting.png)|
 
 Here's a grid of all the formats available with -123456.789 as input:
 

diff --git a/docs/images/Col_menu.png b/docs/images/Col_menu.png
diff --git a/docs/images/Describe.png b/docs/images/Describe.png
diff --git a/docs/images/Describe_date.png b/docs/images/Describe_date.png
diff --git a/docs/images/Describe_float.png b/docs/images/Describe_float.png
diff --git a/docs/images/Describe_int.png b/docs/images/Describe_int.png
diff --git a/docs/images/Describe_string.png b/docs/images/Describe_string.png
diff --git a/docs/images/Formatting_date_apply.png b/docs/images/Formatting_date_apply.png
diff --git a/docs/images/Formatting_string_apply.png b/docs/images/Formatting_string_apply.png
diff --git a/docs/images/Heatmap_toggle.png b/docs/images/Heatmap_toggle.png
diff --git a/docs/images/Hiding_columns.gif b/docs/images/Hiding_columns.gif
diff --git a/docs/images/Info_menu_small.png b/docs/images/Info_menu_small.png
diff --git a/docs/images/Moving_columns.gif b/docs/images/Moving_columns.gif
diff --git a/docs/images/Post_date_formatting.png b/docs/images/Post_date_formatting.png
diff --git a/docs/images/Post_string_formatting.png b/docs/images/Post_string_formatting.png
diff --git a/dtale/cli/loaders/__init__.py b/dtale/cli/loaders/__init__.py
@@ -4,7 +4,7 @@
 
 import click
 
-from dtale.cli.loaders import arctic_loader, csv_loader
+from dtale.cli.loaders import arctic_loader, csv_loader, json_loader
 
 logger = getLogger(__name__)
 
@@ -72,7 +72,8 @@ def custom_module_loader():
 
 LOADERS = {
     arctic_loader.LOADER_KEY: arctic_loader,
-    csv_loader.LOADER_KEY: csv_loader
+    csv_loader.LOADER_KEY: csv_loader,
+    json_loader.LOADER_KEY: json_loader,
 }
 
 

diff --git a/dtale/cli/loaders/json_loader.py b/dtale/cli/loaders/json_loader.py
@@ -0,0 +1,34 @@
+import pandas as pd
+
+from dtale.cli.clickutils import get_loader_options
+
+'''
+  IMPORTANT!!! These global variables are required for building any customized CLI loader.
+  When build_loaders runs startup it will search for any modules containing the global variable LOADER_KEY.
+'''
+LOADER_KEY = 'json'
+LOADER_PROPS = [
+    dict(name='path', help='path to JSON file or URL to JSON endpoint'),
+    dict(name='convert_dates', help='comma-separated string of column names which should be parsed as dates')
+]
+
+
+# IMPORTANT!!! This function is required for building any customized CLI loader.
+def find_loader(kwargs):
+    """
+    CSV implementation of data loader which will return a function if any of the
+    `click` options based on LOADER_KEY & LOADER_PROPS have been used, otherwise return None
+
+    :param kwargs: Optional keyword arguments to be passed from `click`
+    :return: data loader function for CSV implementation
+    """
+    json_opts = get_loader_options(LOADER_KEY, kwargs)
+    if len([f for f in json_opts.values() if f]):
+        def _json_loader():
+            json_arg_parsers = {  # TODO: add additional arg parsers
+                'parse_dates': lambda v: v.split(',') if v else None
+            }
+            kwargs = {k: json_arg_parsers.get(k, lambda v: v)(v) for k, v in json_opts.items() if k != 'path'}
+            return pd.read_json(json_opts['path'], **kwargs)
+        return _json_loader
+    return None
diff --git a/dtale/dash_application/charts.py b/dtale/dash_application/charts.py
@@ -20,7 +20,7 @@
                                            update_label_for_freq)
 from dtale.utils import (classify_type, dict_merge, divide_chunks,
                          flatten_lists, get_dtypes, make_list,
-                         make_timeout_request)
+                         make_timeout_request, run_query)
 from dtale.views import DATA
 from dtale.views import build_chart as build_chart_data
 
@@ -720,7 +720,7 @@ def build_figure_data(data_id, chart_type=None, query=None, x=None, y=None, z=No
                                   rolling_comp=rolling_comp)):
             return None
 
-        data = DATA[data_id] if (query or '') == '' else DATA[data_id].query(query)
+        data = run_query(DATA[data_id], query)
         chart_kwargs = dict(group_col=group, agg=agg, allow_duplicates=chart_type == 'scatter', rolling_win=window,
                             rolling_comp=rolling_comp)
         if chart_type in ZAXIS_CHARTS:

diff --git a/dtale/dash_application/views.py b/dtale/dash_application/views.py
@@ -16,8 +16,8 @@
                                            show_chart_per_group,
                                            show_input_handler,
                                            show_yaxis_ranges)
-from dtale.utils import dict_merge, make_list
-from dtale.views import DATA, _test_filter
+from dtale.utils import dict_merge, make_list, run_query
+from dtale.views import DATA
 
 logger = getLogger(__name__)
 
@@ -139,8 +139,7 @@ def query_input(query, pathname, curr_query):
         :rtype: tuple of (str, str, str)
         """
         try:
-            if query is not None and query != '':
-                _test_filter(DATA[get_data_id(pathname)], query)
+            run_query(DATA[get_data_id(pathname)], query)
             return query, {'line-height': 'inherit'}, ''
         except BaseException as ex:
             return curr_query, {'line-height': 'inherit', 'background-color': 'pink'}, str(ex)

diff --git a/dtale/static/css/main.css b/dtale/static/css/main.css
@@ -4551,10 +4551,20 @@ button.close {
   }
 }
 
+div.build-modal > div.modal-lg {
+  min-width: 720px;
+}
+
 @media (min-width: 992px) {
   .modal-lg {
     max-width: 800px;
   }
+  div.build-modal > div.modal-lg {
+    max-width: 720px;
+  }
+  div.histogram-modal > div.modal-lg {
+    max-width: 700px;
+  }
 }
 
 .tooltip {
@@ -10429,3 +10439,34 @@ table tbody tr.highlight-row {
 [data-tip]:hover:after {
 	display:block;
 }
+
+div#popup-content > div.modal-body > div.row {
+    margin-left: 0;
+    margin-right: 0;
+}
+
+div.container-fluid.build > div#popup-content > div.modal-body {
+  height: 260px;
+}
+
+div.container-fluid.describe > div#popup-content > div.modal-body {
+  height: 450px;
+}
+
+@media (min-height: 330px) {
+  div.container-fluid.build > div#popup-content > div.modal-footer {
+    position: absolute;
+    bottom: 0;
+    width: 100%;
+  }
+}
+
+@media (min-height: 510px) {
+  div.container-fluid.describe > div#popup-content > div.modal-footer {
+    position: absolute;
+    bottom: 0;
+    width: 100%;
+  }
+}
+
+
diff --git a/dtale/utils.py b/dtale/utils.py
@@ -15,6 +15,7 @@
 import numpy as np
 import pandas as pd
 from past.utils import old_div
+from six import PY3
 
 logger = getLogger(__name__)
 
@@ -538,21 +539,25 @@ def filter_df_for_grid(df, params):
                 df = df[stringified_col.astype(str) == filter_val[1:]]
             else:
                 df = df[stringified_col.astype(str).str.lower().str.contains(filter_val.lower(), na=False)]
-    if params.get('query'):
-        df = df.query(params['query'])
+    df = run_query(df, params.get('query'))
     return df
 
 
+def find_dtype(s):
+    if s.dtype.name == 'object':
+        return pd.api.types.infer_dtype(s, skipna=True)
+    else:
+        return s.dtype.name
+
+
 def get_dtypes(df):
     """
     Build dictionary of column/dtype name pairs from :class:`pandas:pandas.DataFrame`
     """
     def _load():
-        for col, dtype in df.dtypes.to_dict().items():
-            if dtype.name == 'object':
-                yield col, pd.api.types.infer_dtype(df[col], skipna=True)
-            else:
-                yield col, dtype.name
+        for col in df.columns:
+            yield col, find_dtype(df[col])
+
     return dict(list(_load()))
 
 
@@ -749,3 +754,33 @@ def make_timeout_request(target, args=None, kwargs=None, timeout=60):
             'Request took longer than {} seconds. Please try adding additional filtering...'.format(timeout)
         )
     return results
+
+
+def run_query(df, query):
+    if (query or '') == '':
+        return df
+
+    # https://stackoverflow.com/a/40083013/12616360 only supporting filter cleanup for python 3+
+    if PY3:
+        invalid_column_names = [x for x in df.columns.values if not x.isidentifier()]
+
+        # Make replacements in the query and keep track
+        # NOTE: This method fails if the frame has columns called REPL_0 etc.
+        replacements = dict()
+        final_query = str(query)
+        for cn in invalid_column_names:
+            r = 'REPL_{}'.format(str(invalid_column_names.index(cn)))
+            final_query = final_query.replace(cn, r)
+            replacements[cn] = r
+
+        inv_replacements = {replacements[k]: k for k in replacements.keys()}
+        df = df.rename(columns=replacements)  # Rename the columns
+
+        df = df.query(final_query)  # Carry out query
+        df = df.rename(columns=inv_replacements)
+    else:
+        df = df.query(query)
+
+    if not len(df):
+        raise Exception('query "{}" found no data, please alter'.format(query))
+    return df