Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[druid] fix 'Unorderable types' when col has nulls #4771

Closed
wants to merge 2 commits into from
Closed

[druid] fix 'Unorderable types' when col has nulls #4771

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6aaab71
[druid] fix 'Unorderable types' when col has nuls
mistercrunch Apr 6, 2018
5e4888a
typo
mistercrunch Apr 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions superset/connectors/druid/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1277,13 +1277,30 @@ def run_query( # noqa / druid
client.query_builder.last_query.query_dict, indent=2)
return query_str

@staticmethod
def homogenize_types(df, groupby_cols):
"""Converting all GROUPBY columns to strings

When grouping by a numeric (say FLOAT) column, pydruid returns
strings in the dataframe. This creates issues downstream related
to having mixed types in the dataframe

Here we replace None with <NULL> and make the whole series a
str instead of an object.
"""
for col in groupby_cols:
df[col] = df[col].fillna('<NULL>').astype(str)
return df

def query(self, query_obj):
qry_start_dttm = datetime.now()
client = self.cluster.get_pydruid_client()
query_str = self.get_query_str(
client=client, query_obj=query_obj, phase=2)
df = client.export_pandas()

df = self.homogenize_types(df, query_obj.get('groupby', []))

if df is None or df.size == 0:
raise Exception(_('No data was returned.'))
df.columns = [
Expand Down
4 changes: 3 additions & 1 deletion tests/druid_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ def __reduce__(self):
'timestamp': '2012-01-01T00:00:00.000Z',
'event': {
'dim1': 'Canada',
'dim2': 'boy',
'metric1': 12345678,
},
},
Expand All @@ -69,6 +70,7 @@ def __reduce__(self):
'timestamp': '2012-01-01T00:00:00.000Z',
'event': {
'dim1': 'USA',
'dim2': 'girl',
'metric1': 12345678 / 2,
},
},
Expand Down Expand Up @@ -165,7 +167,7 @@ def test_client(self, PyDruid):
'row_limit': 5000,
'include_search': 'false',
'metrics': ['count'],
'groupby': ['dim1', 'dim2d'],
'groupby': ['dim1', 'dim2'],
'force': 'true',
}
# two groupby
Expand Down