From ee4f6a56ebbfe45c5215f86e0a6c29b70f864b84 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Fri, 29 Jan 2021 11:09:18 -0800 Subject: [PATCH 1/8] WIP --- .../src/SqlLab/actions/sqlLab.js | 22 +++++++++ .../SqlLab/components/AceEditorWrapper.tsx | 5 ++ .../src/SqlLab/components/SqlEditor.jsx | 4 +- .../SqlLab/components/SqlEditorLeftBar.jsx | 4 ++ .../src/SqlLab/reducers/getInitialState.js | 2 + .../src/SqlLab/reducers/sqlLab.js | 5 ++ superset/databases/api.py | 47 +++++++++++++++++-- superset/databases/schemas.py | 6 ++- superset/db_engine_specs/base.py | 2 +- 9 files changed, 89 insertions(+), 8 deletions(-) diff --git a/superset-frontend/src/SqlLab/actions/sqlLab.js b/superset-frontend/src/SqlLab/actions/sqlLab.js index 647cfad3d8b1c..5a26294d33cca 100644 --- a/superset-frontend/src/SqlLab/actions/sqlLab.js +++ b/superset-frontend/src/SqlLab/actions/sqlLab.js @@ -57,6 +57,8 @@ export const QUERY_EDITOR_SET_QUERY_LIMIT = 'QUERY_EDITOR_SET_QUERY_LIMIT'; export const QUERY_EDITOR_SET_TEMPLATE_PARAMS = 'QUERY_EDITOR_SET_TEMPLATE_PARAMS'; export const QUERY_EDITOR_SET_SELECTED_TEXT = 'QUERY_EDITOR_SET_SELECTED_TEXT'; +export const QUERY_EDITOR_SET_FUNCTION_NAMES = + 'QUERY_EDITOR_SET_FUNCTION_NAMES'; export const QUERY_EDITOR_PERSIST_HEIGHT = 'QUERY_EDITOR_PERSIST_HEIGHT'; export const MIGRATE_QUERY_EDITOR = 'MIGRATE_QUERY_EDITOR'; export const MIGRATE_TAB_HISTORY = 'MIGRATE_TAB_HISTORY'; @@ -1300,3 +1302,23 @@ export function createCtasDatasource(vizOptions) { }); }; } + +export function queryEditorSetFunctionNames(queryEditor, dbId) { + return function (dispatch) { + return SupersetClient.get({ + endpoint: encodeURI(`/api/v1/database/${dbId}/function_names/`), + }) + .then(({ json }) => + dispatch({ + type: QUERY_EDITOR_SET_FUNCTION_NAMES, + queryEditor, + functionNames: json.function_names, + }), + ) + .catch(() => + dispatch( + addDangerToast(t('An error occurred while fetching function names')), + ), + ); + }; +} diff --git a/superset-frontend/src/SqlLab/components/AceEditorWrapper.tsx b/superset-frontend/src/SqlLab/components/AceEditorWrapper.tsx index 95c6eb8cfcfb1..3c33f4c362419 100644 --- a/superset-frontend/src/SqlLab/components/AceEditorWrapper.tsx +++ b/superset-frontend/src/SqlLab/components/AceEditorWrapper.tsx @@ -41,6 +41,7 @@ type HotKey = { interface Props { actions: { queryEditorSetSelectedText: (edit: any, text: null | string) => void; + queryEditorSetFunctionNames: (queryEditor: object, dbId: number) => void; addTable: (queryEditor: any, value: any, schema: any) => void; }; autocomplete: boolean; @@ -85,6 +86,10 @@ class AceEditorWrapper extends React.PureComponent { componentDidMount() { // Making sure no text is selected from previous mount this.props.actions.queryEditorSetSelectedText(this.props.queryEditor, null); + this.props.actions.queryEditorSetFunctionNames( + this.props.queryEditor, + this.props.queryEditor.dbId, + ); this.setAutoCompleter(this.props); } diff --git a/superset-frontend/src/SqlLab/components/SqlEditor.jsx b/superset-frontend/src/SqlLab/components/SqlEditor.jsx index ce68a2b03ba35..d132154ce9242 100644 --- a/superset-frontend/src/SqlLab/components/SqlEditor.jsx +++ b/superset-frontend/src/SqlLab/components/SqlEditor.jsx @@ -471,9 +471,7 @@ class SqlEditor extends React.PureComponent { sql={this.props.queryEditor.sql} schemas={this.props.queryEditor.schemaOptions} tables={this.props.queryEditor.tableOptions} - functionNames={ - this.props.database ? this.props.database.function_names : [] - } + functionNames={this.props.queryEditor.functionNames} extendedTables={this.props.tables} height={`${aceEditorHeight}px`} hotkeys={hotkeys} diff --git a/superset-frontend/src/SqlLab/components/SqlEditorLeftBar.jsx b/superset-frontend/src/SqlLab/components/SqlEditorLeftBar.jsx index 550f623412a0e..7136ab0370e48 100644 --- a/superset-frontend/src/SqlLab/components/SqlEditorLeftBar.jsx +++ b/superset-frontend/src/SqlLab/components/SqlEditorLeftBar.jsx @@ -80,6 +80,10 @@ export default class SqlEditorLeftBar extends React.PureComponent { onDbChange(db) { this.props.actions.queryEditorSetDb(this.props.queryEditor, db.id); + this.props.actions.queryEditorSetFunctionNames( + this.props.queryEditor, + db.id, + ); } onTableChange(tableName, schemaName) { diff --git a/superset-frontend/src/SqlLab/reducers/getInitialState.js b/superset-frontend/src/SqlLab/reducers/getInitialState.js index 60cd420da9229..df55748eabc12 100644 --- a/superset-frontend/src/SqlLab/reducers/getInitialState.js +++ b/superset-frontend/src/SqlLab/reducers/getInitialState.js @@ -48,6 +48,7 @@ export default function getInitialState({ autorun: false, templateParams: null, dbId: defaultDbId, + functionNames: [], queryLimit: common.conf.DEFAULT_SQLLAB_LIMIT, validationResult: { id: null, @@ -80,6 +81,7 @@ export default function getInitialState({ autorun: activeTab.autorun, templateParams: activeTab.template_params, dbId: activeTab.database_id, + functionNames: [], schema: activeTab.schema, queryLimit: activeTab.query_limit, validationResult: { diff --git a/superset-frontend/src/SqlLab/reducers/sqlLab.js b/superset-frontend/src/SqlLab/reducers/sqlLab.js index 8a94bcb3ecc2b..a5930d2b3484d 100644 --- a/superset-frontend/src/SqlLab/reducers/sqlLab.js +++ b/superset-frontend/src/SqlLab/reducers/sqlLab.js @@ -434,6 +434,11 @@ export default function sqlLabReducer(state = {}, action) { dbId: action.dbId, }); }, + [actions.QUERY_EDITOR_SET_FUNCTION_NAMES]() { + return alterInArr(state, 'queryEditors', action.queryEditor, { + functionNames: action.functionNames, + }); + }, [actions.QUERY_EDITOR_SET_SCHEMA]() { return alterInArr(state, 'queryEditors', action.queryEditor, { schema: action.schema, diff --git a/superset/databases/api.py b/superset/databases/api.py index e8273eb1e76fa..1350d96c5f960 100644 --- a/superset/databases/api.py +++ b/superset/databases/api.py @@ -53,6 +53,7 @@ from superset.databases.filters import DatabaseFilter from superset.databases.schemas import ( database_schemas_query_schema, + DatabaseFunctionNamesResponse, DatabasePostSchema, DatabasePutSchema, DatabaseRelatedObjectsResponse, @@ -83,6 +84,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi): "schemas", "test_connection", "related_objects", + "function_names", } resource_name = "database" class_permission_name = "Database" @@ -126,7 +128,6 @@ class DatabaseRestApi(BaseSupersetModelRestApi): "explore_database_id", "expose_in_sqllab", "force_ctas_schema", - "function_names", "id", ] add_columns = [ @@ -642,8 +643,8 @@ def related_objects(self, pk: int) -> Response: 500: $ref: '#/components/responses/500' """ - dataset = DatabaseDAO.find_by_id(pk) - if not dataset: + database = DatabaseDAO.find_by_id(pk) + if not database: return self.response_404() data = DatabaseDAO.get_related_objects(pk) charts = [ @@ -799,3 +800,43 @@ def import_(self) -> Response: except DatabaseImportError as exc: logger.exception("Import database failed") return self.response_500(message=str(exc)) + + @expose("//function_names/", methods=["GET"]) + @protect() + @safe + @statsd_metrics + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}" + f".function_names", + log_to_statsd=False, + ) + def function_names(self, pk: int) -> Response: + """Get function names supported by a database + --- + get: + description: + Get function names supported by a database + parameters: + - in: path + name: pk + schema: + type: integer + responses: + 200: + 200: + description: Query result + content: + application/json: + schema: + $ref: "#/components/schemas/DatabaseFunctionNamesResponse" + 401: + $ref: '#/components/responses/401' + 404: + $ref: '#/components/responses/404' + 500: + $ref: '#/components/responses/500' + """ + database = DatabaseDAO.find_by_id(pk) + if not database: + return self.response_404() + return self.response(200, function_names=database.function_names,) diff --git a/superset/databases/schemas.py b/superset/databases/schemas.py index 2c705f35b1fa5..0fbcf1dcb6ab0 100644 --- a/superset/databases/schemas.py +++ b/superset/databases/schemas.py @@ -413,11 +413,15 @@ class DatabaseRelatedObjectsResponse(Schema): dashboards = fields.Nested(DatabaseRelatedDashboards) +class DatabaseFunctionNamesResponse(Schema): + function_names = fields.List(fields.String()) + + class ImportV1DatabaseExtraSchema(Schema): metadata_params = fields.Dict(keys=fields.Str(), values=fields.Raw()) engine_params = fields.Dict(keys=fields.Str(), values=fields.Raw()) metadata_cache_timeout = fields.Dict(keys=fields.Str(), values=fields.Integer()) - schemas_allowed_for_csv_upload = fields.List(fields.String) + schemas_allowed_for_csv_upload = fields.List(fields.String()) cost_estimate_enabled = fields.Boolean() diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index cae31ba80664c..edc2773d7df79 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -1019,7 +1019,7 @@ def get_function_names(cls, database: "Database") -> List[str]: :param database: The database to get functions for :return: A list of function names useable in the database """ - return [] + return ["foo", "bar", "baz"] @staticmethod def pyodbc_rows_to_tuples(data: List[Any]) -> List[Tuple[Any, ...]]: From 19258cf58c4051541a3db783b5507b54709f7278 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Mon, 1 Feb 2021 10:47:57 -0800 Subject: [PATCH 2/8] Add unit test for API --- superset/db_engine_specs/base.py | 2 +- tests/databases/api_tests.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index edc2773d7df79..cae31ba80664c 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -1019,7 +1019,7 @@ def get_function_names(cls, database: "Database") -> List[str]: :param database: The database to get functions for :return: A list of function names useable in the database """ - return ["foo", "bar", "baz"] + return [] @staticmethod def pyodbc_rows_to_tuples(data: List[Any]) -> List[Tuple[Any, ...]]: diff --git a/tests/databases/api_tests.py b/tests/databases/api_tests.py index 8d603ccd9b385..17756d1f45faf 100644 --- a/tests/databases/api_tests.py +++ b/tests/databases/api_tests.py @@ -1125,3 +1125,16 @@ def test_import_database_masked_password_provided(self): db.session.delete(database) db.session.commit() + + @mock.patch("superset.db_engine_specs.base.BaseEngineSpec.get_function_names",) + def test_function_names(self, mock_get_function_names): + mock_get_function_names.return_value = ["AVG", "MAX", "SUM"] + + self.login(username="admin") + uri = "api/v1/database/1/function_names/" + + rv = self.client.get(uri) + response = json.loads(rv.data.decode("utf-8")) + + assert rv.status_code == 200 + assert response == {"function_names": ["AVG", "MAX", "SUM"]} From d2732c49f7af2cd0deef05f7a33d448a46a4ace2 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Mon, 1 Feb 2021 12:59:23 -0800 Subject: [PATCH 3/8] Add spec --- superset/databases/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/superset/databases/api.py b/superset/databases/api.py index 1350d96c5f960..ba665c1cb412a 100644 --- a/superset/databases/api.py +++ b/superset/databases/api.py @@ -171,6 +171,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi): } openapi_spec_tag = "Database" openapi_spec_component_schemas = ( + DatabaseFunctionNamesResponse, DatabaseRelatedObjectsResponse, DatabaseTestConnectionSchema, TableMetadataResponseSchema, From 79423c5da01da4da268a17bdf1b3da21be5cd7b5 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Mon, 1 Feb 2021 13:16:39 -0800 Subject: [PATCH 4/8] Fix unit test --- tests/databases/api_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/databases/api_tests.py b/tests/databases/api_tests.py index 17756d1f45faf..2901f783b8fa0 100644 --- a/tests/databases/api_tests.py +++ b/tests/databases/api_tests.py @@ -137,7 +137,6 @@ def test_get_items(self): "explore_database_id", "expose_in_sqllab", "force_ctas_schema", - "function_names", "id", ] self.assertGreater(response["count"], 0) From 4317a8fd677253599f12ee672716d5d7d395136b Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Mon, 1 Feb 2021 13:33:22 -0800 Subject: [PATCH 5/8] Fix unit test --- tests/databases/api_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/databases/api_tests.py b/tests/databases/api_tests.py index 2901f783b8fa0..7b076d152149c 100644 --- a/tests/databases/api_tests.py +++ b/tests/databases/api_tests.py @@ -588,7 +588,8 @@ def test_info_security_database(self): assert rv.status_code == 200 assert "can_read" in data["permissions"] assert "can_write" in data["permissions"] - assert len(data["permissions"]) == 2 + assert "can_function_names" in data["permissions"] + assert len(data["permissions"]) == 3 def test_get_invalid_database_table_metadata(self): """ From 150b55ef773af807424282d3822ed160f5beb785 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Tue, 2 Feb 2021 06:33:52 -0800 Subject: [PATCH 6/8] Fix test --- .../spec/javascripts/sqllab/SqlEditor_spec.jsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/superset-frontend/spec/javascripts/sqllab/SqlEditor_spec.jsx b/superset-frontend/spec/javascripts/sqllab/SqlEditor_spec.jsx index eca063c7f4db3..2ba62687c817e 100644 --- a/superset-frontend/spec/javascripts/sqllab/SqlEditor_spec.jsx +++ b/superset-frontend/spec/javascripts/sqllab/SqlEditor_spec.jsx @@ -33,7 +33,10 @@ import ConnectedSouthPane from 'src/SqlLab/components/SouthPane'; import SqlEditor from 'src/SqlLab/components/SqlEditor'; import SqlEditorLeftBar from 'src/SqlLab/components/SqlEditorLeftBar'; import { Dropdown } from 'src/common/components'; -import { queryEditorSetSelectedText } from 'src/SqlLab/actions/sqlLab'; +import { + queryEditorSetFunctionNames, + queryEditorSetSelectedText, +} from 'src/SqlLab/actions/sqlLab'; import { initialState, queries, table } from './fixtures'; @@ -45,7 +48,7 @@ const store = mockStore(initialState); describe('SqlEditor', () => { const mockedProps = { - actions: { queryEditorSetSelectedText }, + actions: { queryEditorSetFunctionNames, queryEditorSetSelectedText }, database: {}, queryEditorId: initialState.sqlLab.queryEditors[0].id, latestQuery: queries[0], From 0c841a19c2476874b3065c4378a6121d8a70283d Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Tue, 2 Feb 2021 07:42:56 -0800 Subject: [PATCH 7/8] Fix test --- tests/databases/api_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/databases/api_tests.py b/tests/databases/api_tests.py index 7b076d152149c..a72c7eb12a9be 100644 --- a/tests/databases/api_tests.py +++ b/tests/databases/api_tests.py @@ -1128,6 +1128,10 @@ def test_import_database_masked_password_provided(self): @mock.patch("superset.db_engine_specs.base.BaseEngineSpec.get_function_names",) def test_function_names(self, mock_get_function_names): + example_db = get_example_database() + if example_db.backend in {"hive", "presto"}: + return + mock_get_function_names.return_value = ["AVG", "MAX", "SUM"] self.login(username="admin") From fdbbbd9c680ebf48306c7b55866b16414c6a9503 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Tue, 2 Feb 2021 13:34:13 -0800 Subject: [PATCH 8/8] Add period to error message --- superset-frontend/src/SqlLab/actions/sqlLab.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset-frontend/src/SqlLab/actions/sqlLab.js b/superset-frontend/src/SqlLab/actions/sqlLab.js index 5a26294d33cca..090d69ca3babc 100644 --- a/superset-frontend/src/SqlLab/actions/sqlLab.js +++ b/superset-frontend/src/SqlLab/actions/sqlLab.js @@ -1317,7 +1317,7 @@ export function queryEditorSetFunctionNames(queryEditor, dbId) { ) .catch(() => dispatch( - addDangerToast(t('An error occurred while fetching function names')), + addDangerToast(t('An error occurred while fetching function names.')), ), ); };