From ce668d46cc5d429a249fdd9e091650457da20361 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Wed, 8 May 2024 17:19:36 -0400 Subject: [PATCH] feat(SIP-95): catalogs in SQL Lab and datasets (#28376) --- .../superset-ui-core/src/query/types/Query.ts | 1 + .../src/ui-overrides/types.ts | 1 + .../src/SqlLab/actions/sqlLab.js | 30 ++++- .../src/SqlLab/actions/sqlLab.test.js | 31 ++++- .../components/AceEditorWrapper/index.tsx | 3 + .../AceEditorWrapper/useKeywords.test.ts | 15 ++- .../AceEditorWrapper/useKeywords.ts | 9 +- .../SaveDatasetModal.test.tsx | 32 +++++ .../components/SaveDatasetModal/index.tsx | 2 + .../components/SaveQuery/SaveQuery.test.tsx | 1 + .../src/SqlLab/components/SaveQuery/index.tsx | 4 +- .../SqlEditorLeftBar.test.tsx | 61 ++++++++- .../components/SqlEditorLeftBar/index.tsx | 34 ++++- .../components/TabbedSqlEditors/index.tsx | 2 + .../SqlLab/components/TableElement/index.tsx | 4 +- superset-frontend/src/SqlLab/fixtures.ts | 15 +++ .../src/SqlLab/reducers/getInitialState.ts | 2 + .../src/SqlLab/reducers/sqlLab.js | 14 ++ superset-frontend/src/SqlLab/types.ts | 2 + .../utils/reduxStateToLocalStorageHelper.ts | 38 +++--- .../DatabaseSelector.test.tsx | 8 ++ .../src/components/DatabaseSelector/index.tsx | 104 ++++++++++++-- .../Datasource/DatasourceEditor.jsx | 13 ++ .../TableSelector/TableSelector.test.tsx | 7 + .../src/components/TableSelector/index.tsx | 36 ++++- .../DndFilterSelect.tsx | 10 +- .../AdhocFilterControl/index.jsx | 10 +- superset-frontend/src/explore/types.ts | 1 + .../databases/DatabaseModal/ExtraOptions.tsx | 23 +++- .../DatabaseModal/SSHTunnelSwitch.test.tsx | 1 + .../UploadDataModel/UploadDataModal.test.tsx | 4 + .../src/features/databases/types.ts | 3 + .../AddDataset/DatasetPanel/index.tsx | 13 +- .../datasets/AddDataset/Footer/index.tsx | 1 + .../datasets/AddDataset/LeftPanel/index.tsx | 10 ++ .../features/datasets/AddDataset/types.tsx | 3 + .../src/hooks/apiResources/catalogs.ts | 127 ++++++++++++++++++ .../src/hooks/apiResources/index.ts | 1 + .../src/hooks/apiResources/queryApi.ts | 1 + .../hooks/apiResources/queryValidations.ts | 4 +- .../src/hooks/apiResources/schemas.test.ts | 10 +- .../src/hooks/apiResources/schemas.ts | 31 ++++- .../src/hooks/apiResources/sqlEditorTabs.ts | 2 + .../src/hooks/apiResources/sqlLab.ts | 2 + .../src/hooks/apiResources/tables.test.ts | 16 +++ .../src/hooks/apiResources/tables.ts | 35 +++-- .../src/pages/DatasetCreation/index.tsx | 9 ++ superset-frontend/src/types/Database.ts | 1 + .../src/utils/datasourceUtils.js | 1 + superset-frontend/src/utils/urlUtils.test.ts | 46 ++++++- superset-frontend/src/utils/urlUtils.ts | 13 ++ superset/cachekeys/api.py | 1 + superset/commands/dashboard/importers/v0.py | 1 + superset/commands/database/validate_sql.py | 2 +- superset/connectors/sqla/models.py | 10 +- superset/dashboards/schemas.py | 1 + superset/databases/api.py | 1 + superset/databases/schemas.py | 22 ++- superset/datasets/api.py | 10 +- superset/datasets/schemas.py | 7 + superset/db_engine_specs/base.py | 1 + superset/models/core.py | 5 + superset/models/sql_lab.py | 1 + superset/queries/saved_queries/api.py | 16 ++- superset/sqllab/schemas.py | 1 + superset/sqllab/sqllab_execution_context.py | 4 + superset/sqllab/utils.py | 1 + superset/views/database/mixins.py | 4 +- superset/views/datasource/schemas.py | 3 + superset/views/datasource/views.py | 1 + superset/views/sql_lab/views.py | 1 + 71 files changed, 841 insertions(+), 99 deletions(-) create mode 100644 superset-frontend/src/hooks/apiResources/catalogs.ts diff --git a/superset-frontend/packages/superset-ui-core/src/query/types/Query.ts b/superset-frontend/packages/superset-ui-core/src/query/types/Query.ts index 4d5ccaf6f09b1..b6b1fd3a638cc 100644 --- a/superset-frontend/packages/superset-ui-core/src/query/types/Query.ts +++ b/superset-frontend/packages/superset-ui-core/src/query/types/Query.ts @@ -316,6 +316,7 @@ export type Query = { link?: string; progress: number; resultsKey: string | null; + catalog?: string | null; schema?: string; sql: string; sqlEditorId: string; diff --git a/superset-frontend/packages/superset-ui-core/src/ui-overrides/types.ts b/superset-frontend/packages/superset-ui-core/src/ui-overrides/types.ts index f8f10982bd1f9..736a57da900a3 100644 --- a/superset-frontend/packages/superset-ui-core/src/ui-overrides/types.ts +++ b/superset-frontend/packages/superset-ui-core/src/ui-overrides/types.ts @@ -168,6 +168,7 @@ export interface SubMenuProps { export interface CustomAutoCompleteArgs { queryEditorId: string; dbId?: string | number; + catalog?: string | null; schema?: string; } diff --git a/superset-frontend/src/SqlLab/actions/sqlLab.js b/superset-frontend/src/SqlLab/actions/sqlLab.js index 6befa17ac18d4..a153c4eb450a1 100644 --- a/superset-frontend/src/SqlLab/actions/sqlLab.js +++ b/superset-frontend/src/SqlLab/actions/sqlLab.js @@ -55,6 +55,7 @@ export const REMOVE_QUERY = 'REMOVE_QUERY'; export const EXPAND_TABLE = 'EXPAND_TABLE'; export const COLLAPSE_TABLE = 'COLLAPSE_TABLE'; export const QUERY_EDITOR_SETDB = 'QUERY_EDITOR_SETDB'; +export const QUERY_EDITOR_SET_CATALOG = 'QUERY_EDITOR_SET_CATALOG'; export const QUERY_EDITOR_SET_SCHEMA = 'QUERY_EDITOR_SET_SCHEMA'; export const QUERY_EDITOR_SET_TITLE = 'QUERY_EDITOR_SET_TITLE'; export const QUERY_EDITOR_SET_AUTORUN = 'QUERY_EDITOR_SET_AUTORUN'; @@ -326,6 +327,7 @@ export function runQuery(query) { database_id: query.dbId, json: true, runAsync: query.runAsync, + catalog: query.catalog, schema: query.schema, sql: query.sql, sql_editor_id: query.sqlEditorId, @@ -381,6 +383,7 @@ export function runQueryFromSqlEditor( sql: qe.selectedText || qe.sql, sqlEditorId: qe.id, tab: qe.name, + catalog: qe.catalog, schema: qe.schema, tempTable, templateParams: qe.templateParams, @@ -556,7 +559,7 @@ export function addNewQueryEditor() { ); const dbIds = Object.values(databases).map(database => database.id); const firstDbId = dbIds.length > 0 ? Math.min(...dbIds) : undefined; - const { dbId, schema, queryLimit, autorun } = { + const { dbId, catalog, schema, queryLimit, autorun } = { ...queryEditors[0], ...activeQueryEditor, ...(unsavedQueryEditor.id === activeQueryEditor?.id && @@ -578,6 +581,7 @@ export function addNewQueryEditor() { return dispatch( addQueryEditor({ dbId: dbId || defaultDbId || firstDbId, + catalog: catalog ?? null, schema: schema ?? null, autorun: autorun ?? false, sql: `${warning}SELECT ...`, @@ -600,6 +604,7 @@ export function cloneQueryToNewTab(query, autorun) { const queryEditor = { name: t('Copy of %s', sourceQueryEditor.name), dbId: query.dbId ? query.dbId : null, + catalog: query.catalog ? query.catalog : null, schema: query.schema ? query.schema : null, autorun, sql: query.sql, @@ -656,6 +661,7 @@ export function setTables(tableSchemas) { return { dbId: tableSchema.database_id, queryEditorId: tableSchema.tab_state_id.toString(), + catalog: tableSchema.catalog, schema: tableSchema.schema, name: tableSchema.table, expanded: tableSchema.expanded, @@ -694,6 +700,7 @@ export function switchQueryEditor(queryEditor, displayLimit) { autorun: json.autorun, dbId: json.database_id, templateParams: json.template_params, + catalog: json.catalog, schema: json.schema, queryLimit: json.query_limit, remoteId: json.saved_query?.id, @@ -797,6 +804,14 @@ export function queryEditorSetDb(queryEditor, dbId) { return { type: QUERY_EDITOR_SETDB, queryEditor, dbId }; } +export function queryEditorSetCatalog(queryEditor, catalog) { + return { + type: QUERY_EDITOR_SET_CATALOG, + queryEditor: queryEditor || {}, + catalog, + }; +} + export function queryEditorSetSchema(queryEditor, schema) { return { type: QUERY_EDITOR_SET_SCHEMA, @@ -954,12 +969,13 @@ export function mergeTable(table, query, prepend) { return { type: MERGE_TABLE, table, query, prepend }; } -export function addTable(queryEditor, tableName, schemaName) { +export function addTable(queryEditor, tableName, catalogName, schemaName) { return function (dispatch, getState) { const query = getUpToDateQuery(getState(), queryEditor, queryEditor.id); const table = { dbId: query.dbId, queryEditorId: query.id, + catalog: catalogName, schema: schemaName, name: tableName, }; @@ -983,12 +999,14 @@ export function runTablePreviewQuery(newTable) { sqlLab: { databases }, } = getState(); const database = databases[newTable.dbId]; - const { dbId } = newTable; + const { dbId, catalog, schema } = newTable; if (database && !database.disable_data_preview) { const dataPreviewQuery = { id: shortid.generate(), dbId, + catalog, + schema, sql: newTable.selectStar, tableName: newTable.name, sqlEditorId: null, @@ -1003,6 +1021,7 @@ export function runTablePreviewQuery(newTable) { { id: newTable.id, dbId: newTable.dbId, + catalog: newTable.catalog, schema: newTable.schema, name: newTable.name, queryEditorId: newTable.queryEditorId, @@ -1180,6 +1199,7 @@ export function popStoredQuery(urlId) { addQueryEditor({ name: json.name ? json.name : t('Shared query'), dbId: json.dbId ? parseInt(json.dbId, 10) : null, + catalog: json.catalog ? json.catalog : null, schema: json.schema ? json.schema : null, autorun: json.autorun ? json.autorun : false, sql: json.sql ? json.sql : 'SELECT ...', @@ -1215,6 +1235,7 @@ export function popQuery(queryId) { const queryData = json.result; const queryEditorProps = { dbId: queryData.database.id, + catalog: queryData.catalog, schema: queryData.schema, sql: queryData.sql, name: t('Copy of %s', queryData.tab_name), @@ -1268,12 +1289,13 @@ export function createDatasourceFailed(err) { export function createDatasource(vizOptions) { return dispatch => { dispatch(createDatasourceStarted()); - const { dbId, schema, datasourceName, sql } = vizOptions; + const { dbId, catalog, schema, datasourceName, sql } = vizOptions; return SupersetClient.post({ endpoint: '/api/v1/dataset/', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ database: dbId, + catalog, schema, sql, table_name: datasourceName, diff --git a/superset-frontend/src/SqlLab/actions/sqlLab.test.js b/superset-frontend/src/SqlLab/actions/sqlLab.test.js index 871b3ff6f6b4f..d6b70bf6a0b40 100644 --- a/superset-frontend/src/SqlLab/actions/sqlLab.test.js +++ b/superset-frontend/src/SqlLab/actions/sqlLab.test.js @@ -419,6 +419,7 @@ describe('async actions', () => { queryEditor: { name: 'Copy of Dummy query editor', dbId: 1, + catalog: query.catalog, schema: query.schema, autorun: true, sql: 'SELECT * FROM something', @@ -481,6 +482,7 @@ describe('async actions', () => { sql: expect.stringContaining('SELECT ...'), name: `Untitled Query 7`, dbId: defaultQueryEditor.dbId, + catalog: defaultQueryEditor.catalog, schema: defaultQueryEditor.schema, autorun: false, queryLimit: @@ -607,6 +609,24 @@ describe('async actions', () => { }); }); + describe('queryEditorSetCatalog', () => { + it('updates the tab state in the backend', () => { + expect.assertions(1); + + const catalog = 'public'; + const store = mockStore({}); + const expectedActions = [ + { + type: actions.QUERY_EDITOR_SET_CATALOG, + queryEditor, + catalog, + }, + ]; + store.dispatch(actions.queryEditorSetCatalog(queryEditor, catalog)); + expect(store.getActions()).toEqual(expectedActions); + }); + }); + describe('queryEditorSetSchema', () => { it('updates the tab state in the backend', () => { expect.assertions(1); @@ -747,6 +767,7 @@ describe('async actions', () => { describe('addTable', () => { it('dispatches table state from unsaved change', () => { const tableName = 'table'; + const catalogName = null; const schemaName = 'schema'; const expectedDbId = 473892; const store = mockStore({ @@ -759,12 +780,18 @@ describe('async actions', () => { }, }, }); - const request = actions.addTable(query, tableName, schemaName); + const request = actions.addTable( + query, + tableName, + catalogName, + schemaName, + ); request(store.dispatch, store.getState); expect(store.getActions()[0]).toEqual( expect.objectContaining({ table: expect.objectContaining({ name: tableName, + catalog: catalogName, schema: schemaName, dbId: expectedDbId, }), @@ -811,6 +838,7 @@ describe('async actions', () => { }); const tableName = 'table'; + const catalogName = null; const schemaName = 'schema'; const store = mockStore({ ...initialState, @@ -829,6 +857,7 @@ describe('async actions', () => { const request = actions.runTablePreviewQuery({ dbId: 1, name: tableName, + catalog: catalogName, schema: schemaName, }); return request(store.dispatch, store.getState).then(() => { diff --git a/superset-frontend/src/SqlLab/components/AceEditorWrapper/index.tsx b/superset-frontend/src/SqlLab/components/AceEditorWrapper/index.tsx index 354d6b1c8fb81..219a1f9bbaad2 100644 --- a/superset-frontend/src/SqlLab/components/AceEditorWrapper/index.tsx +++ b/superset-frontend/src/SqlLab/components/AceEditorWrapper/index.tsx @@ -74,6 +74,7 @@ const AceEditorWrapper = ({ 'id', 'dbId', 'sql', + 'catalog', 'schema', 'templateParams', 'cursorPosition', @@ -161,6 +162,7 @@ const AceEditorWrapper = ({ const { data: annotations } = useAnnotations({ dbId: queryEditor.dbId, + catalog: queryEditor.catalog, schema: queryEditor.schema, sql: currentSql, templateParams: queryEditor.templateParams, @@ -170,6 +172,7 @@ const AceEditorWrapper = ({ { queryEditorId, dbId: queryEditor.dbId, + catalog: queryEditor.catalog, schema: queryEditor.schema, }, !autocomplete, diff --git a/superset-frontend/src/SqlLab/components/AceEditorWrapper/useKeywords.test.ts b/superset-frontend/src/SqlLab/components/AceEditorWrapper/useKeywords.test.ts index adee7d681815b..193e715fb220f 100644 --- a/superset-frontend/src/SqlLab/components/AceEditorWrapper/useKeywords.test.ts +++ b/superset-frontend/src/SqlLab/components/AceEditorWrapper/useKeywords.test.ts @@ -189,7 +189,12 @@ test('returns column keywords among selected tables', async () => { storeWithSqlLab.dispatch( tableApiUtil.upsertQueryData( 'tableMetadata', - { dbId: expectDbId, schema: expectSchema, table: expectTable }, + { + dbId: expectDbId, + catalog: null, + schema: expectSchema, + table: expectTable, + }, { name: expectTable, columns: [ @@ -205,7 +210,12 @@ test('returns column keywords among selected tables', async () => { storeWithSqlLab.dispatch( tableApiUtil.upsertQueryData( 'tableMetadata', - { dbId: expectDbId, schema: expectSchema, table: unexpectedTable }, + { + dbId: expectDbId, + catalog: null, + schema: expectSchema, + table: unexpectedTable, + }, { name: unexpectedTable, columns: [ @@ -227,6 +237,7 @@ test('returns column keywords among selected tables', async () => { useKeywords({ queryEditorId: expectQueryEditorId, dbId: expectDbId, + catalog: null, schema: expectSchema, }), { diff --git a/superset-frontend/src/SqlLab/components/AceEditorWrapper/useKeywords.ts b/superset-frontend/src/SqlLab/components/AceEditorWrapper/useKeywords.ts index 09a50352600de..2eac21ff390b0 100644 --- a/superset-frontend/src/SqlLab/components/AceEditorWrapper/useKeywords.ts +++ b/superset-frontend/src/SqlLab/components/AceEditorWrapper/useKeywords.ts @@ -42,6 +42,7 @@ import { SqlLabRootState } from 'src/SqlLab/types'; type Params = { queryEditorId: string | number; dbId?: string | number; + catalog?: string | null; schema?: string; }; @@ -58,7 +59,7 @@ const getHelperText = (value: string) => const extensionsRegistry = getExtensionsRegistry(); export function useKeywords( - { queryEditorId, dbId, schema }: Params, + { queryEditorId, dbId, catalog, schema }: Params, skip = false, ) { const useCustomKeywords = extensionsRegistry.get( @@ -68,6 +69,7 @@ export function useKeywords( const customKeywords = useCustomKeywords?.({ queryEditorId: String(queryEditorId), dbId, + catalog, schema, }); const dispatch = useDispatch(); @@ -78,6 +80,7 @@ export function useKeywords( const { data: schemaOptions } = useSchemasQueryState( { dbId, + catalog: catalog || undefined, forceRefresh: false, }, { skip: skipFetch || !dbId }, @@ -85,6 +88,7 @@ export function useKeywords( const { data: tableData } = useTablesQueryState( { dbId, + catalog, schema, forceRefresh: false, }, @@ -125,6 +129,7 @@ export function useKeywords( dbId && schema ? { dbId, + catalog, schema, table, } @@ -137,7 +142,7 @@ export function useKeywords( }); }); return [...columns]; - }, [dbId, schema, apiState, tablesForColumnMetadata]); + }, [dbId, catalog, schema, apiState, tablesForColumnMetadata]); const insertMatch = useEffectEvent((editor: Editor, data: any) => { if (data.meta === 'table') { diff --git a/superset-frontend/src/SqlLab/components/SaveDatasetModal/SaveDatasetModal.test.tsx b/superset-frontend/src/SqlLab/components/SaveDatasetModal/SaveDatasetModal.test.tsx index 8568bf20809e5..c3d1658f3a4ef 100644 --- a/superset-frontend/src/SqlLab/components/SaveDatasetModal/SaveDatasetModal.test.tsx +++ b/superset-frontend/src/SqlLab/components/SaveDatasetModal/SaveDatasetModal.test.tsx @@ -210,6 +210,38 @@ describe('SaveDatasetModal', () => { expect(createDatasource).toHaveBeenCalledWith({ datasourceName: 'my dataset', dbId: 1, + catalog: null, + schema: 'main', + sql: 'SELECT *', + templateParams: undefined, + }); + }); + + it('sends the catalog when creating the dataset', async () => { + const dummyDispatch = jest.fn().mockResolvedValue({}); + useDispatchMock.mockReturnValue(dummyDispatch); + useSelectorMock.mockReturnValue({ ...user }); + + render( + , + { useRedux: true }, + ); + + const inputFieldText = screen.getByDisplayValue(/unimportant/i); + fireEvent.change(inputFieldText, { target: { value: 'my dataset' } }); + + const saveConfirmationBtn = screen.getByRole('button', { + name: /save/i, + }); + userEvent.click(saveConfirmationBtn); + + expect(createDatasource).toHaveBeenCalledWith({ + datasourceName: 'my dataset', + dbId: 1, + catalog: 'public', schema: 'main', sql: 'SELECT *', templateParams: undefined, diff --git a/superset-frontend/src/SqlLab/components/SaveDatasetModal/index.tsx b/superset-frontend/src/SqlLab/components/SaveDatasetModal/index.tsx index 885265fe8d2df..011d4a7d21189 100644 --- a/superset-frontend/src/SqlLab/components/SaveDatasetModal/index.tsx +++ b/superset-frontend/src/SqlLab/components/SaveDatasetModal/index.tsx @@ -77,6 +77,7 @@ export interface ISaveableDatasource { dbId: number; sql: string; templateParams?: string | object | null; + catalog?: string | null; schema?: string | null; database?: Database; } @@ -292,6 +293,7 @@ export const SaveDatasetModal = ({ createDatasource({ sql: datasource.sql, dbId: datasource.dbId || datasource?.database?.id, + catalog: datasource?.catalog, schema: datasource?.schema, templateParams, datasourceName: datasetName, diff --git a/superset-frontend/src/SqlLab/components/SaveQuery/SaveQuery.test.tsx b/superset-frontend/src/SqlLab/components/SaveQuery/SaveQuery.test.tsx index 54b81df96013d..3d0413f62c72d 100644 --- a/superset-frontend/src/SqlLab/components/SaveQuery/SaveQuery.test.tsx +++ b/superset-frontend/src/SqlLab/components/SaveQuery/SaveQuery.test.tsx @@ -42,6 +42,7 @@ const mockState = { { id: mockedProps.queryEditorId, dbId: 1, + catalog: null, schema: 'main', sql: 'SELECT * FROM t', }, diff --git a/superset-frontend/src/SqlLab/components/SaveQuery/index.tsx b/superset-frontend/src/SqlLab/components/SaveQuery/index.tsx index a7ac8b1b2a896..2cb50b0c066cb 100644 --- a/superset-frontend/src/SqlLab/components/SaveQuery/index.tsx +++ b/superset-frontend/src/SqlLab/components/SaveQuery/index.tsx @@ -48,7 +48,7 @@ export type QueryPayload = { description?: string; id?: string; remoteId?: number; -} & Pick; +} & Pick; const Styles = styled.span` span[role='img'] { @@ -78,6 +78,7 @@ const SaveQuery = ({ 'dbId', 'latestQueryId', 'queryLimit', + 'catalog', 'schema', 'selectedText', 'sql', @@ -115,6 +116,7 @@ const SaveQuery = ({ description, dbId: query.dbId ?? 0, sql: query.sql, + catalog: query.catalog, schema: query.schema, templateParams: query.templateParams, remoteId: query?.remoteId || undefined, diff --git a/superset-frontend/src/SqlLab/components/SqlEditorLeftBar/SqlEditorLeftBar.test.tsx b/superset-frontend/src/SqlLab/components/SqlEditorLeftBar/SqlEditorLeftBar.test.tsx index b5003b16f7b47..27d6c44d016b3 100644 --- a/superset-frontend/src/SqlLab/components/SqlEditorLeftBar/SqlEditorLeftBar.test.tsx +++ b/superset-frontend/src/SqlLab/components/SqlEditorLeftBar/SqlEditorLeftBar.test.tsx @@ -44,6 +44,10 @@ const mockedProps = { beforeEach(() => { fetchMock.get('glob:*/api/v1/database/?*', { result: [] }); + fetchMock.get('glob:*/api/v1/database/*/catalogs/?*', { + count: 0, + result: [], + }); fetchMock.get('glob:*/api/v1/database/*/schemas/?*', { count: 2, result: ['main', 'new_schema'], @@ -103,11 +107,14 @@ test('renders a TableElement', async () => { }); test('table should be visible when expanded is true', async () => { - const { container, getByText, getByRole, queryAllByText } = - await renderAndWait(mockedProps, undefined, { + const { container, getByText, getByRole } = await renderAndWait( + mockedProps, + undefined, + { ...initialState, sqlLab: { ...initialState.sqlLab, tables: [table] }, - }); + }, + ); const dbSelect = getByRole('combobox', { name: 'Select database or type to search databases', @@ -115,14 +122,56 @@ test('table should be visible when expanded is true', async () => { const schemaSelect = getByRole('combobox', { name: 'Select schema or type to search schemas', }); - const dropdown = getByText(/Table/i); - const abUser = queryAllByText(/ab_user/i); + const dropdown = getByText(/Select table/i); + const abUser = getByText(/ab_user/i); + + expect(getByText(/Database/i)).toBeInTheDocument(); + expect(dbSelect).toBeInTheDocument(); + expect(schemaSelect).toBeInTheDocument(); + expect(dropdown).toBeInTheDocument(); + expect(abUser).toBeInTheDocument(); + expect( + container.querySelector('.ant-collapse-content-active'), + ).toBeInTheDocument(); + table.columns.forEach(({ name }) => { + expect(getByText(name)).toBeInTheDocument(); + }); +}); + +test('catalog selector should be visible when enabled in the database', async () => { + const { container, getByText, getByRole } = await renderAndWait( + { + ...mockedProps, + database: { + ...mockedProps.database, + allow_multi_catalog: true, + }, + }, + undefined, + { + ...initialState, + sqlLab: { ...initialState.sqlLab, tables: [table] }, + }, + ); + + const dbSelect = getByRole('combobox', { + name: 'Select database or type to search databases', + }); + const catalogSelect = getByRole('combobox', { + name: 'Select catalog or type to search catalogs', + }); + const schemaSelect = getByRole('combobox', { + name: 'Select schema or type to search schemas', + }); + const dropdown = getByText(/Select table/i); + const abUser = getByText(/ab_user/i); expect(getByText(/Database/i)).toBeInTheDocument(); expect(dbSelect).toBeInTheDocument(); + expect(catalogSelect).toBeInTheDocument(); expect(schemaSelect).toBeInTheDocument(); expect(dropdown).toBeInTheDocument(); - expect(abUser).toHaveLength(2); + expect(abUser).toBeInTheDocument(); expect( container.querySelector('.ant-collapse-content-active'), ).toBeInTheDocument(); diff --git a/superset-frontend/src/SqlLab/components/SqlEditorLeftBar/index.tsx b/superset-frontend/src/SqlLab/components/SqlEditorLeftBar/index.tsx index 15a17356268e3..1eee80f485a00 100644 --- a/superset-frontend/src/SqlLab/components/SqlEditorLeftBar/index.tsx +++ b/superset-frontend/src/SqlLab/components/SqlEditorLeftBar/index.tsx @@ -34,6 +34,7 @@ import { removeTables, collapseTable, expandTable, + queryEditorSetCatalog, queryEditorSetSchema, setDatabases, addDangerToast, @@ -115,13 +116,17 @@ const SqlEditorLeftBar = ({ shallowEqual, ); const dispatch = useDispatch(); - const queryEditor = useQueryEditor(queryEditorId, ['dbId', 'schema']); + const queryEditor = useQueryEditor(queryEditorId, [ + 'dbId', + 'catalog', + 'schema', + ]); const [emptyResultsWithSearch, setEmptyResultsWithSearch] = useState(false); const [userSelectedDb, setUserSelected] = useState( null, ); - const { schema } = queryEditor; + const { catalog, schema } = queryEditor; useEffect(() => { const bool = querystring.parse(window.location.search).db; @@ -138,9 +143,9 @@ const SqlEditorLeftBar = ({ } }, [database]); - const onEmptyResults = (searchText?: string) => { + const onEmptyResults = useCallback((searchText?: string) => { setEmptyResultsWithSearch(!!searchText); - }; + }, []); const onDbChange = ({ id: dbId }: { id: number }) => { setEmptyState?.(false); @@ -152,7 +157,11 @@ const SqlEditorLeftBar = ({ [tables], ); - const onTablesChange = (tableNames: string[], schemaName: string) => { + const onTablesChange = ( + tableNames: string[], + catalogName: string | null, + schemaName: string, + ) => { if (!schemaName) { return; } @@ -169,7 +178,7 @@ const SqlEditorLeftBar = ({ }); tablesToAdd.forEach(tableName => { - dispatch(addTable(queryEditor, tableName, schemaName)); + dispatch(addTable(queryEditor, tableName, catalogName, schemaName)); }); dispatch(removeTables(currentTables)); @@ -210,6 +219,15 @@ const SqlEditorLeftBar = ({ const shouldShowReset = window.location.search === '?reset=1'; const tableMetaDataHeight = height - 130; // 130 is the height of the selects above + const handleCatalogChange = useCallback( + (catalog: string | null) => { + if (queryEditor) { + dispatch(queryEditorSetCatalog(queryEditor, catalog)); + } + }, + [dispatch, queryEditor], + ); + const handleSchemaChange = useCallback( (schema: string) => { if (queryEditor) { @@ -246,9 +264,11 @@ const SqlEditorLeftBar = ({ getDbList={handleDbList} handleError={handleError} onDbChange={onDbChange} + onCatalogChange={handleCatalogChange} + catalog={catalog} onSchemaChange={handleSchemaChange} - onTableSelectChange={onTablesChange} schema={schema} + onTableSelectChange={onTablesChange} tableValue={selectedTableNames} sqlLabMode /> diff --git a/superset-frontend/src/SqlLab/components/TabbedSqlEditors/index.tsx b/superset-frontend/src/SqlLab/components/TabbedSqlEditors/index.tsx index 078276ad26abf..7b4db1cbe844c 100644 --- a/superset-frontend/src/SqlLab/components/TabbedSqlEditors/index.tsx +++ b/superset-frontend/src/SqlLab/components/TabbedSqlEditors/index.tsx @@ -111,6 +111,7 @@ class TabbedSqlEditors extends React.PureComponent { queryId, dbid, dbname, + catalog, schema, autorun, new: isNewQuery, @@ -149,6 +150,7 @@ class TabbedSqlEditors extends React.PureComponent { const newQueryEditor = { name, dbId: databaseId, + catalog, schema, autorun, sql, diff --git a/superset-frontend/src/SqlLab/components/TableElement/index.tsx b/superset-frontend/src/SqlLab/components/TableElement/index.tsx index e29a654c227c7..2f93491e88817 100644 --- a/superset-frontend/src/SqlLab/components/TableElement/index.tsx +++ b/superset-frontend/src/SqlLab/components/TableElement/index.tsx @@ -101,7 +101,7 @@ const StyledCollapsePanel = styled(Collapse.Panel)` `; const TableElement = ({ table, ...props }: TableElementProps) => { - const { dbId, schema, name, expanded } = table; + const { dbId, catalog, schema, name, expanded } = table; const theme = useTheme(); const dispatch = useDispatch(); const { @@ -112,6 +112,7 @@ const TableElement = ({ table, ...props }: TableElementProps) => { } = useTableMetadataQuery( { dbId, + catalog, schema, table: name, }, @@ -125,6 +126,7 @@ const TableElement = ({ table, ...props }: TableElementProps) => { } = useTableExtendedMetadataQuery( { dbId, + catalog, schema, table: name, }, diff --git a/superset-frontend/src/SqlLab/fixtures.ts b/superset-frontend/src/SqlLab/fixtures.ts index 742145c58ca6d..54f1c278f8f2b 100644 --- a/superset-frontend/src/SqlLab/fixtures.ts +++ b/superset-frontend/src/SqlLab/fixtures.ts @@ -36,6 +36,7 @@ export const table = { dbId: 1, selectStar: 'SELECT * FROM ab_user', queryEditorId: 'dfsadfs', + catalog: null, schema: 'superset', name: 'ab_user', id: 'r11Vgt60', @@ -191,6 +192,7 @@ export const defaultQueryEditor = { selectedText: undefined, sql: 'SELECT *\nFROM\nWHERE', name: 'Untitled Query 1', + catalog: null, schema: 'main', remoteId: null, hideLeftBar: false, @@ -233,6 +235,7 @@ export const queries = [ queryLimit: 100, endDttm: 1476910566798, limit_reached: false, + catalog: null, schema: 'test_schema', errorMessage: null, db: 'main', @@ -294,6 +297,7 @@ export const queries = [ rows: 42, endDttm: 1476910579693, limit_reached: false, + catalog: null, schema: null, errorMessage: null, db: 'main', @@ -323,6 +327,7 @@ export const queryWithNoQueryLimit = { rows: 42, endDttm: 1476910566798, limit_reached: false, + catalog: null, schema: 'test_schema', errorMessage: null, db: 'main', @@ -456,18 +461,21 @@ export const tables = { options: [ { value: 'birth_names', + catalog: null, schema: 'main', label: 'birth_names', title: 'birth_names', }, { value: 'energy_usage', + catalog: null, schema: 'main', label: 'energy_usage', title: 'energy_usage', }, { value: 'wb_health_population', + catalog: null, schema: 'main', label: 'wb_health_population', title: 'wb_health_population', @@ -483,6 +491,7 @@ export const stoppedQuery = { progress: 0, results: [], runAsync: false, + catalog: null, schema: 'main', sql: 'SELECT ...', sqlEditorId: 'rJaf5u9WZ', @@ -501,6 +510,7 @@ export const failedQueryWithErrorMessage = { progress: 0, results: [], runAsync: false, + catalog: null, schema: 'main', sql: 'SELECT ...', sqlEditorId: 'rJaf5u9WZ', @@ -526,6 +536,7 @@ export const failedQueryWithErrors = { progress: 0, results: [], runAsync: false, + catalog: null, schema: 'main', sql: 'SELECT ...', sqlEditorId: 'rJaf5u9WZ', @@ -555,6 +566,7 @@ const baseQuery: QueryResponse = { started: 'started', queryLimit: 100, endDttm: 1476910566798, + catalog: null, schema: 'test_schema', errorMessage: null, db: { key: 'main' }, @@ -689,6 +701,7 @@ export const query = { dbId: 1, sql: 'SELECT * FROM something', description: 'test description', + catalog: null, schema: 'test schema', resultsKey: 'test', }; @@ -698,6 +711,7 @@ export const queryId = 'clientId2353'; export const testQuery: ISaveableDatasource = { name: 'unimportant', dbId: 1, + catalog: null, schema: 'main', sql: 'SELECT *', columns: [ @@ -727,6 +741,7 @@ export const mockdatasets = [...new Array(3)].map((_, i) => ({ database_name: `db ${i}`, explore_url: `/explore/?datasource_type=table&datasource_id=${i}`, id: i, + catalog: null, schema: `schema ${i}`, table_name: `coolest table ${i}`, owners: [{ username: 'admin', userId: 1 }], diff --git a/superset-frontend/src/SqlLab/reducers/getInitialState.ts b/superset-frontend/src/SqlLab/reducers/getInitialState.ts index bcdc1f40c33da..52a9770854a06 100644 --- a/superset-frontend/src/SqlLab/reducers/getInitialState.ts +++ b/superset-frontend/src/SqlLab/reducers/getInitialState.ts @@ -89,6 +89,7 @@ export default function getInitialState({ autorun: Boolean(activeTab.autorun), templateParams: activeTab.template_params || undefined, dbId: activeTab.database_id, + catalog: activeTab.catalog, schema: activeTab.schema, queryLimit: activeTab.query_limit, hideLeftBar: activeTab.hide_left_bar, @@ -121,6 +122,7 @@ export default function getInitialState({ const table = { dbId: tableSchema.database_id, queryEditorId: tableSchema.tab_state_id.toString(), + catalog: tableSchema.catalog, schema: tableSchema.schema, name: tableSchema.table, expanded: tableSchema.expanded, diff --git a/superset-frontend/src/SqlLab/reducers/sqlLab.js b/superset-frontend/src/SqlLab/reducers/sqlLab.js index 9ffcb4dfcb856..ecc0f090a9aaf 100644 --- a/superset-frontend/src/SqlLab/reducers/sqlLab.js +++ b/superset-frontend/src/SqlLab/reducers/sqlLab.js @@ -109,6 +109,7 @@ export default function sqlLabReducer(state = {}, action) { remoteId: progenitor.remoteId, name: t('Copy of %s', progenitor.name), dbId: action.query.dbId ? action.query.dbId : null, + catalog: action.query.catalog ? action.query.catalog : null, schema: action.query.schema ? action.query.schema : null, autorun: true, sql: action.query.sql, @@ -180,6 +181,7 @@ export default function sqlLabReducer(state = {}, action) { if ( xt.dbId === at.dbId && xt.queryEditorId === at.queryEditorId && + xt.catalog === at.catalog && xt.schema === at.schema && xt.name === at.name ) { @@ -503,6 +505,18 @@ export default function sqlLabReducer(state = {}, action) { ), }; }, + [actions.QUERY_EDITOR_SET_CATALOG]() { + return { + ...state, + ...alterUnsavedQueryEditorState( + state, + { + catalog: action.catalog, + }, + action.queryEditor.id, + ), + }; + }, [actions.QUERY_EDITOR_SET_SCHEMA]() { return { ...state, diff --git a/superset-frontend/src/SqlLab/types.ts b/superset-frontend/src/SqlLab/types.ts index cac9ceb5d91c4..b1dea6f2e31de 100644 --- a/superset-frontend/src/SqlLab/types.ts +++ b/superset-frontend/src/SqlLab/types.ts @@ -50,6 +50,7 @@ export interface QueryEditor { dbId?: number; name: string; title?: string; // keep it optional for backward compatibility + catalog?: string | null; schema?: string; autorun: boolean; sql: string; @@ -81,6 +82,7 @@ export type UnsavedQueryEditor = Partial; export interface Table { id: string; dbId: number; + catalog: string | null; schema: string; name: string; queryEditorId: QueryEditor['id']; diff --git a/superset-frontend/src/SqlLab/utils/reduxStateToLocalStorageHelper.ts b/superset-frontend/src/SqlLab/utils/reduxStateToLocalStorageHelper.ts index 8b7f41f9f7481..3061de1f69b07 100644 --- a/superset-frontend/src/SqlLab/utils/reduxStateToLocalStorageHelper.ts +++ b/superset-frontend/src/SqlLab/utils/reduxStateToLocalStorageHelper.ts @@ -109,22 +109,24 @@ export function rehydratePersistedState( state: SqlLabRootState, ) { // Rehydrate server side persisted table metadata - state.sqlLab.tables.forEach(({ name: table, schema, dbId, persistData }) => { - if (dbId && schema && table && persistData?.columns) { - dispatch( - tableApiUtil.upsertQueryData( - 'tableMetadata', - { dbId, schema, table }, - persistData, - ), - ); - dispatch( - tableApiUtil.upsertQueryData( - 'tableExtendedMetadata', - { dbId, schema, table }, - {}, - ), - ); - } - }); + state.sqlLab.tables.forEach( + ({ name: table, catalog, schema, dbId, persistData }) => { + if (dbId && schema && table && persistData?.columns) { + dispatch( + tableApiUtil.upsertQueryData( + 'tableMetadata', + { dbId, catalog, schema, table }, + persistData, + ), + ); + dispatch( + tableApiUtil.upsertQueryData( + 'tableExtendedMetadata', + { dbId, catalog, schema, table }, + {}, + ), + ); + } + }, + ); } diff --git a/superset-frontend/src/components/DatabaseSelector/DatabaseSelector.test.tsx b/superset-frontend/src/components/DatabaseSelector/DatabaseSelector.test.tsx index c3ad51cf60799..32373301f6030 100644 --- a/superset-frontend/src/components/DatabaseSelector/DatabaseSelector.test.tsx +++ b/superset-frontend/src/components/DatabaseSelector/DatabaseSelector.test.tsx @@ -40,6 +40,7 @@ const createProps = (): DatabaseSelectorProps => ({ formMode: false, isDatabaseSelectEnabled: true, readOnly: false, + catalog: null, schema: 'public', sqlLabMode: true, getDbList: jest.fn(), @@ -158,16 +159,23 @@ const fakeSchemaApiResult = { result: ['information_schema', 'public'], }; +const fakeCatalogApiResult = { + count: 0, + result: [], +}; + const fakeFunctionNamesApiResult = { function_names: [], }; const databaseApiRoute = 'glob:*/api/v1/database/?*'; +const catalogApiRoute = 'glob:*/api/v1/database/*/catalogs/?*'; const schemaApiRoute = 'glob:*/api/v1/database/*/schemas/?*'; const tablesApiRoute = 'glob:*/api/v1/database/*/tables/*'; function setupFetchMock() { fetchMock.get(databaseApiRoute, fakeDatabaseApiResult); + fetchMock.get(catalogApiRoute, fakeCatalogApiResult); fetchMock.get(schemaApiRoute, fakeSchemaApiResult); fetchMock.get(tablesApiRoute, fakeFunctionNamesApiResult); } diff --git a/superset-frontend/src/components/DatabaseSelector/index.tsx b/superset-frontend/src/components/DatabaseSelector/index.tsx index 0c0268db5c9e6..6eb1340d5bcc3 100644 --- a/superset-frontend/src/components/DatabaseSelector/index.tsx +++ b/superset-frontend/src/components/DatabaseSelector/index.tsx @@ -24,7 +24,12 @@ import Label from 'src/components/Label'; import { FormLabel } from 'src/components/Form'; import RefreshLabel from 'src/components/RefreshLabel'; import { useToasts } from 'src/components/MessageToasts/withToasts'; -import { useSchemas, SchemaOption } from 'src/hooks/apiResources'; +import { + useCatalogs, + CatalogOption, + useSchemas, + SchemaOption, +} from 'src/hooks/apiResources'; const DatabaseSelectorWrapper = styled.div` ${({ theme }) => ` @@ -81,6 +86,7 @@ export type DatabaseObject = { id: number; database_name: string; backend?: string; + allow_multi_catalog?: boolean; }; export interface DatabaseSelectorProps { @@ -92,9 +98,11 @@ export interface DatabaseSelectorProps { isDatabaseSelectEnabled?: boolean; onDbChange?: (db: DatabaseObject) => void; onEmptyResults?: (searchText?: string) => void; + onCatalogChange?: (catalog?: string) => void; + catalog?: string | null; onSchemaChange?: (schema?: string) => void; - readOnly?: boolean; schema?: string; + readOnly?: boolean; sqlLabMode?: boolean; } @@ -113,6 +121,7 @@ const SelectLabel = ({ ); +const EMPTY_CATALOG_OPTIONS: CatalogOption[] = []; const EMPTY_SCHEMA_OPTIONS: SchemaOption[] = []; export default function DatabaseSelector({ @@ -124,12 +133,20 @@ export default function DatabaseSelector({ isDatabaseSelectEnabled = true, onDbChange, onEmptyResults, + onCatalogChange, + catalog, onSchemaChange, - readOnly = false, schema, + readOnly = false, sqlLabMode = false, }: DatabaseSelectorProps) { + const showCatalogSelector = !!db?.allow_multi_catalog; const [currentDb, setCurrentDb] = useState(); + const [currentCatalog, setCurrentCatalog] = useState< + CatalogOption | undefined + >(catalog ? { label: catalog, value: catalog, title: catalog } : undefined); + const catalogRef = useRef(catalog); + catalogRef.current = catalog; const [currentSchema, setCurrentSchema] = useState( schema ? { label: schema, value: schema, title: schema } : undefined, ); @@ -185,6 +202,7 @@ export default function DatabaseSelector({ id: row.id, database_name: row.database_name, backend: row.backend, + allow_multi_catalog: row.allow_multi_catalog, })); return { @@ -193,7 +211,7 @@ export default function DatabaseSelector({ }; }); }, - [formMode, getDbList, sqlLabMode], + [formMode, getDbList, sqlLabMode, onEmptyResults], ); useEffect(() => { @@ -223,11 +241,12 @@ export default function DatabaseSelector({ } const { - data, + data: schemaData, isFetching: loadingSchemas, - refetch, + refetch: refetchSchemas, } = useSchemas({ dbId: currentDb?.value, + catalog: currentCatalog?.value, onSuccess: (schemas, isFetched) => { if (schemas.length === 1) { changeSchema(schemas[0]); @@ -244,17 +263,55 @@ export default function DatabaseSelector({ onError: () => handleError(t('There was an error loading the schemas')), }); - const schemaOptions = data || EMPTY_SCHEMA_OPTIONS; + const schemaOptions = schemaData || EMPTY_SCHEMA_OPTIONS; + + function changeCatalog(catalog: CatalogOption | undefined) { + setCurrentCatalog(catalog); + setCurrentSchema(undefined); + if (onCatalogChange && catalog?.value !== catalogRef.current) { + onCatalogChange(catalog?.value); + } + } + + const { + data: catalogData, + isFetching: loadingCatalogs, + refetch: refetchCatalogs, + } = useCatalogs({ + dbId: currentDb?.value, + onSuccess: (catalogs, isFetched) => { + if (catalogs.length === 1) { + changeCatalog(catalogs[0]); + } else if ( + !catalogs.find( + catalogOption => catalogRef.current === catalogOption.value, + ) + ) { + changeCatalog(undefined); + } + + if (isFetched) { + addSuccessToast('List refreshed'); + } + }, + onError: () => handleError(t('There was an error loading the catalogs')), + }); + + const catalogOptions = catalogData || EMPTY_CATALOG_OPTIONS; - function changeDataBase( + function changeDatabase( value: { label: string; value: number }, database: DatabaseValue, ) { setCurrentDb(database); + setCurrentCatalog(undefined); setCurrentSchema(undefined); if (onDbChange) { onDbChange(database); } + if (onCatalogChange) { + onCatalogChange(undefined); + } if (onSchemaChange) { onSchemaChange(undefined); } @@ -278,7 +335,7 @@ export default function DatabaseSelector({ header={{t('Database')}} lazyLoading={false} notFoundContent={emptyState} - onChange={changeDataBase} + onChange={changeDatabase} value={currentDb} placeholder={t('Select database or type to search databases')} disabled={!isDatabaseSelectEnabled || readOnly} @@ -288,10 +345,36 @@ export default function DatabaseSelector({ ); } + function renderCatalogSelect() { + const refreshIcon = !readOnly && ( + + ); + return renderSelectRow( +