From 54de3647091c119a64645d81240e61a5ecbb3df3 Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Tue, 9 Aug 2022 17:07:18 -0700 Subject: [PATCH 1/7] register data source route handler context Signed-off-by: Zhongnan Su --- .../server/client/data_source_client.ts | 47 +++++++++++++++++++ .../data_source/server/client/index.ts | 6 +++ .../data_source_route_handler_context.ts | 39 +++++++++++++++ src/plugins/data_source/server/plugin.ts | 30 +++++++++++- src/plugins/data_source/server/types.ts | 13 +++++ 5 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 src/plugins/data_source/server/client/data_source_client.ts create mode 100644 src/plugins/data_source/server/client/index.ts create mode 100644 src/plugins/data_source/server/data_source_route_handler_context.ts diff --git a/src/plugins/data_source/server/client/data_source_client.ts b/src/plugins/data_source/server/client/data_source_client.ts new file mode 100644 index 000000000000..3c3be5993d88 --- /dev/null +++ b/src/plugins/data_source/server/client/data_source_client.ts @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Logger, OpenSearchClient, SavedObjectsClientContract } from 'src/core/server'; + +/** + * Represents an OpenSearch cluster API client created by the platform. + * It allows to call API on behalf of user defined in "data source" saved object + * + * @public + **/ +export interface IDataSourceClient { + /** + * Creates a {@link OpenSearchClient } bound to given data source + */ + asDataSource: (dataSourceId: string) => Promise; +} + +/** + * See {@link IDataSourceClient} + * + * @public + */ +export interface ICustomDataSourceClient extends IDataSourceClient { + /** + * Closes the data source client. After that client cannot be used and one should + * create a new client instance to be able to interact with OpenSearch API. + */ + close: () => Promise; +} + +// TODO: This needs further implementation. See https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1981 +export class DataSourceClient implements ICustomDataSourceClient { + private savedObjectClient?: SavedObjectsClientContract; + + constructor(logger: Logger) {} + asDataSource!: (dataSourceId: string) => Promise; + // asDataSource: (dataSourceId: string) => Promise; + + public attachSavedObjectClient(savedObjectClient: SavedObjectsClientContract) { + this.savedObjectClient = savedObjectClient; + } + + public async close() {} +} diff --git a/src/plugins/data_source/server/client/index.ts b/src/plugins/data_source/server/client/index.ts new file mode 100644 index 000000000000..62fdc135345e --- /dev/null +++ b/src/plugins/data_source/server/client/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { IDataSourceClient, ICustomDataSourceClient, DataSourceClient } from './data_source_client'; diff --git a/src/plugins/data_source/server/data_source_route_handler_context.ts b/src/plugins/data_source/server/data_source_route_handler_context.ts new file mode 100644 index 000000000000..3182f2c6e8d6 --- /dev/null +++ b/src/plugins/data_source/server/data_source_route_handler_context.ts @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +// eslint-disable-next-line max-classes-per-file +import { Logger } from 'src/core/server'; +import { IDataSourceClient } from './client/data_source_client'; + +class OpenSearchDataSourceRouteHandlerContext { + private logger: Logger; + constructor(private dataSourceClient: IDataSourceClient, logger: Logger) { + this.logger = logger; + } + + public async getClient(dataSourceId: string) { + try { + const client = await this.dataSourceClient.asDataSource(dataSourceId); + return client; + } catch (error) { + // TODO: convert as audit log when integrate with osd auditing + this.logger.error( + `Fail to get data source client for dataSource id: [${dataSourceId}]. Detail: ${error.messages}` + ); + throw error; + } + } +} + +export class DataSourceRouteHandlerContext { + readonly opensearch: OpenSearchDataSourceRouteHandlerContext; + + constructor(private readonly dataSourceClient: IDataSourceClient, logger: Logger) { + this.opensearch = new OpenSearchDataSourceRouteHandlerContext( + this.dataSourceClient, + logger.get('opensearch') + ); + } +} diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts index 5a4ed403457c..1a6e0e2a862c 100644 --- a/src/plugins/data_source/server/plugin.ts +++ b/src/plugins/data_source/server/plugin.ts @@ -5,9 +5,19 @@ import { first } from 'rxjs/operators'; -import { PluginInitializerContext, CoreSetup, CoreStart, Plugin, Logger } from 'src/core/server'; import { dataSource, credential, CredentialSavedObjectsClientWrapper } from './saved_objects'; import { DataSourcePluginConfigType } from '../config'; +import { + PluginInitializerContext, + CoreSetup, + CoreStart, + Plugin, + Logger, + IContextProvider, + RequestHandler, +} from '../../../../src/core/server'; +import { DataSourceClient } from './client/data_source_client'; +import { DataSourceRouteHandlerContext } from './data_source_route_handler_context'; import { DataSourcePluginSetup, DataSourcePluginStart } from './types'; @@ -15,9 +25,11 @@ import { CryptographyClient } from './cryptography'; export class DataSourcePlugin implements Plugin { private readonly logger: Logger; + private readonly dataSourceClient: DataSourceClient; constructor(private initializerContext: PluginInitializerContext) { this.logger = this.initializerContext.logger.get(); + this.dataSourceClient = new DataSourceClient(this.logger); } public async setup(core: CoreSetup) { @@ -44,6 +56,8 @@ export class DataSourcePlugin implements Plugin, 'data_source'> => { + return async (context, req) => { + const [{ savedObjects }] = await core.getStartServices(); + this.dataSourceClient.attachSavedObjectClient(savedObjects.getScopedClient(req)); + return new DataSourceRouteHandlerContext(this.dataSourceClient, this.logger); + }; + }; } diff --git a/src/plugins/data_source/server/types.ts b/src/plugins/data_source/server/types.ts index a70c8c1bf199..f8ffdc1598d1 100644 --- a/src/plugins/data_source/server/types.ts +++ b/src/plugins/data_source/server/types.ts @@ -3,6 +3,19 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { OpenSearchClient } from 'src/core/server'; + +export interface DataSourcePluginRequestContext { + opensearch: { + getClient: (dataSourceId: string) => Promise; + }; +} +declare module 'src/core/server' { + interface RequestHandlerContext { + data_source: DataSourcePluginRequestContext; + } +} + // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface DataSourcePluginSetup {} // eslint-disable-next-line @typescript-eslint/no-empty-interface From b0591e91839098ce51d5970206cd6d0e660fc457 Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Wed, 10 Aug 2022 15:43:58 -0700 Subject: [PATCH 2/7] address comments Signed-off-by: Zhongnan Su --- .../data_source/server/client/data_source_client.ts | 9 ++++----- .../server/data_source_route_handler_context.ts | 6 ++---- src/plugins/data_source/server/plugin.ts | 4 ++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/plugins/data_source/server/client/data_source_client.ts b/src/plugins/data_source/server/client/data_source_client.ts index 3c3be5993d88..88d874bf467d 100644 --- a/src/plugins/data_source/server/client/data_source_client.ts +++ b/src/plugins/data_source/server/client/data_source_client.ts @@ -7,7 +7,7 @@ import { Logger, OpenSearchClient, SavedObjectsClientContract } from 'src/core/s /** * Represents an OpenSearch cluster API client created by the platform. - * It allows to call API on behalf of user defined in "data source" saved object + * It allows to call API on behalf of the user(credential) associated to "data source" * * @public **/ @@ -33,14 +33,13 @@ export interface ICustomDataSourceClient extends IDataSourceClient { // TODO: This needs further implementation. See https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1981 export class DataSourceClient implements ICustomDataSourceClient { - private savedObjectClient?: SavedObjectsClientContract; + private scopedSavedObjectsClient?: SavedObjectsClientContract; constructor(logger: Logger) {} asDataSource!: (dataSourceId: string) => Promise; - // asDataSource: (dataSourceId: string) => Promise; - public attachSavedObjectClient(savedObjectClient: SavedObjectsClientContract) { - this.savedObjectClient = savedObjectClient; + public attachScopedSavedObjectsClient(scopedSavedObjectsClient: SavedObjectsClientContract) { + this.scopedSavedObjectsClient = scopedSavedObjectsClient; } public async close() {} diff --git a/src/plugins/data_source/server/data_source_route_handler_context.ts b/src/plugins/data_source/server/data_source_route_handler_context.ts index 3182f2c6e8d6..b238b254a93c 100644 --- a/src/plugins/data_source/server/data_source_route_handler_context.ts +++ b/src/plugins/data_source/server/data_source_route_handler_context.ts @@ -8,10 +8,7 @@ import { Logger } from 'src/core/server'; import { IDataSourceClient } from './client/data_source_client'; class OpenSearchDataSourceRouteHandlerContext { - private logger: Logger; - constructor(private dataSourceClient: IDataSourceClient, logger: Logger) { - this.logger = logger; - } + constructor(private dataSourceClient: IDataSourceClient, private logger: Logger) {} public async getClient(dataSourceId: string) { try { @@ -19,6 +16,7 @@ class OpenSearchDataSourceRouteHandlerContext { return client; } catch (error) { // TODO: convert as audit log when integrate with osd auditing + // https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1986 this.logger.error( `Fail to get data source client for dataSource id: [${dataSourceId}]. Detail: ${error.messages}` ); diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts index 1a6e0e2a862c..7fb1938f50bb 100644 --- a/src/plugins/data_source/server/plugin.ts +++ b/src/plugins/data_source/server/plugin.ts @@ -32,7 +32,7 @@ export class DataSourcePlugin implements Plugin, 'data_source'> => { return async (context, req) => { const [{ savedObjects }] = await core.getStartServices(); - this.dataSourceClient.attachSavedObjectClient(savedObjects.getScopedClient(req)); + this.dataSourceClient.attachScopedSavedObjectsClient(savedObjects.getScopedClient(req)); return new DataSourceRouteHandlerContext(this.dataSourceClient, this.logger); }; }; From d9b236865b976869228ae4a76fe03b9fc148cf9f Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Wed, 10 Aug 2022 23:55:50 -0700 Subject: [PATCH 3/7] init data source service Signed-off-by: Zhongnan Su --- .../server/client/data_source_client.ts | 21 +++++++---- .../data_source/server/data_source_service.ts | 35 +++++++++++++++++++ src/plugins/data_source/server/plugin.ts | 17 +++++---- src/plugins/data_source/server/types.ts | 13 ++++++- 4 files changed, 73 insertions(+), 13 deletions(-) create mode 100644 src/plugins/data_source/server/data_source_service.ts diff --git a/src/plugins/data_source/server/client/data_source_client.ts b/src/plugins/data_source/server/client/data_source_client.ts index 88d874bf467d..f425a8f78311 100644 --- a/src/plugins/data_source/server/client/data_source_client.ts +++ b/src/plugins/data_source/server/client/data_source_client.ts @@ -4,6 +4,7 @@ */ import { Logger, OpenSearchClient, SavedObjectsClientContract } from 'src/core/server'; +import { DataSourceService } from '../data_source_service'; /** * Represents an OpenSearch cluster API client created by the platform. @@ -31,16 +32,24 @@ export interface ICustomDataSourceClient extends IDataSourceClient { close: () => Promise; } +interface DataSourceClientCtorParams { + dataSourceService: DataSourceService; + logger: Logger; + scopedSavedObjectsClient: SavedObjectsClientContract; +} // TODO: This needs further implementation. See https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1981 export class DataSourceClient implements ICustomDataSourceClient { - private scopedSavedObjectsClient?: SavedObjectsClientContract; - - constructor(logger: Logger) {} - asDataSource!: (dataSourceId: string) => Promise; + private dataSourceService: DataSourceService; + private log: Logger; + private scopedSavedObjectClient; - public attachScopedSavedObjectsClient(scopedSavedObjectsClient: SavedObjectsClientContract) { - this.scopedSavedObjectsClient = scopedSavedObjectsClient; + constructor(ctorParams: DataSourceClientCtorParams) { + this.dataSourceService = ctorParams.dataSourceService; + this.log = ctorParams.logger; + this.scopedSavedObjectClient = ctorParams.scopedSavedObjectsClient; } + asDataSource!: (dataSourceId: string) => Promise; + public async close() {} } diff --git a/src/plugins/data_source/server/data_source_service.ts b/src/plugins/data_source/server/data_source_service.ts new file mode 100644 index 000000000000..8497b60c2217 --- /dev/null +++ b/src/plugins/data_source/server/data_source_service.ts @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Logger, OpenSearchClient, SavedObjectsClientContract } from 'src/core/server'; +import { DataSourceClient } from './client'; +import { IDataSourceService } from './types'; + +export class DataSourceService implements IDataSourceService { + private openSearchClientsPool: Map; + constructor() { + this.openSearchClientsPool = new Map(); + } + // TODO: placeholders, need implement when adding global config + isEnabled(): boolean { + throw new Error('Method not implemented.'); + } + + getDataSourceClient(logger: Logger, savedObjectClient: SavedObjectsClientContract) { + return new DataSourceClient({ + logger, + dataSourceService: this, + scopedSavedObjectsClient: savedObjectClient, + }); + } + // TODO: placeholders, need implement client pooling strategy + addOpenSearchClient() {} + getOpenSearchClient(): OpenSearchClient { + throw new Error('Method not implemented.'); + } + + // TODO: close all data source clients in the clients pool + stop() {} +} diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts index 7fb1938f50bb..53449e6436e0 100644 --- a/src/plugins/data_source/server/plugin.ts +++ b/src/plugins/data_source/server/plugin.ts @@ -18,6 +18,7 @@ import { } from '../../../../src/core/server'; import { DataSourceClient } from './client/data_source_client'; import { DataSourceRouteHandlerContext } from './data_source_route_handler_context'; +import { DataSourceService } from './data_source_service'; import { DataSourcePluginSetup, DataSourcePluginStart } from './types'; @@ -25,14 +26,13 @@ import { CryptographyClient } from './cryptography'; export class DataSourcePlugin implements Plugin { private readonly logger: Logger; - private readonly dataSourceClient: DataSourceClient; + private dataSourceService?: DataSourceService; constructor(private initializerContext: PluginInitializerContext) { this.logger = this.initializerContext.logger.get(); - this.dataSourceClient = new DataSourceClient(this.logger); } - public setup(core: CoreSetup) { + public async setup(core: CoreSetup) { this.logger.debug('data_source: Setup'); // Register credential saved object type @@ -56,6 +56,8 @@ export class DataSourcePlugin implements Plugin, 'data_source'> => { return async (context, req) => { const [{ savedObjects }] = await core.getStartServices(); - this.dataSourceClient.attachScopedSavedObjectsClient(savedObjects.getScopedClient(req)); - return new DataSourceRouteHandlerContext(this.dataSourceClient, this.logger); + const dataSourceClient = this.dataSourceService!.getDataSourceClient( + this.logger, + savedObjects.getScopedClient(req) + ); + return new DataSourceRouteHandlerContext(dataSourceClient, this.logger); }; }; } diff --git a/src/plugins/data_source/server/types.ts b/src/plugins/data_source/server/types.ts index f8ffdc1598d1..e3cb57963091 100644 --- a/src/plugins/data_source/server/types.ts +++ b/src/plugins/data_source/server/types.ts @@ -3,8 +3,19 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { OpenSearchClient } from 'src/core/server'; +import { Logger, OpenSearchClient, SavedObjectsClientContract } from 'src/core/server'; +import { DataSourceClient } from './client'; +export interface IDataSourceService { + isEnabled(): boolean; + getDataSourceClient( + logger: Logger, + savedObjectClient: SavedObjectsClientContract + ): DataSourceClient; + addOpenSearchClient(): void; + getOpenSearchClient(): OpenSearchClient; + stop(): void; +} export interface DataSourcePluginRequestContext { opensearch: { getClient: (dataSourceId: string) => Promise; From e31e1614fb7747729a571a89e2ca5c3e5b8c6106 Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Thu, 11 Aug 2022 12:25:21 -0700 Subject: [PATCH 4/7] clean up & address comments Signed-off-by: Zhongnan Su --- .../server/client/data_source_client.ts | 17 +------------- .../data_source_route_handler_context.ts | 16 ++++++++++++- .../data_source/server/data_source_service.ts | 5 ++-- src/plugins/data_source/server/plugin.ts | 23 ++++--------------- 4 files changed, 24 insertions(+), 37 deletions(-) diff --git a/src/plugins/data_source/server/client/data_source_client.ts b/src/plugins/data_source/server/client/data_source_client.ts index f425a8f78311..c95aef580f94 100644 --- a/src/plugins/data_source/server/client/data_source_client.ts +++ b/src/plugins/data_source/server/client/data_source_client.ts @@ -19,26 +19,13 @@ export interface IDataSourceClient { asDataSource: (dataSourceId: string) => Promise; } -/** - * See {@link IDataSourceClient} - * - * @public - */ -export interface ICustomDataSourceClient extends IDataSourceClient { - /** - * Closes the data source client. After that client cannot be used and one should - * create a new client instance to be able to interact with OpenSearch API. - */ - close: () => Promise; -} - interface DataSourceClientCtorParams { dataSourceService: DataSourceService; logger: Logger; scopedSavedObjectsClient: SavedObjectsClientContract; } // TODO: This needs further implementation. See https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1981 -export class DataSourceClient implements ICustomDataSourceClient { +export class DataSourceClient implements IDataSourceClient { private dataSourceService: DataSourceService; private log: Logger; private scopedSavedObjectClient; @@ -50,6 +37,4 @@ export class DataSourceClient implements ICustomDataSourceClient { } asDataSource!: (dataSourceId: string) => Promise; - - public async close() {} } diff --git a/src/plugins/data_source/server/data_source_route_handler_context.ts b/src/plugins/data_source/server/data_source_route_handler_context.ts index b238b254a93c..e66e69785a47 100644 --- a/src/plugins/data_source/server/data_source_route_handler_context.ts +++ b/src/plugins/data_source/server/data_source_route_handler_context.ts @@ -4,8 +4,9 @@ */ // eslint-disable-next-line max-classes-per-file -import { Logger } from 'src/core/server'; +import { IContextProvider, Logger, RequestHandler } from 'src/core/server'; import { IDataSourceClient } from './client/data_source_client'; +import { DataSourceService } from './data_source_service'; class OpenSearchDataSourceRouteHandlerContext { constructor(private dataSourceClient: IDataSourceClient, private logger: Logger) {} @@ -35,3 +36,16 @@ export class DataSourceRouteHandlerContext { ); } } + +export const createDataSourceRouteHandlerContext = ( + dataSourceService: DataSourceService, + logger: Logger +): IContextProvider, 'data_source'> => { + return async (context, req) => { + const dataSourceClient = dataSourceService!.getDataSourceClient( + logger, + context.core.savedObjects.client + ); + return new DataSourceRouteHandlerContext(dataSourceClient, logger); + }; +}; diff --git a/src/plugins/data_source/server/data_source_service.ts b/src/plugins/data_source/server/data_source_service.ts index 8497b60c2217..98162a4324c9 100644 --- a/src/plugins/data_source/server/data_source_service.ts +++ b/src/plugins/data_source/server/data_source_service.ts @@ -3,14 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { Client } from '@opensearch-project/opensearch'; import { Logger, OpenSearchClient, SavedObjectsClientContract } from 'src/core/server'; import { DataSourceClient } from './client'; import { IDataSourceService } from './types'; export class DataSourceService implements IDataSourceService { - private openSearchClientsPool: Map; + private openSearchClientsPool: Map; constructor() { - this.openSearchClientsPool = new Map(); + this.openSearchClientsPool = new Map(); } // TODO: placeholders, need implement when adding global config isEnabled(): boolean { diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts index 53449e6436e0..1337f182d290 100644 --- a/src/plugins/data_source/server/plugin.ts +++ b/src/plugins/data_source/server/plugin.ts @@ -13,12 +13,9 @@ import { CoreStart, Plugin, Logger, - IContextProvider, - RequestHandler, } from '../../../../src/core/server'; -import { DataSourceClient } from './client/data_source_client'; -import { DataSourceRouteHandlerContext } from './data_source_route_handler_context'; import { DataSourceService } from './data_source_service'; +import { createDataSourceRouteHandlerContext } from './data_source_route_handler_context'; import { DataSourcePluginSetup, DataSourcePluginStart } from './types'; @@ -59,7 +56,10 @@ export class DataSourcePlugin implements Plugin, 'data_source'> => { - return async (context, req) => { - const [{ savedObjects }] = await core.getStartServices(); - const dataSourceClient = this.dataSourceService!.getDataSourceClient( - this.logger, - savedObjects.getScopedClient(req) - ); - return new DataSourceRouteHandlerContext(dataSourceClient, this.logger); - }; - }; } From 76f310489a0487e5872aaad997de0f7702073e0a Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Mon, 15 Aug 2022 18:25:25 -0700 Subject: [PATCH 5/7] add pooling impl in data source service Signed-off-by: Zhongnan Su --- src/plugins/data_source/common/index.ts | 2 + src/plugins/data_source/config.ts | 3 + .../server/client/data_source_client.ts | 125 +++++++++++++++++- .../data_source/server/client/index.ts | 2 +- .../data_source_route_handler_context.ts | 2 +- .../data_source/server/data_source_service.ts | 58 ++++++-- src/plugins/data_source/server/plugin.ts | 31 ++++- src/plugins/data_source/server/types.ts | 3 - 8 files changed, 196 insertions(+), 30 deletions(-) diff --git a/src/plugins/data_source/common/index.ts b/src/plugins/data_source/common/index.ts index bf5c6b1b0197..a2ae41868343 100644 --- a/src/plugins/data_source/common/index.ts +++ b/src/plugins/data_source/common/index.ts @@ -5,5 +5,7 @@ export const PLUGIN_ID = 'dataSource'; export const PLUGIN_NAME = 'data_source'; +export const DATA_SOURCE_SAVED_OBJECT_TYPE = 'data-source'; +export const CREDENTIAL_SAVED_OBJECT_TYPE = 'credential'; export { Credential } from './credentials'; diff --git a/src/plugins/data_source/config.ts b/src/plugins/data_source/config.ts index 5a8a654cd059..95e8bc3f96bb 100644 --- a/src/plugins/data_source/config.ts +++ b/src/plugins/data_source/config.ts @@ -29,6 +29,9 @@ export const configSchema = schema.object({ defaultValue: new Array(32).fill(0), }), }), + clientPool: schema.object({ + size: schema.number({ defaultValue: 5 }), + }), }); export type DataSourcePluginConfigType = TypeOf; diff --git a/src/plugins/data_source/server/client/data_source_client.ts b/src/plugins/data_source/server/client/data_source_client.ts index c95aef580f94..13ad71138ae5 100644 --- a/src/plugins/data_source/server/client/data_source_client.ts +++ b/src/plugins/data_source/server/client/data_source_client.ts @@ -3,7 +3,21 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { Logger, OpenSearchClient, SavedObjectsClientContract } from 'src/core/server'; +import { Client } from '@opensearch-project/opensearch'; +import { + Logger, + OpenSearchClient, + SavedObject, + SavedObjectsClientContract, + SavedObjectsErrorHelpers, +} from 'src/core/server'; +import { CREDENTIAL_SAVED_OBJECT_TYPE, DATA_SOURCE_SAVED_OBJECT_TYPE } from '../../common'; +import { + CredentialMaterials, + CredentialSavedObjectAttributes, +} from '../../common/credentials/types'; +import { DataSourceAttributes } from '../../common/data_sources'; +import { DataSourcePluginConfigType } from '../../config'; import { DataSourceService } from '../data_source_service'; /** @@ -23,18 +37,121 @@ interface DataSourceClientCtorParams { dataSourceService: DataSourceService; logger: Logger; scopedSavedObjectsClient: SavedObjectsClientContract; + config: DataSourcePluginConfigType; } -// TODO: This needs further implementation. See https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1981 export class DataSourceClient implements IDataSourceClient { private dataSourceService: DataSourceService; private log: Logger; - private scopedSavedObjectClient; + // scoped saved object client to fetch save object on behalf of user + private scopedSavedObjectClient: SavedObjectsClientContract; + private config: DataSourcePluginConfigType; constructor(ctorParams: DataSourceClientCtorParams) { this.dataSourceService = ctorParams.dataSourceService; this.log = ctorParams.logger; this.scopedSavedObjectClient = ctorParams.scopedSavedObjectsClient; + this.config = ctorParams.config; } - asDataSource!: (dataSourceId: string) => Promise; + async asDataSource(dataSourceId: string) { + const dataSource = await this.getDataSource(dataSourceId); + const rootClient = this.getRootClient(dataSource.attributes, this.config); + const credential = await this.getCredential(dataSource.references[0].id); // assuming there is 1 and only 1 credential for each data source + + return this.getQueryClient(rootClient, credential.attributes, dataSource.attributes.withAuth); + } + + private async getDataSource(dataSourceId: string): Promise> { + try { + const dataSource = await this.scopedSavedObjectClient.get( + DATA_SOURCE_SAVED_OBJECT_TYPE, + dataSourceId + ); + return dataSource; + } catch (error: any) { + // it will cause 500 error when failed to get saved objects, need to handle such error gracefully + throw SavedObjectsErrorHelpers.createBadRequestError(error.message); + } + } + + private async getCredential( + credentialId: string + ): Promise> { + try { + const dataSource = await this.scopedSavedObjectClient.get( + CREDENTIAL_SAVED_OBJECT_TYPE, + credentialId + ); + return dataSource; + } catch (error: any) { + // it will cause 500 error when failed to get saved objects, need to handle such error gracefully + throw SavedObjectsErrorHelpers.createBadRequestError(error.message); + } + } + + /** + * Create a child client object with given auth info. + * + * @param rootClient root client for the connection with given data source endpoint. + * @param credentialAttr credential saved object attribute. + * @returns child client. + */ + private getQueryClient( + rootClient: Client, + credentialAttr: CredentialSavedObjectAttributes, + withAuth = false + ): Client { + if (withAuth) { + return this.getBasicAuthClient(rootClient, credentialAttr.credentialMaterials); + } else { + return rootClient.child(); + } + } + + /** + * Gets a root client object of the OpenSearch endpoint. + * Will attempt to get from cache, if cache miss, create a new one and load into cache. + * + * @param dataSourceAttr data source saved objects attributes. + * @returns OpenSearch client for the given data source endpoint. + */ + private getRootClient( + dataSourceAttr: DataSourceAttributes, + config: DataSourcePluginConfigType + ): Client { + const endpoint = dataSourceAttr.endpoint; + const cachedClient = this.dataSourceService.getCachedClient(endpoint); + + if (cachedClient) { + return cachedClient; + } else { + const client = this.configureDataSourceClient(config, endpoint); + + this.dataSourceService.addClientToPool(endpoint, client); + return client; + } + } + + private getBasicAuthClient(rootClient: Client, credential: CredentialMaterials): Client { + const { username, password } = credential.credentialMaterialsContent; + return rootClient.child({ + auth: { + username, + password, + }, + }); + } + + // TODO: will use client configs, that comes from a merge result of user config and defaults + private configureDataSourceClient(config: DataSourcePluginConfigType, endpoint: string) { + const client = new Client({ + node: endpoint, + ssl: { + requestCert: true, + rejectUnauthorized: true, + }, + }); + + return client; + } } diff --git a/src/plugins/data_source/server/client/index.ts b/src/plugins/data_source/server/client/index.ts index 62fdc135345e..ef1b392c16ca 100644 --- a/src/plugins/data_source/server/client/index.ts +++ b/src/plugins/data_source/server/client/index.ts @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { IDataSourceClient, ICustomDataSourceClient, DataSourceClient } from './data_source_client'; +export { IDataSourceClient, DataSourceClient } from './data_source_client'; diff --git a/src/plugins/data_source/server/data_source_route_handler_context.ts b/src/plugins/data_source/server/data_source_route_handler_context.ts index e66e69785a47..352564a1b7d7 100644 --- a/src/plugins/data_source/server/data_source_route_handler_context.ts +++ b/src/plugins/data_source/server/data_source_route_handler_context.ts @@ -15,7 +15,7 @@ class OpenSearchDataSourceRouteHandlerContext { try { const client = await this.dataSourceClient.asDataSource(dataSourceId); return client; - } catch (error) { + } catch (error: any) { // TODO: convert as audit log when integrate with osd auditing // https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1986 this.logger.error( diff --git a/src/plugins/data_source/server/data_source_service.ts b/src/plugins/data_source/server/data_source_service.ts index 98162a4324c9..0e129ac46e95 100644 --- a/src/plugins/data_source/server/data_source_service.ts +++ b/src/plugins/data_source/server/data_source_service.ts @@ -4,18 +4,46 @@ */ import { Client } from '@opensearch-project/opensearch'; -import { Logger, OpenSearchClient, SavedObjectsClientContract } from 'src/core/server'; +import LRUCache from 'lru-cache'; +import { Logger, SavedObjectsClientContract } from 'src/core/server'; +import { DataSourcePluginConfigType } from '../config'; import { DataSourceClient } from './client'; import { IDataSourceService } from './types'; export class DataSourceService implements IDataSourceService { - private openSearchClientsPool: Map; - constructor() { - this.openSearchClientsPool = new Map(); + private openSearchClientPool?: LRUCache; + private isClosed = false; + + constructor(private logger: Logger, private config: DataSourcePluginConfigType) {} + + public setup() { + const logger = this.logger; + const { size } = this.config.clientPool; + + this.openSearchClientPool = new LRUCache({ + max: size, + maxAge: 15 * 60 * 1000, // by default, TCP connection times out in 15 minutes + + async dispose(endpoint, client) { + try { + await client.close(); + } catch (error: any) { + // log and do nothing since we are anyways evicting the client object from cache + logger.warn( + `Error closing OpenSearch client when removing from client pool: ${error.message}` + ); + } + }, + }); + this.logger.info(`Created data source client pool of size ${size}`); } - // TODO: placeholders, need implement when adding global config - isEnabled(): boolean { - throw new Error('Method not implemented.'); + + public getCachedClient(endpoint: string) { + return this.openSearchClientPool!.get(endpoint); + } + + public addClientToPool(endpoint: string, client: Client) { + this.openSearchClientPool!.set(endpoint, client); } getDataSourceClient(logger: Logger, savedObjectClient: SavedObjectsClientContract) { @@ -23,14 +51,16 @@ export class DataSourceService implements IDataSourceService { logger, dataSourceService: this, scopedSavedObjectsClient: savedObjectClient, + config: this.config, }); } - // TODO: placeholders, need implement client pooling strategy - addOpenSearchClient() {} - getOpenSearchClient(): OpenSearchClient { - throw new Error('Method not implemented.'); - } - // TODO: close all data source clients in the clients pool - stop() {} + // close all data source clients in the pool + async stop() { + if (this.isClosed) { + return; + } + this.isClosed = true; + Promise.all(this.openSearchClientPool!.values().map((client) => client.close())); + } } diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts index 1337f182d290..fb59d766eb76 100644 --- a/src/plugins/data_source/server/plugin.ts +++ b/src/plugins/data_source/server/plugin.ts @@ -4,7 +4,6 @@ */ import { first } from 'rxjs/operators'; - import { dataSource, credential, CredentialSavedObjectsClientWrapper } from './saved_objects'; import { DataSourcePluginConfigType } from '../config'; import { @@ -16,9 +15,7 @@ import { } from '../../../../src/core/server'; import { DataSourceService } from './data_source_service'; import { createDataSourceRouteHandlerContext } from './data_source_route_handler_context'; - import { DataSourcePluginSetup, DataSourcePluginStart } from './types'; - import { CryptographyClient } from './cryptography'; export class DataSourcePlugin implements Plugin { @@ -38,9 +35,11 @@ export class DataSourcePlugin implements Plugin(); + const config: DataSourcePluginConfigType = await config$.pipe(first()).toPromise(); + + // Fetch configs used to create credential saved objects client wrapper + const { wrappingKeyName, wrappingKeyNamespace, wrappingKey } = config.encryption; // Create credential saved objects client wrapper const credentialSavedObjectsClientWrapper = new CredentialSavedObjectsClientWrapper( @@ -53,7 +52,9 @@ export class DataSourcePlugin implements Plugin { + // const client = await context.dataSources.getOpenSearchClient('37df1970-b6b0-11ec-a339-c18008b701cd'); + const client = await context.data_source.opensearch.getClient('aaa'); + return response.ok(); + } + ); + return {}; } diff --git a/src/plugins/data_source/server/types.ts b/src/plugins/data_source/server/types.ts index e3cb57963091..6995cb206599 100644 --- a/src/plugins/data_source/server/types.ts +++ b/src/plugins/data_source/server/types.ts @@ -7,13 +7,10 @@ import { Logger, OpenSearchClient, SavedObjectsClientContract } from 'src/core/s import { DataSourceClient } from './client'; export interface IDataSourceService { - isEnabled(): boolean; getDataSourceClient( logger: Logger, savedObjectClient: SavedObjectsClientContract ): DataSourceClient; - addOpenSearchClient(): void; - getOpenSearchClient(): OpenSearchClient; stop(): void; } export interface DataSourcePluginRequestContext { From c21819484298d843cc6219572629d20303d0a2e3 Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Tue, 16 Aug 2022 17:11:10 -0700 Subject: [PATCH 6/7] update interface Signed-off-by: Zhongnan Su --- .../server/client/data_source_client.ts | 157 -------------- .../server/client_pool/client_pool.ts | 83 ++++++++ .../server/{client => client_pool}/index.ts | 2 +- .../data_source_route_handler_context.ts | 51 ----- .../data_source/server/data_source_service.ts | 201 +++++++++++++----- src/plugins/data_source/server/plugin.ts | 58 ++--- src/plugins/data_source/server/types.ts | 10 +- 7 files changed, 272 insertions(+), 290 deletions(-) delete mode 100644 src/plugins/data_source/server/client/data_source_client.ts create mode 100644 src/plugins/data_source/server/client_pool/client_pool.ts rename src/plugins/data_source/server/{client => client_pool}/index.ts (52%) delete mode 100644 src/plugins/data_source/server/data_source_route_handler_context.ts diff --git a/src/plugins/data_source/server/client/data_source_client.ts b/src/plugins/data_source/server/client/data_source_client.ts deleted file mode 100644 index 13ad71138ae5..000000000000 --- a/src/plugins/data_source/server/client/data_source_client.ts +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { Client } from '@opensearch-project/opensearch'; -import { - Logger, - OpenSearchClient, - SavedObject, - SavedObjectsClientContract, - SavedObjectsErrorHelpers, -} from 'src/core/server'; -import { CREDENTIAL_SAVED_OBJECT_TYPE, DATA_SOURCE_SAVED_OBJECT_TYPE } from '../../common'; -import { - CredentialMaterials, - CredentialSavedObjectAttributes, -} from '../../common/credentials/types'; -import { DataSourceAttributes } from '../../common/data_sources'; -import { DataSourcePluginConfigType } from '../../config'; -import { DataSourceService } from '../data_source_service'; - -/** - * Represents an OpenSearch cluster API client created by the platform. - * It allows to call API on behalf of the user(credential) associated to "data source" - * - * @public - **/ -export interface IDataSourceClient { - /** - * Creates a {@link OpenSearchClient } bound to given data source - */ - asDataSource: (dataSourceId: string) => Promise; -} - -interface DataSourceClientCtorParams { - dataSourceService: DataSourceService; - logger: Logger; - scopedSavedObjectsClient: SavedObjectsClientContract; - config: DataSourcePluginConfigType; -} -export class DataSourceClient implements IDataSourceClient { - private dataSourceService: DataSourceService; - private log: Logger; - // scoped saved object client to fetch save object on behalf of user - private scopedSavedObjectClient: SavedObjectsClientContract; - private config: DataSourcePluginConfigType; - - constructor(ctorParams: DataSourceClientCtorParams) { - this.dataSourceService = ctorParams.dataSourceService; - this.log = ctorParams.logger; - this.scopedSavedObjectClient = ctorParams.scopedSavedObjectsClient; - this.config = ctorParams.config; - } - - async asDataSource(dataSourceId: string) { - const dataSource = await this.getDataSource(dataSourceId); - const rootClient = this.getRootClient(dataSource.attributes, this.config); - const credential = await this.getCredential(dataSource.references[0].id); // assuming there is 1 and only 1 credential for each data source - - return this.getQueryClient(rootClient, credential.attributes, dataSource.attributes.withAuth); - } - - private async getDataSource(dataSourceId: string): Promise> { - try { - const dataSource = await this.scopedSavedObjectClient.get( - DATA_SOURCE_SAVED_OBJECT_TYPE, - dataSourceId - ); - return dataSource; - } catch (error: any) { - // it will cause 500 error when failed to get saved objects, need to handle such error gracefully - throw SavedObjectsErrorHelpers.createBadRequestError(error.message); - } - } - - private async getCredential( - credentialId: string - ): Promise> { - try { - const dataSource = await this.scopedSavedObjectClient.get( - CREDENTIAL_SAVED_OBJECT_TYPE, - credentialId - ); - return dataSource; - } catch (error: any) { - // it will cause 500 error when failed to get saved objects, need to handle such error gracefully - throw SavedObjectsErrorHelpers.createBadRequestError(error.message); - } - } - - /** - * Create a child client object with given auth info. - * - * @param rootClient root client for the connection with given data source endpoint. - * @param credentialAttr credential saved object attribute. - * @returns child client. - */ - private getQueryClient( - rootClient: Client, - credentialAttr: CredentialSavedObjectAttributes, - withAuth = false - ): Client { - if (withAuth) { - return this.getBasicAuthClient(rootClient, credentialAttr.credentialMaterials); - } else { - return rootClient.child(); - } - } - - /** - * Gets a root client object of the OpenSearch endpoint. - * Will attempt to get from cache, if cache miss, create a new one and load into cache. - * - * @param dataSourceAttr data source saved objects attributes. - * @returns OpenSearch client for the given data source endpoint. - */ - private getRootClient( - dataSourceAttr: DataSourceAttributes, - config: DataSourcePluginConfigType - ): Client { - const endpoint = dataSourceAttr.endpoint; - const cachedClient = this.dataSourceService.getCachedClient(endpoint); - - if (cachedClient) { - return cachedClient; - } else { - const client = this.configureDataSourceClient(config, endpoint); - - this.dataSourceService.addClientToPool(endpoint, client); - return client; - } - } - - private getBasicAuthClient(rootClient: Client, credential: CredentialMaterials): Client { - const { username, password } = credential.credentialMaterialsContent; - return rootClient.child({ - auth: { - username, - password, - }, - }); - } - - // TODO: will use client configs, that comes from a merge result of user config and defaults - private configureDataSourceClient(config: DataSourcePluginConfigType, endpoint: string) { - const client = new Client({ - node: endpoint, - ssl: { - requestCert: true, - rejectUnauthorized: true, - }, - }); - - return client; - } -} diff --git a/src/plugins/data_source/server/client_pool/client_pool.ts b/src/plugins/data_source/server/client_pool/client_pool.ts new file mode 100644 index 000000000000..be9f3394614a --- /dev/null +++ b/src/plugins/data_source/server/client_pool/client_pool.ts @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { first } from 'rxjs/operators'; +import { Client } from '@opensearch-project/opensearch'; +import LRUCache from 'lru-cache'; +import { Logger, PluginInitializerContext } from 'src/core/server'; +import { DataSourcePluginConfigType } from '../../config'; + +export interface OpenSearchClientPoolSetup { + getClientFromPool: (id: string) => Client | undefined; + addClientToPool: (endpoint: string, client: Client) => void; +} + +/** + * OpenSearch client pool. + * + * This client pool uses an LRU cache to manage OpenSearch Js client objects. + * It reuse TPC connections for each OpenSearch endpoint. + */ +export class OpenSearchClientPool { + // LRU cache + // key: data source endpoint url + // value: OpenSearch client object + private cache?: LRUCache; + private isClosed = false; + + constructor( + private logger: Logger, + private initializerContext: PluginInitializerContext + ) {} + + public async setup() { + const config$ = this.initializerContext.config.create(); + const config: DataSourcePluginConfigType = await config$.pipe(first()).toPromise(); + + const logger = this.logger; + const { size } = config.clientPool; + + this.cache = new LRUCache({ + max: size, + maxAge: 15 * 60 * 1000, // by default, TCP connection times out in 15 minutes + + async dispose(endpoint, client) { + try { + await client.close(); + } catch (error: any) { + // log and do nothing since we are anyways evicting the client object from cache + logger.warn( + `Error closing OpenSearch client when removing from client pool: ${error.message}` + ); + } + }, + }); + this.logger.info(`Created data source client pool of size ${size}`); + + const getClientFromPool = (endpoint: string) => { + return this.cache!.get(endpoint); + }; + + const addClientToPool = (endpoint: string, client: Client) => { + this.cache!.set(endpoint, client); + }; + + return { + getClientFromPool, + addClientToPool, + }; + } + + start() {} + + // close all clients in the pool + async stop() { + if (this.isClosed) { + return; + } + this.isClosed = true; + Promise.all(this.cache!.values().map((client) => client.close())); + } +} diff --git a/src/plugins/data_source/server/client/index.ts b/src/plugins/data_source/server/client_pool/index.ts similarity index 52% rename from src/plugins/data_source/server/client/index.ts rename to src/plugins/data_source/server/client_pool/index.ts index ef1b392c16ca..435291fbef9c 100644 --- a/src/plugins/data_source/server/client/index.ts +++ b/src/plugins/data_source/server/client_pool/index.ts @@ -3,4 +3,4 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { IDataSourceClient, DataSourceClient } from './data_source_client'; +export { OpenSearchClientPool } from './client_pool'; diff --git a/src/plugins/data_source/server/data_source_route_handler_context.ts b/src/plugins/data_source/server/data_source_route_handler_context.ts deleted file mode 100644 index 352564a1b7d7..000000000000 --- a/src/plugins/data_source/server/data_source_route_handler_context.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -// eslint-disable-next-line max-classes-per-file -import { IContextProvider, Logger, RequestHandler } from 'src/core/server'; -import { IDataSourceClient } from './client/data_source_client'; -import { DataSourceService } from './data_source_service'; - -class OpenSearchDataSourceRouteHandlerContext { - constructor(private dataSourceClient: IDataSourceClient, private logger: Logger) {} - - public async getClient(dataSourceId: string) { - try { - const client = await this.dataSourceClient.asDataSource(dataSourceId); - return client; - } catch (error: any) { - // TODO: convert as audit log when integrate with osd auditing - // https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1986 - this.logger.error( - `Fail to get data source client for dataSource id: [${dataSourceId}]. Detail: ${error.messages}` - ); - throw error; - } - } -} - -export class DataSourceRouteHandlerContext { - readonly opensearch: OpenSearchDataSourceRouteHandlerContext; - - constructor(private readonly dataSourceClient: IDataSourceClient, logger: Logger) { - this.opensearch = new OpenSearchDataSourceRouteHandlerContext( - this.dataSourceClient, - logger.get('opensearch') - ); - } -} - -export const createDataSourceRouteHandlerContext = ( - dataSourceService: DataSourceService, - logger: Logger -): IContextProvider, 'data_source'> => { - return async (context, req) => { - const dataSourceClient = dataSourceService!.getDataSourceClient( - logger, - context.core.savedObjects.client - ); - return new DataSourceRouteHandlerContext(dataSourceClient, logger); - }; -}; diff --git a/src/plugins/data_source/server/data_source_service.ts b/src/plugins/data_source/server/data_source_service.ts index 0e129ac46e95..8ad53d86d097 100644 --- a/src/plugins/data_source/server/data_source_service.ts +++ b/src/plugins/data_source/server/data_source_service.ts @@ -3,64 +3,169 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { first } from 'rxjs/operators'; import { Client } from '@opensearch-project/opensearch'; -import LRUCache from 'lru-cache'; -import { Logger, SavedObjectsClientContract } from 'src/core/server'; +import { + Logger, + PluginInitializerContext, + SavedObject, + SavedObjectsClientContract, + SavedObjectsErrorHelpers, +} from '../../../../src/core/server'; import { DataSourcePluginConfigType } from '../config'; -import { DataSourceClient } from './client'; -import { IDataSourceService } from './types'; - -export class DataSourceService implements IDataSourceService { - private openSearchClientPool?: LRUCache; - private isClosed = false; - - constructor(private logger: Logger, private config: DataSourcePluginConfigType) {} - - public setup() { - const logger = this.logger; - const { size } = this.config.clientPool; - - this.openSearchClientPool = new LRUCache({ - max: size, - maxAge: 15 * 60 * 1000, // by default, TCP connection times out in 15 minutes - - async dispose(endpoint, client) { - try { - await client.close(); - } catch (error: any) { - // log and do nothing since we are anyways evicting the client object from cache - logger.warn( - `Error closing OpenSearch client when removing from client pool: ${error.message}` - ); - } - }, - }); - this.logger.info(`Created data source client pool of size ${size}`); +import { OpenSearchClientPool } from './client_pool'; +import { DataSourceAttributes } from '../common/data_sources'; +import { CREDENTIAL_SAVED_OBJECT_TYPE, DATA_SOURCE_SAVED_OBJECT_TYPE } from '../common'; +import { CredentialSavedObjectAttributes } from '../common/credentials/types'; +import { OpenSearchClientPoolSetup } from './client_pool/client_pool'; + +export interface DataSourceServiceSetup { + getDataSourceClient: ( + dataSourceId: string, + // this saved objects client is used to fetch data source on behalf of users, caller should pass scoped saved objects client + savedObjects: SavedObjectsClientContract + ) => Promise; +} +export class DataSourceService { + private readonly openSearchClientPool: OpenSearchClientPool; + + constructor( + private logger: Logger, + private initializerContext: PluginInitializerContext + ) { + this.openSearchClientPool = new OpenSearchClientPool(logger, initializerContext); } - public getCachedClient(endpoint: string) { - return this.openSearchClientPool!.get(endpoint); + async setup() { + const config$ = this.initializerContext.config.create(); + const config: DataSourcePluginConfigType = await config$.pipe(first()).toPromise(); + + const openSearchClientPoolSetup = await this.openSearchClientPool.setup(); + + const getDataSourceClient = async ( + dataSourceId: string, + savedObjects: SavedObjectsClientContract + ): Promise => { + const dataSource = await this.getDataSource(dataSourceId, savedObjects); + const rootClient = this.getRootClient( + dataSource.attributes, + config, + openSearchClientPoolSetup + ); + + return this.getQueryClient(rootClient, dataSource, savedObjects); + }; + + return { getDataSourceClient }; } - public addClientToPool(endpoint: string, client: Client) { - this.openSearchClientPool!.set(endpoint, client); + start() {} + + stop() { + this.openSearchClientPool.stop(); } - getDataSourceClient(logger: Logger, savedObjectClient: SavedObjectsClientContract) { - return new DataSourceClient({ - logger, - dataSourceService: this, - scopedSavedObjectsClient: savedObjectClient, - config: this.config, - }); + private async getDataSource( + dataSourceId: string, + savedObjects: SavedObjectsClientContract + ): Promise> { + try { + const dataSource = await savedObjects.get( + DATA_SOURCE_SAVED_OBJECT_TYPE, + dataSourceId + ); + return dataSource; + } catch (error: any) { + // it will cause 500 error when failed to get saved objects, need to handle such error gracefully + throw SavedObjectsErrorHelpers.createBadRequestError(error.message); + } + } + + private async getCredential( + credentialId: string, + savedObjects: SavedObjectsClientContract + ): Promise> { + try { + const credential = await savedObjects.get( + CREDENTIAL_SAVED_OBJECT_TYPE, + credentialId + ); + return credential; + } catch (error: any) { + // it will cause 500 error when failed to get saved objects, need to handle such error gracefully + throw SavedObjectsErrorHelpers.createBadRequestError(error.message); + } } - // close all data source clients in the pool - async stop() { - if (this.isClosed) { - return; + /** + * Create a child client object with given auth info. + * + * @param rootClient root client for the connection with given data source endpoint. + * @param dataSource data source saved object + * @param savedObjects scoped saved object client + * @returns child client. + */ + private async getQueryClient( + rootClient: Client, + dataSource: SavedObject, + savedObjects: SavedObjectsClientContract + ): Promise { + if (dataSource.attributes.noAuth) { + return rootClient.child(); + } else { + const credential = await this.getCredential(dataSource.references[0].id, savedObjects); + return this.getBasicAuthClient(rootClient, credential.attributes); } - this.isClosed = true; - Promise.all(this.openSearchClientPool!.values().map((client) => client.close())); + } + + /** + * Gets a root client object of the OpenSearch endpoint. + * Will attempt to get from cache, if cache miss, create a new one and load into cache. + * + * @param dataSourceAttr data source saved objects attributes. + * @returns OpenSearch client for the given data source endpoint. + */ + private getRootClient( + dataSourceAttr: DataSourceAttributes, + config: DataSourcePluginConfigType, + { getClientFromPool, addClientToPool }: OpenSearchClientPoolSetup + ): Client { + const endpoint = dataSourceAttr.endpoint; + const cachedClient = getClientFromPool(endpoint); + + if (cachedClient) { + return cachedClient; + } else { + const client = this.configureDataSourceClient(config, endpoint); + + addClientToPool(endpoint, client); + return client; + } + } + + private getBasicAuthClient( + rootClient: Client, + credentialAttr: CredentialSavedObjectAttributes + ): Client { + const { username, password } = credentialAttr.credentialMaterials.credentialMaterialsContent; + return rootClient.child({ + auth: { + username, + password, + }, + }); + } + + // TODO: will use client configs, that comes from a merge result of user config and defaults + private configureDataSourceClient(config: DataSourcePluginConfigType, endpoint: string) { + const client = new Client({ + node: endpoint, + ssl: { + requestCert: true, + rejectUnauthorized: true, + }, + }); + + return client; } } diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts index fb59d766eb76..35615f5fcb1f 100644 --- a/src/plugins/data_source/server/plugin.ts +++ b/src/plugins/data_source/server/plugin.ts @@ -12,18 +12,20 @@ import { CoreStart, Plugin, Logger, + IContextProvider, + RequestHandler, } from '../../../../src/core/server'; -import { DataSourceService } from './data_source_service'; -import { createDataSourceRouteHandlerContext } from './data_source_route_handler_context'; +import { DataSourceService, DataSourceServiceSetup } from './data_source_service'; import { DataSourcePluginSetup, DataSourcePluginStart } from './types'; import { CryptographyClient } from './cryptography'; export class DataSourcePlugin implements Plugin { private readonly logger: Logger; - private dataSourceService?: DataSourceService; + private readonly dataSourceService: DataSourceService; constructor(private initializerContext: PluginInitializerContext) { - this.logger = this.initializerContext.logger.get(); + this.logger = this.initializerContext.logger.get('dataSource'); + this.dataSourceService = new DataSourceService(this.logger, this.initializerContext); } public async setup(core: CoreSetup) { @@ -53,29 +55,12 @@ export class DataSourcePlugin implements Plugin { - // const client = await context.dataSources.getOpenSearchClient('37df1970-b6b0-11ec-a339-c18008b701cd'); - const client = await context.data_source.opensearch.getClient('aaa'); - return response.ok(); - } + this.createDataSourceRouteHandlerContext(dataSourceService, this.logger) ); return {}; @@ -89,4 +74,29 @@ export class DataSourcePlugin implements Plugin, 'data_source'> => { + return (context, req) => { + return { + opensearch: { + getClient: (dataSourceId: string) => { + try { + return dataSourceService.getDataSourceClient( + dataSourceId, + context.core.savedObjects.client + ); + } catch (error: any) { + logger.error( + `Fail to get data source client for dataSourceId: [${dataSourceId}]. Detail: ${error.messages}` + ); + throw error; + } + }, + }, + }; + }; + }; } diff --git a/src/plugins/data_source/server/types.ts b/src/plugins/data_source/server/types.ts index 6995cb206599..f8ffdc1598d1 100644 --- a/src/plugins/data_source/server/types.ts +++ b/src/plugins/data_source/server/types.ts @@ -3,16 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { Logger, OpenSearchClient, SavedObjectsClientContract } from 'src/core/server'; -import { DataSourceClient } from './client'; +import { OpenSearchClient } from 'src/core/server'; -export interface IDataSourceService { - getDataSourceClient( - logger: Logger, - savedObjectClient: SavedObjectsClientContract - ): DataSourceClient; - stop(): void; -} export interface DataSourcePluginRequestContext { opensearch: { getClient: (dataSourceId: string) => Promise; From b705be082b2499cd51cce5b12f5bd57c74c0f9d9 Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Wed, 17 Aug 2022 04:58:02 -0700 Subject: [PATCH 7/7] add tests Signed-off-by: Zhongnan Su --- .../server/client/client_config.test.ts | 29 ++++ .../server/client/client_config.ts | 29 ++++ .../server/client/client_pool.test.ts | 39 +++++ .../{client_pool => client}/client_pool.ts | 15 +- .../client/configure_client.test.mocks.ts | 18 +++ .../server/client/configure_client.test.ts | 135 ++++++++++++++++ .../server/client/configure_client.ts | 124 +++++++++++++++ .../server/{client_pool => client}/index.ts | 1 + .../server/data_source_service.test.ts | 38 +++++ .../data_source/server/data_source_service.ts | 150 ++---------------- src/plugins/data_source/server/plugin.ts | 8 +- src/plugins/data_source/server/types.ts | 2 +- 12 files changed, 435 insertions(+), 153 deletions(-) create mode 100644 src/plugins/data_source/server/client/client_config.test.ts create mode 100644 src/plugins/data_source/server/client/client_config.ts create mode 100644 src/plugins/data_source/server/client/client_pool.test.ts rename src/plugins/data_source/server/{client_pool => client}/client_pool.ts (78%) create mode 100644 src/plugins/data_source/server/client/configure_client.test.mocks.ts create mode 100644 src/plugins/data_source/server/client/configure_client.test.ts create mode 100644 src/plugins/data_source/server/client/configure_client.ts rename src/plugins/data_source/server/{client_pool => client}/index.ts (71%) create mode 100644 src/plugins/data_source/server/data_source_service.test.ts diff --git a/src/plugins/data_source/server/client/client_config.test.ts b/src/plugins/data_source/server/client/client_config.test.ts new file mode 100644 index 000000000000..39a3607ccba8 --- /dev/null +++ b/src/plugins/data_source/server/client/client_config.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import { DataSourcePluginConfigType } from '../../config'; +import { parseClientOptions } from './client_config'; + +const TEST_DATA_SOURCE_ENDPOINT = 'http://datasource.com'; + +const config = { + enabled: true, + clientPool: { + size: 5, + }, +} as DataSourcePluginConfigType; + +describe('parseClientOptions', () => { + test('include the ssl client configs as defaults', () => { + expect(parseClientOptions(config, TEST_DATA_SOURCE_ENDPOINT)).toEqual( + expect.objectContaining({ + node: TEST_DATA_SOURCE_ENDPOINT, + ssl: { + requestCert: true, + rejectUnauthorized: true, + }, + }) + ); + }); +}); diff --git a/src/plugins/data_source/server/client/client_config.ts b/src/plugins/data_source/server/client/client_config.ts new file mode 100644 index 000000000000..5973e5a0813f --- /dev/null +++ b/src/plugins/data_source/server/client/client_config.ts @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { ClientOptions } from '@opensearch-project/opensearch'; +import { DataSourcePluginConfigType } from '../../config'; + +/** + * Parse the client options from given data source config and endpoint + * + * @param config The config to generate the client options from. + * @param endpoint endpoint url of data source + */ +export function parseClientOptions( + // TODO: will use client configs, that comes from a merge result of user config and default opensearch client config, + config: DataSourcePluginConfigType, + endpoint: string +): ClientOptions { + const clientOptions: ClientOptions = { + node: endpoint, + ssl: { + requestCert: true, + rejectUnauthorized: true, + }, + }; + + return clientOptions; +} diff --git a/src/plugins/data_source/server/client/client_pool.test.ts b/src/plugins/data_source/server/client/client_pool.test.ts new file mode 100644 index 000000000000..92320e9610ad --- /dev/null +++ b/src/plugins/data_source/server/client/client_pool.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { loggingSystemMock } from '../../../../core/server/mocks'; +import { DataSourcePluginConfigType } from '../../config'; +import { OpenSearchClientPool } from './client_pool'; + +const logger = loggingSystemMock.create(); + +describe('Client Pool', () => { + let service: OpenSearchClientPool; + let config: DataSourcePluginConfigType; + + beforeEach(() => { + const mockLogger = logger.get('dataSource'); + service = new OpenSearchClientPool(mockLogger); + config = { + enabled: true, + clientPool: { + size: 5, + }, + } as DataSourcePluginConfigType; + }); + + afterEach(() => { + service.stop(); + jest.clearAllMocks(); + }); + + describe('setup()', () => { + test('exposes proper contract', async () => { + const setup = await service.setup(config); + expect(setup).toHaveProperty('getClientFromPool'); + expect(setup).toHaveProperty('addClientToPool'); + }); + }); +}); diff --git a/src/plugins/data_source/server/client_pool/client_pool.ts b/src/plugins/data_source/server/client/client_pool.ts similarity index 78% rename from src/plugins/data_source/server/client_pool/client_pool.ts rename to src/plugins/data_source/server/client/client_pool.ts index be9f3394614a..fe6f20e51ae4 100644 --- a/src/plugins/data_source/server/client_pool/client_pool.ts +++ b/src/plugins/data_source/server/client/client_pool.ts @@ -3,10 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { first } from 'rxjs/operators'; import { Client } from '@opensearch-project/opensearch'; import LRUCache from 'lru-cache'; -import { Logger, PluginInitializerContext } from 'src/core/server'; +import { Logger } from 'src/core/server'; import { DataSourcePluginConfigType } from '../../config'; export interface OpenSearchClientPoolSetup { @@ -27,15 +26,9 @@ export class OpenSearchClientPool { private cache?: LRUCache; private isClosed = false; - constructor( - private logger: Logger, - private initializerContext: PluginInitializerContext - ) {} - - public async setup() { - const config$ = this.initializerContext.config.create(); - const config: DataSourcePluginConfigType = await config$.pipe(first()).toPromise(); + constructor(private logger: Logger) {} + public async setup(config: DataSourcePluginConfigType) { const logger = this.logger; const { size } = config.clientPool; @@ -78,6 +71,6 @@ export class OpenSearchClientPool { return; } this.isClosed = true; - Promise.all(this.cache!.values().map((client) => client.close())); + await Promise.all(this.cache!.values().map((client) => client.close())); } } diff --git a/src/plugins/data_source/server/client/configure_client.test.mocks.ts b/src/plugins/data_source/server/client/configure_client.test.mocks.ts new file mode 100644 index 000000000000..38a585ff2020 --- /dev/null +++ b/src/plugins/data_source/server/client/configure_client.test.mocks.ts @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const ClientMock = jest.fn(); +jest.doMock('@opensearch-project/opensearch', () => { + const actual = jest.requireActual('@opensearch-project/opensearch'); + return { + ...actual, + Client: ClientMock, + }; +}); + +export const parseClientOptionsMock = jest.fn(); +jest.doMock('./client_config', () => ({ + parseClientOptions: parseClientOptionsMock, +})); diff --git a/src/plugins/data_source/server/client/configure_client.test.ts b/src/plugins/data_source/server/client/configure_client.test.ts new file mode 100644 index 000000000000..70dd078d4fdb --- /dev/null +++ b/src/plugins/data_source/server/client/configure_client.test.ts @@ -0,0 +1,135 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract } from '../../../../core/server'; +import { loggingSystemMock, savedObjectsClientMock } from '../../../../core/server/mocks'; +import { DATA_SOURCE_SAVED_OBJECT_TYPE, CREDENTIAL_SAVED_OBJECT_TYPE } from '../../common'; +import { + CredentialMaterialsType, + CredentialSavedObjectAttributes, +} from '../../common/credentials/types'; +import { DataSourceAttributes } from '../../common/data_sources'; +import { DataSourcePluginConfigType } from '../../config'; +import { ClientMock, parseClientOptionsMock } from './configure_client.test.mocks'; +import { OpenSearchClientPoolSetup } from './client_pool'; +import { configureClient } from './configure_client'; +import { ClientOptions } from '@opensearch-project/opensearch'; +// eslint-disable-next-line @osd/eslint/no-restricted-paths +import { opensearchClientMock } from '../../../../core/server/opensearch/client/mocks'; + +const DATA_SOURCE_ID = 'a54b76ec86771ee865a0f74a305dfff8'; +const CREDENETIAL_ID = 'a54dsaadasfasfwe22d23d23d2453df3'; + +describe('configureClient', () => { + let logger: ReturnType; + let config: DataSourcePluginConfigType; + let savedObjectsMock: jest.Mocked; + let clientPoolSetup: OpenSearchClientPoolSetup; + let clientOptions: ClientOptions; + let dataSourceAttr: DataSourceAttributes; + let dsClient: ReturnType; + + beforeEach(() => { + dsClient = opensearchClientMock.createInternalClient(); + logger = loggingSystemMock.createLogger(); + savedObjectsMock = savedObjectsClientMock.create(); + + config = { + enabled: true, + clientPool: { + size: 5, + }, + } as DataSourcePluginConfigType; + clientOptions = { + nodes: 'http://localhost', + ssl: { + requestCert: true, + rejectUnauthorized: true, + }, + } as ClientOptions; + dataSourceAttr = { + title: 'title', + endpoint: 'http://localhost', + noAuth: false, + } as DataSourceAttributes; + + clientPoolSetup = { + getClientFromPool: jest.fn(), + addClientToPool: jest.fn(), + }; + + const crendentialAttr = { + title: 'cred', + credentialMaterials: { + credentialMaterialsType: CredentialMaterialsType.UsernamePasswordType, + credentialMaterialsContent: { + username: 'username', + password: 'password', + }, + }, + } as CredentialSavedObjectAttributes; + + savedObjectsMock.get + .mockResolvedValueOnce({ + id: DATA_SOURCE_ID, + type: DATA_SOURCE_SAVED_OBJECT_TYPE, + attributes: dataSourceAttr, + references: [{ name: 'user', type: CREDENTIAL_SAVED_OBJECT_TYPE, id: CREDENETIAL_ID }], + }) + .mockResolvedValueOnce({ + id: CREDENETIAL_ID, + type: CREDENTIAL_SAVED_OBJECT_TYPE, + attributes: crendentialAttr, + references: [], + }); + + ClientMock.mockImplementation(() => { + return dsClient; + }); + }); + + afterEach(() => { + ClientMock.mockReset(); + }); + // TODO: mark as skip until we fix the issue of mocking "@opensearch-project/opensearch" + test('configure client with noAuth == true, will call new Client() to create client', async () => { + savedObjectsMock.get.mockReset().mockResolvedValueOnce({ + id: DATA_SOURCE_ID, + type: DATA_SOURCE_SAVED_OBJECT_TYPE, + attributes: { ...dataSourceAttr, noAuth: true }, + references: [], + }); + + parseClientOptionsMock.mockReturnValue(clientOptions); + + const client = await configureClient( + DATA_SOURCE_ID, + savedObjectsMock, + clientPoolSetup, + config, + logger + ); + + expect(parseClientOptionsMock).toHaveBeenCalled(); + expect(ClientMock).toHaveBeenCalledTimes(1); + expect(ClientMock).toHaveBeenCalledWith(clientOptions); + expect(savedObjectsMock.get).toHaveBeenCalledTimes(1); + expect(client).toBe(dsClient.child.mock.results[0].value); + }); + + test('configure client with noAuth == false, will first call new Client()', async () => { + const client = await configureClient( + DATA_SOURCE_ID, + savedObjectsMock, + clientPoolSetup, + config, + logger + ); + + expect(ClientMock).toHaveBeenCalledTimes(1); + expect(savedObjectsMock.get).toHaveBeenCalledTimes(2); + expect(client).toBe(dsClient.child.mock.results[0].value); + }); +}); diff --git a/src/plugins/data_source/server/client/configure_client.ts b/src/plugins/data_source/server/client/configure_client.ts new file mode 100644 index 000000000000..15a824dabae2 --- /dev/null +++ b/src/plugins/data_source/server/client/configure_client.ts @@ -0,0 +1,124 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Client } from '@opensearch-project/opensearch'; +import { + Logger, + SavedObject, + SavedObjectsClientContract, + SavedObjectsErrorHelpers, +} from '../../../../../src/core/server'; +import { DATA_SOURCE_SAVED_OBJECT_TYPE, CREDENTIAL_SAVED_OBJECT_TYPE } from '../../common'; +import { CredentialSavedObjectAttributes } from '../../common/credentials/types'; +import { DataSourceAttributes } from '../../common/data_sources'; +import { DataSourcePluginConfigType } from '../../config'; +import { parseClientOptions } from './client_config'; +import { OpenSearchClientPoolSetup } from './client_pool'; + +export const configureClient = async ( + dataSourceId: string, + savedObjects: SavedObjectsClientContract, + openSearchClientPoolSetup: OpenSearchClientPoolSetup, + config: DataSourcePluginConfigType, + logger: Logger +): Promise => { + const dataSource = await getDataSource(dataSourceId, savedObjects); + const rootClient = getRootClient(dataSource.attributes, config, openSearchClientPoolSetup); + + return getQueryClient(rootClient, dataSource, savedObjects); +}; + +export const getDataSource = async ( + dataSourceId: string, + savedObjects: SavedObjectsClientContract +): Promise> => { + try { + const dataSource = await savedObjects.get( + DATA_SOURCE_SAVED_OBJECT_TYPE, + dataSourceId + ); + return dataSource; + } catch (error: any) { + // it will cause 500 error when failed to get saved objects, need to handle such error gracefully + throw SavedObjectsErrorHelpers.createBadRequestError(error.message); + } +}; + +export const getCredential = async ( + credentialId: string, + savedObjects: SavedObjectsClientContract +): Promise> => { + try { + const credential = await savedObjects.get( + CREDENTIAL_SAVED_OBJECT_TYPE, + credentialId + ); + return credential; + } catch (error: any) { + // it will cause 500 error when failed to get saved objects, need to handle such error gracefully + throw SavedObjectsErrorHelpers.createBadRequestError(error.message); + } +}; + +/** + * Create a child client object with given auth info. + * + * @param rootClient root client for the connection with given data source endpoint. + * @param dataSource data source saved object + * @param savedObjects scoped saved object client + * @returns child client. + */ +const getQueryClient = async ( + rootClient: Client, + dataSource: SavedObject, + savedObjects: SavedObjectsClientContract +): Promise => { + if (dataSource.attributes.noAuth) { + return rootClient.child(); + } else { + const credential = await getCredential(dataSource.references[0].id, savedObjects); + return getBasicAuthClient(rootClient, credential.attributes); + } +}; + +/** + * Gets a root client object of the OpenSearch endpoint. + * Will attempt to get from cache, if cache miss, create a new one and load into cache. + * + * @param dataSourceAttr data source saved objects attributes. + * @param config data source config + * @returns OpenSearch client for the given data source endpoint. + */ +const getRootClient = ( + dataSourceAttr: DataSourceAttributes, + config: DataSourcePluginConfigType, + { getClientFromPool, addClientToPool }: OpenSearchClientPoolSetup +): Client => { + const endpoint = dataSourceAttr.endpoint; + const cachedClient = getClientFromPool(endpoint); + if (cachedClient) { + return cachedClient; + } else { + const clientOptions = parseClientOptions(config, endpoint); + + const client = new Client(clientOptions); + addClientToPool(endpoint, client); + + return client; + } +}; + +const getBasicAuthClient = ( + rootClient: Client, + credentialAttr: CredentialSavedObjectAttributes +): Client => { + const { username, password } = credentialAttr.credentialMaterials.credentialMaterialsContent; + return rootClient.child({ + auth: { + username, + password, + }, + }); +}; diff --git a/src/plugins/data_source/server/client_pool/index.ts b/src/plugins/data_source/server/client/index.ts similarity index 71% rename from src/plugins/data_source/server/client_pool/index.ts rename to src/plugins/data_source/server/client/index.ts index 435291fbef9c..8adc96115b91 100644 --- a/src/plugins/data_source/server/client_pool/index.ts +++ b/src/plugins/data_source/server/client/index.ts @@ -4,3 +4,4 @@ */ export { OpenSearchClientPool } from './client_pool'; +export { configureClient } from './configure_client'; diff --git a/src/plugins/data_source/server/data_source_service.test.ts b/src/plugins/data_source/server/data_source_service.test.ts new file mode 100644 index 000000000000..53dfb6f273eb --- /dev/null +++ b/src/plugins/data_source/server/data_source_service.test.ts @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { loggingSystemMock } from '../../../core/server/mocks'; +import { DataSourcePluginConfigType } from '../config'; +import { DataSourceService } from './data_source_service'; + +const logger = loggingSystemMock.create(); + +describe('Data Source Service', () => { + let service: DataSourceService; + let config: DataSourcePluginConfigType; + + beforeEach(() => { + const mockLogger = logger.get('dataSource'); + service = new DataSourceService(mockLogger); + config = { + enabled: true, + clientPool: { + size: 5, + }, + } as DataSourcePluginConfigType; + }); + + afterEach(() => { + service.stop(); + jest.clearAllMocks(); + }); + + describe('setup()', () => { + test('exposes proper contract', async () => { + const setup = await service.setup(config); + expect(setup).toHaveProperty('getDataSourceClient'); + }); + }); +}); diff --git a/src/plugins/data_source/server/data_source_service.ts b/src/plugins/data_source/server/data_source_service.ts index 8ad53d86d097..5bb25492f596 100644 --- a/src/plugins/data_source/server/data_source_service.ts +++ b/src/plugins/data_source/server/data_source_service.ts @@ -3,57 +3,37 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { first } from 'rxjs/operators'; -import { Client } from '@opensearch-project/opensearch'; -import { - Logger, - PluginInitializerContext, - SavedObject, - SavedObjectsClientContract, - SavedObjectsErrorHelpers, -} from '../../../../src/core/server'; +import { Logger, OpenSearchClient, SavedObjectsClientContract } from '../../../../src/core/server'; import { DataSourcePluginConfigType } from '../config'; -import { OpenSearchClientPool } from './client_pool'; -import { DataSourceAttributes } from '../common/data_sources'; -import { CREDENTIAL_SAVED_OBJECT_TYPE, DATA_SOURCE_SAVED_OBJECT_TYPE } from '../common'; -import { CredentialSavedObjectAttributes } from '../common/credentials/types'; -import { OpenSearchClientPoolSetup } from './client_pool/client_pool'; - +import { OpenSearchClientPool, configureClient } from './client'; export interface DataSourceServiceSetup { getDataSourceClient: ( dataSourceId: string, // this saved objects client is used to fetch data source on behalf of users, caller should pass scoped saved objects client savedObjects: SavedObjectsClientContract - ) => Promise; + ) => Promise; } export class DataSourceService { private readonly openSearchClientPool: OpenSearchClientPool; - constructor( - private logger: Logger, - private initializerContext: PluginInitializerContext - ) { - this.openSearchClientPool = new OpenSearchClientPool(logger, initializerContext); + constructor(private logger: Logger) { + this.openSearchClientPool = new OpenSearchClientPool(logger); } - async setup() { - const config$ = this.initializerContext.config.create(); - const config: DataSourcePluginConfigType = await config$.pipe(first()).toPromise(); - - const openSearchClientPoolSetup = await this.openSearchClientPool.setup(); + async setup(config: DataSourcePluginConfigType) { + const openSearchClientPoolSetup = await this.openSearchClientPool.setup(config); const getDataSourceClient = async ( dataSourceId: string, savedObjects: SavedObjectsClientContract - ): Promise => { - const dataSource = await this.getDataSource(dataSourceId, savedObjects); - const rootClient = this.getRootClient( - dataSource.attributes, + ): Promise => { + return configureClient( + dataSourceId, + savedObjects, + openSearchClientPoolSetup, config, - openSearchClientPoolSetup + this.logger ); - - return this.getQueryClient(rootClient, dataSource, savedObjects); }; return { getDataSourceClient }; @@ -64,108 +44,4 @@ export class DataSourceService { stop() { this.openSearchClientPool.stop(); } - - private async getDataSource( - dataSourceId: string, - savedObjects: SavedObjectsClientContract - ): Promise> { - try { - const dataSource = await savedObjects.get( - DATA_SOURCE_SAVED_OBJECT_TYPE, - dataSourceId - ); - return dataSource; - } catch (error: any) { - // it will cause 500 error when failed to get saved objects, need to handle such error gracefully - throw SavedObjectsErrorHelpers.createBadRequestError(error.message); - } - } - - private async getCredential( - credentialId: string, - savedObjects: SavedObjectsClientContract - ): Promise> { - try { - const credential = await savedObjects.get( - CREDENTIAL_SAVED_OBJECT_TYPE, - credentialId - ); - return credential; - } catch (error: any) { - // it will cause 500 error when failed to get saved objects, need to handle such error gracefully - throw SavedObjectsErrorHelpers.createBadRequestError(error.message); - } - } - - /** - * Create a child client object with given auth info. - * - * @param rootClient root client for the connection with given data source endpoint. - * @param dataSource data source saved object - * @param savedObjects scoped saved object client - * @returns child client. - */ - private async getQueryClient( - rootClient: Client, - dataSource: SavedObject, - savedObjects: SavedObjectsClientContract - ): Promise { - if (dataSource.attributes.noAuth) { - return rootClient.child(); - } else { - const credential = await this.getCredential(dataSource.references[0].id, savedObjects); - return this.getBasicAuthClient(rootClient, credential.attributes); - } - } - - /** - * Gets a root client object of the OpenSearch endpoint. - * Will attempt to get from cache, if cache miss, create a new one and load into cache. - * - * @param dataSourceAttr data source saved objects attributes. - * @returns OpenSearch client for the given data source endpoint. - */ - private getRootClient( - dataSourceAttr: DataSourceAttributes, - config: DataSourcePluginConfigType, - { getClientFromPool, addClientToPool }: OpenSearchClientPoolSetup - ): Client { - const endpoint = dataSourceAttr.endpoint; - const cachedClient = getClientFromPool(endpoint); - - if (cachedClient) { - return cachedClient; - } else { - const client = this.configureDataSourceClient(config, endpoint); - - addClientToPool(endpoint, client); - return client; - } - } - - private getBasicAuthClient( - rootClient: Client, - credentialAttr: CredentialSavedObjectAttributes - ): Client { - const { username, password } = credentialAttr.credentialMaterials.credentialMaterialsContent; - return rootClient.child({ - auth: { - username, - password, - }, - }); - } - - // TODO: will use client configs, that comes from a merge result of user config and defaults - private configureDataSourceClient(config: DataSourcePluginConfigType, endpoint: string) { - const client = new Client({ - node: endpoint, - ssl: { - requestCert: true, - rejectUnauthorized: true, - }, - }); - - return client; - } } diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts index 35615f5fcb1f..82a9ae65e6a9 100644 --- a/src/plugins/data_source/server/plugin.ts +++ b/src/plugins/data_source/server/plugin.ts @@ -25,7 +25,7 @@ export class DataSourcePlugin implements Plugin) { this.logger = this.initializerContext.logger.get('dataSource'); - this.dataSourceService = new DataSourceService(this.logger, this.initializerContext); + this.dataSourceService = new DataSourceService(this.logger); } public async setup(core: CoreSetup) { @@ -55,11 +55,11 @@ export class DataSourcePlugin implements Plugin, 'data_source'> => { + ): IContextProvider, 'dataSource'> => { return (context, req) => { return { opensearch: { diff --git a/src/plugins/data_source/server/types.ts b/src/plugins/data_source/server/types.ts index f8ffdc1598d1..bad309b4b871 100644 --- a/src/plugins/data_source/server/types.ts +++ b/src/plugins/data_source/server/types.ts @@ -12,7 +12,7 @@ export interface DataSourcePluginRequestContext { } declare module 'src/core/server' { interface RequestHandlerContext { - data_source: DataSourcePluginRequestContext; + dataSource: DataSourcePluginRequestContext; } }