[ci] run server integration tests

packages/kbn-dev-utils/src/index.ts

@@ -34,7 +34,7 @@ export {
   KBN_P12_PATH,
   KBN_P12_PASSWORD,
 } from './certs';
-export { KbnClient } from './kbn_client';
+export * from './kbn_client';
 export * from './run';
 export * from './axios';
 export * from './stdio';

packages/kbn-dev-utils/src/kbn_client/index.ts

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-export { KbnClient } from './kbn_client';
+export * from './kbn_client';
 export { uriencode } from './kbn_client_requester';

packages/kbn-dev-utils/src/kbn_client/kbn_client.ts

@@ -18,37 +18,53 @@
  */
 
 import { ToolingLog } from '../tooling_log';
-import { KibanaConfig, KbnClientRequester, ReqOptions } from './kbn_client_requester';
+import { KbnClientRequester, ReqOptions } from './kbn_client_requester';
 import { KbnClientStatus } from './kbn_client_status';
 import { KbnClientPlugins } from './kbn_client_plugins';
 import { KbnClientVersion } from './kbn_client_version';
 import { KbnClientSavedObjects } from './kbn_client_saved_objects';
 import { KbnClientUiSettings, UiSettingValues } from './kbn_client_ui_settings';
 
+export interface KbnClientOptions {
+  url: string;
+  certificateAuthorities?: Buffer[];
+  log: ToolingLog;
+  uiSettingDefaults?: UiSettingValues;
+}
+
 export class KbnClient {
-  private readonly requester = new KbnClientRequester(this.log, this.kibanaConfig);
-  readonly status = new KbnClientStatus(this.requester);
-  readonly plugins = new KbnClientPlugins(this.status);
-  readonly version = new KbnClientVersion(this.status);
-  readonly savedObjects = new KbnClientSavedObjects(this.log, this.requester);
-  readonly uiSettings = new KbnClientUiSettings(this.log, this.requester, this.uiSettingDefaults);
+  readonly status: KbnClientStatus;
+  readonly plugins: KbnClientPlugins;
+  readonly version: KbnClientVersion;
+  readonly savedObjects: KbnClientSavedObjects;
+  readonly uiSettings: KbnClientUiSettings;
+
+  private readonly requester: KbnClientRequester;
+  private readonly log: ToolingLog;
+  private readonly uiSettingDefaults?: UiSettingValues;
 
   /**
    * Basic Kibana server client that implements common behaviors for talking
    * to the Kibana server from dev tooling.
-   *
-   * @param log ToolingLog
-   * @param kibanaUrls Array of kibana server urls to send requests to
-   * @param uiSettingDefaults Map of uiSetting values that will be merged with all uiSetting resets
    */
-  constructor(
-    private readonly log: ToolingLog,
-    private readonly kibanaConfig: KibanaConfig,
-    private readonly uiSettingDefaults?: UiSettingValues
-  ) {
-    if (!kibanaConfig.url) {
-      throw new Error('missing Kibana urls');
+  constructor(options: KbnClientOptions) {
+    if (!options.url) {
+      throw new Error('missing Kibana url');
     }
+    if (!options.log) {
+      throw new Error('missing ToolingLog');
+    }
+
+    this.log = options.log;
+    this.requester = new KbnClientRequester(this.log, {
+      url: options.url,
+      certificateAuthorities: options.certificateAuthorities,
+    });
+    this.status = new KbnClientStatus(this.requester);
+    this.plugins = new KbnClientPlugins(this.status);
+    this.version = new KbnClientVersion(this.status);
+    this.savedObjects = new KbnClientSavedObjects(this.log, this.requester);
+    this.uiSettings = new KbnClientUiSettings(this.log, this.requester, this.uiSettingDefaults);
   }
 
   /**

packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts

@@ -69,31 +69,27 @@ const delay = (ms: number) =>
     setTimeout(resolve, ms);
   });
 
-export interface KibanaConfig {
+interface Options {
   url: string;
-  ssl?: {
-    enabled: boolean;
-    key: string;
-    certificate: string;
-    certificateAuthorities: string;
-  };
+  certificateAuthorities?: Buffer[];
 }
 
 export class KbnClientRequester {
+  private readonly url: string;
   private readonly httpsAgent: Https.Agent | null;
-  constructor(private readonly log: ToolingLog, private readonly kibanaConfig: KibanaConfig) {
+
+  constructor(private readonly log: ToolingLog, options: Options) {
+    this.url = options.url;
     this.httpsAgent =
-      kibanaConfig.ssl && kibanaConfig.ssl.enabled
+      Url.parse(options.url).protocol === 'https:'
         ? new Https.Agent({
-            cert: kibanaConfig.ssl.certificate,
-            key: kibanaConfig.ssl.key,
-            ca: kibanaConfig.ssl.certificateAuthorities,
+            ca: options.certificateAuthorities,
           })
         : null;
   }
 
   private pickUrl() {
-    return this.kibanaConfig.url;
+    return this.url;
   }
 
   public resolveUrl(relativeUrl: string = '/') {
@@ -132,7 +128,7 @@ export class KbnClientRequester {
           errorMessage = `Conflict on GET (path=${options.path}, attempt=${attempt}/${maxAttempts})`;
           this.log.error(errorMessage);
         } else if (requestedRetries || failedToGetResponse) {
-          errorMessage = `[${description}] request failed (attempt=${attempt}/${maxAttempts})`;
+          errorMessage = `[${description}] request failed (attempt=${attempt}/${maxAttempts}): ${error.message}`;
           this.log.error(errorMessage);
         } else {
           throw error;

packages/kbn-es-archiver/src/es_archiver.ts

@@ -49,7 +49,7 @@ export class EsArchiver {
     this.client = client;
     this.dataDir = dataDir;
     this.log = log;
-    this.kbnClient = new KbnClient(log, { url: kibanaUrl });
+    this.kbnClient = new KbnClient({ log, url: kibanaUrl });
   }
 
   /**

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -38,14 +38,7 @@ const urlPartsSchema = () =>
       password: Joi.string(),
       pathname: Joi.string().regex(/^\//, 'start with a /'),
       hash: Joi.string().regex(/^\//, 'start with a /'),
-      ssl: Joi.object()
-        .keys({
-          enabled: Joi.boolean().default(false),
-          certificate: Joi.string().optional(),
-          certificateAuthorities: Joi.string().optional(),
-          key: Joi.string().optional(),
-        })
-        .default(),
+      certificateAuthorities: Joi.array().items(Joi.binary()).optional(),
     })
     .default();
 

test/common/services/kibana_server/kibana_server.ts

@@ -27,9 +27,13 @@ export function KibanaServerProvider({ getService }: FtrProviderContext) {
   const config = getService('config');
   const lifecycle = getService('lifecycle');
   const url = Url.format(config.get('servers.kibana'));
-  const ssl = config.get('servers.kibana').ssl;
   const defaults = config.get('uiSettings.defaults');
-  const kbn = new KbnClient(log, { url, ssl }, defaults);
+  const kbn = new KbnClient({
+    log,
+    url,
+    certificateAuthorities: config.get('servers.kibana.certificateAuthorities'),
+    uiSettingDefaults: defaults,
+  });
 
   if (defaults) {
     lifecycle.beforeTests.add(async () => {

test/functional/page_objects/time_picker.ts

@@ -35,12 +35,18 @@ export type CommonlyUsed =
 
 export function TimePickerProvider({ getService, getPageObjects }: FtrProviderContext) {
   const log = getService('log');
-  const retry = getService('retry');
   const find = getService('find');
   const browser = getService('browser');
   const testSubjects = getService('testSubjects');
   const { header } = getPageObjects(['header']);
   const kibanaServer = getService('kibanaServer');
+  const MenuToggle = getService('MenuToggle');
+
+  const quickSelectTimeMenuToggle = new MenuToggle({
+    name: 'QuickSelectTime Menu',
+    menuTestSubject: 'superDatePickerQuickMenu',
+    toggleButtonTestSubject: 'superDatePickerToggleQuickMenuButton',
+  });
 
   class TimePicker {
     defaultStartTime = 'Sep 19, 2015 @ 06:31:44.000';
@@ -158,30 +164,12 @@ export function TimePickerProvider({ getService, getPageObjects }: FtrProviderCo
       return await find.existsByCssSelector('.euiDatePickerRange--readOnly');
     }
 
-    public async isQuickSelectMenuOpen() {
-      return await testSubjects.exists('superDatePickerQuickMenu');
-    }
-
     public async openQuickSelectTimeMenu() {
-      log.debug('openQuickSelectTimeMenu');
-      const isMenuOpen = await this.isQuickSelectMenuOpen();
-      if (!isMenuOpen) {
-        log.debug('opening quick select menu');
-        await retry.try(async () => {
-          await testSubjects.click('superDatePickerToggleQuickMenuButton');
-        });
-      }
+      await quickSelectTimeMenuToggle.open();
     }
 
     public async closeQuickSelectTimeMenu() {
-      log.debug('closeQuickSelectTimeMenu');
-      const isMenuOpen = await this.isQuickSelectMenuOpen();
-      if (isMenuOpen) {
-        log.debug('closing quick select menu');
-        await retry.try(async () => {
-          await testSubjects.click('superDatePickerToggleQuickMenuButton');
-        });
-      }
+      await quickSelectTimeMenuToggle.close();
     }
 
     public async getRefreshConfig(keepQuickSelectOpen = false) {
@@ -283,10 +271,10 @@ export function TimePickerProvider({ getService, getPageObjects }: FtrProviderCo
     public async pauseAutoRefresh() {
       log.debug('pauseAutoRefresh');
       const refreshConfig = await this.getRefreshConfig(true);
+
       if (!refreshConfig.isPaused) {
         log.debug('pause auto refresh');
         await testSubjects.click('superDatePickerToggleRefreshButton');
-        await this.closeQuickSelectTimeMenu();
       }
 
       await this.closeQuickSelectTimeMenu();

test/functional/services/common/test_subjects.ts

@@ -170,11 +170,27 @@ export function TestSubjectsProvider({ getService }: FtrProviderContext) {
     public async getAttribute(
       selector: string,
       attribute: string,
-      timeout?: number
+      options?:
+        | number
+        | {
+            findTimeout?: number;
+            tryTimeout?: number;
+          }
     ): Promise<string> {
-      return await retry.try(async () => {
-        log.debug(`TestSubjects.getAttribute(${selector}, ${attribute})`);
-        const element = await this.find(selector, timeout);
+      const findTimeout =
+        (typeof options === 'number' ? options : options?.findTimeout) ??
+        config.get('timeouts.find');
+
+      const tryTimeout =
+        (typeof options !== 'number' ? options?.tryTimeout : undefined) ??
+        config.get('timeouts.try');
+
+      log.debug(
+        `TestSubjects.getAttribute(${selector}, ${attribute}, tryTimeout=${tryTimeout}, findTimeout=${findTimeout})`
+      );
+
+      return await retry.tryForTime(tryTimeout, async () => {
+        const element = await this.find(selector, findTimeout);
         return await element.getAttribute(attribute);
       });
     }

test/functional/services/index.ts

@@ -57,6 +57,7 @@ import {
 import { ListingTableProvider } from './listing_table';
 import { SavedQueryManagementComponentProvider } from './saved_query_management_component';
 import { KibanaSupertestProvider } from './supertest';
+import { MenuToggleProvider } from './menu_toggle';
 
 export const services = {
   ...commonServiceProviders,
@@ -93,4 +94,5 @@ export const services = {
   elasticChart: ElasticChartProvider,
   supertest: KibanaSupertestProvider,
   managementMenu: ManagementMenuProvider,
+  MenuToggle: MenuToggleProvider,
 };

test/functional/services/menu_toggle.ts

@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export function MenuToggleProvider({ getService }: FtrProviderContext) {
+  const log = getService('log');
+  const retry = getService('retry');
+  const testSubjects = getService('testSubjects');
+
+  interface Options {
+    name: string;
+    menuTestSubject: string;
+    toggleButtonTestSubject: string;
+  }
+
+  return class MenuToggle {
+    private readonly name: string;
+    private readonly menuTestSubject: string;
+    private readonly toggleButtonTestSubject: string;
+
+    constructor(options: Options) {
+      this.name = options.name;
+      this.menuTestSubject = options.menuTestSubject;
+      this.toggleButtonTestSubject = options.toggleButtonTestSubject;
+    }
+
+    async open() {
+      await this.setState(true);
+    }
+
+    async close() {
+      await this.setState(false);
+    }
+
+    private async setState(expectedState: boolean) {
+      log.debug(
+        `setting menu open state [name=${this.name}] [state=${expectedState ? 'open' : 'closed'}]`
+      );
+
+      await retry.try(async () => {
+        // if the menu is clearly in the expected state already, bail out quickly if so
+        const isOpen = await testSubjects.exists(this.menuTestSubject, { timeout: 500 });
+        if (isOpen === expectedState) {
+          return;
+        }
+
+        // toggle the view state by clicking the button
+        await testSubjects.click(this.toggleButtonTestSubject);
+
+        if (expectedState === true) {
+          // wait for up to 10 seconds for the menu to show up, otherwise fail and retry
+          await testSubjects.existOrFail(this.menuTestSubject, { timeout: 10000 });
+        } else {
+          // wait for the form to hide, otherwise fail and retry
+          await testSubjects.waitForDeleted(this.menuTestSubject);
+        }
+      });
+    }
+  };
+}

test/scripts/server_integration.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source test/scripts/jenkins_test_setup_oss.sh
+
+yarn run grunt run:serverIntegrationTests

test/server_integration/http/ssl/config.js

@@ -23,20 +23,22 @@ import { createKibanaSupertestProvider } from '../../services';
 
 export default async function ({ readConfigFile }) {
   const httpConfig = await readConfigFile(require.resolve('../../config'));
+  const certificateAuthorities = [readFileSync(CA_CERT_PATH)];
 
   return {
     testFiles: [require.resolve('./')],
     services: {
       ...httpConfig.get('services'),
       supertest: createKibanaSupertestProvider({
-        certificateAuthorities: [readFileSync(CA_CERT_PATH)],
+        certificateAuthorities,
       }),
     },
     servers: {
       ...httpConfig.get('servers'),
       kibana: {
         ...httpConfig.get('servers.kibana'),
         protocol: 'https',
+        certificateAuthorities,
       },
     },
     junit: {