From 5b9f9a41448984914c117d03036f7649e8c616f3 Mon Sep 17 00:00:00 2001 From: Elastic Jasper Date: Fri, 23 Sep 2016 12:21:40 -0400 Subject: [PATCH] Backport PR #8421 --------- **Commit 1:** Add field_capabilities API This adds a simple API for getting the searchable/aggregatable status of a list of fields in a given index, list of indices, or index pattern. In the future this will probably evolve into a full blown fields info API that we can use when removing the index pattern mapping cache. For now though it's built to provide the minimum info needed to fix https://github.com/elastic/kibana/issues/6769 Usage: The API exposes a single GET endpoint. ``` GET /api/kibana/{indices}/field_capabilities ``` `indices` can be a single index, a comma delimited list, or a wildcard pattern Example response: ``` { "fields": { "imsearchable": { "searchable": true, "aggregatable": false }, "imaggregatable": { "searchable": true, "aggregatable": true }, } } ``` * Original sha: bea909d97634b69f07013485eee41f62d5d017e0 [formerly 1af6b76bd4f790a02d1f211d57bb6585dd458584] * Authored by Matthew Bargar on 2016-09-21T18:38:34Z **Commit 2:** Filter non-aggregatable fields from vis editor UI Using the field_capabilities API added in the previous commit, this commit enhances the client side index pattern object with information about the searchable and aggregatable status of each field in the index pattern. We then use this information to filter out non-aggregatable fields from the vis editor so that users won't accidentally select them and get nasty errors. An example of a non-aggregatable field would be a `text` field without fielddata enabled (which is the default). I also added the searchable and aggregatable flags to the index pattern page so users can see the status of their fields. I removed the `indexed` column because it was mostly redundant with `searchable` and I needed the horizontal space. The addition of the searchable and aggregatable properties for index pattern fields would require users to manually refresh their field list when upgrading to 5.0. This commit also adds a check for those properties and if they're missing it automatically refreshes the field list for the user in a seamless manner. * Original sha: b823b877f90ce84cb6f789ea90a0fb17e53ad12f [formerly 4a906f30c7d0450361ce5296d3ae7c2973f598c0] * Authored by Matthew Bargar on 2016-09-21T19:18:10Z Former-commit-id: 672f009f6c078f5376725a42e9ea140f2a65e04e --- .../sections/indices/_indexed_fields.js | 15 ++-- .../public/visualize/editor/agg_params.js | 2 +- .../kibana/server/routes/api/ingest/index.js | 2 + .../api/ingest/register_field_capabilities.js | 31 +++++++ .../__tests__/_index_pattern.js | 37 ++++++-- .../_enhance_fields_with_capabilities.js | 16 ++++ src/ui/public/index_patterns/_field.js | 4 + .../public/index_patterns/_index_pattern.js | 21 +++-- src/ui/public/index_patterns/_mapper.js | 10 ++- test/unit/api/ingest/_field_capabilities.js | 87 +++++++++++++++++++ test/unit/api/ingest/index.js | 2 + 11 files changed, 205 insertions(+), 22 deletions(-) create mode 100644 src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js create mode 100644 src/ui/public/index_patterns/_enhance_fields_with_capabilities.js create mode 100644 test/unit/api/ingest/_field_capabilities.js diff --git a/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js b/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js index d95a41ae66d9a..1d4f29ab233e8 100644 --- a/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js +++ b/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js @@ -23,8 +23,9 @@ uiModules.get('apps/management') { title: 'name' }, { title: 'type' }, { title: 'format' }, + { title: 'searchable', info: 'These fields can be used in the filter bar' }, + { title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' }, { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' }, - { title: 'indexed', info: 'Fields that are not indexed are unavailable for search' }, { title: 'controls', sortable: false } ]; @@ -54,12 +55,16 @@ uiModules.get('apps/management') }, _.get($scope.indexPattern, ['fieldFormatMap', field.name, 'type', 'title']), { - markup: field.analyzed ? yesTemplate : noTemplate, - value: field.analyzed + markup: field.searchable ? yesTemplate : noTemplate, + value: field.searchable + }, + { + markup: field.aggregatable ? yesTemplate : noTemplate, + value: field.aggregatable }, { - markup: field.indexed ? yesTemplate : noTemplate, - value: field.indexed + markup: field.analyzed ? yesTemplate : noTemplate, + value: field.analyzed }, { markup: controlsHtml, diff --git a/src/core_plugins/kibana/public/visualize/editor/agg_params.js b/src/core_plugins/kibana/public/visualize/editor/agg_params.js index a525057c017cd..81f40941305bf 100644 --- a/src/core_plugins/kibana/public/visualize/editor/agg_params.js +++ b/src/core_plugins/kibana/public/visualize/editor/agg_params.js @@ -138,7 +138,7 @@ uiModules } function getIndexedFields(param) { - let fields = $scope.agg.vis.indexPattern.fields.raw; + let fields = _.filter($scope.agg.vis.indexPattern.fields.raw, 'aggregatable'); const fieldTypes = param.filterFieldTypes; if (fieldTypes) { diff --git a/src/core_plugins/kibana/server/routes/api/ingest/index.js b/src/core_plugins/kibana/server/routes/api/ingest/index.js index 8b723c800946a..ff31b8104b76e 100644 --- a/src/core_plugins/kibana/server/routes/api/ingest/index.js +++ b/src/core_plugins/kibana/server/routes/api/ingest/index.js @@ -3,6 +3,7 @@ import { registerDelete } from './register_delete'; import { registerProcessors } from './register_processors'; import { registerSimulate } from './register_simulate'; import { registerData } from './register_data'; +import { registerFieldCapabilities } from './register_field_capabilities'; export default function (server) { registerPost(server); @@ -10,4 +11,5 @@ export default function (server) { registerProcessors(server); registerSimulate(server); registerData(server); + registerFieldCapabilities(server); } diff --git a/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js b/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js new file mode 100644 index 0000000000000..3a150804d748a --- /dev/null +++ b/src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js @@ -0,0 +1,31 @@ +import _ from 'lodash'; +import handleESError from '../../../lib/handle_es_error'; + +export function registerFieldCapabilities(server) { + server.route({ + path: '/api/kibana/{indices}/field_capabilities', + method: ['GET'], + handler: function (req, reply) { + const callWithRequest = server.plugins.elasticsearch.callWithRequest; + const indices = req.params.indices || ''; + + return callWithRequest(req, 'fieldStats', { + fields: '*', + level: 'cluster', + index: indices, + allowNoIndices: false + }) + .catch((error) => { + reply(handleESError(error)); + }) + .then((res) => { + const fields = _.get(res, 'indices._all.fields', {}); + const fieldsFilteredValues = _.mapValues(fields, (value) => { + return _.pick(value, ['searchable', 'aggregatable']); + }); + + reply({fields: fieldsFilteredValues}); + }); + } + }); +} diff --git a/src/ui/public/index_patterns/__tests__/_index_pattern.js b/src/ui/public/index_patterns/__tests__/_index_pattern.js index 9503deeb9e446..2b6d7ca3d9e37 100644 --- a/src/ui/public/index_patterns/__tests__/_index_pattern.js +++ b/src/ui/public/index_patterns/__tests__/_index_pattern.js @@ -43,6 +43,9 @@ describe('index pattern', function () { sinon.stub(mapper, 'getFieldsForIndexPattern', function () { return Promise.resolve(_.filter(mockLogstashFields, { scripted: false })); }); + sinon.stub(mapper, 'clearCache', function () { + return Promise.resolve(); + }); // stub mappingSetup mappingSetup = Private(UtilsMappingSetupProvider); @@ -151,15 +154,34 @@ describe('index pattern', function () { const indexPatternId = 'test-pattern'; let indexPattern; let fieldLength; - let truncatedFields; + let customFields; beforeEach(function () { fieldLength = mockLogstashFields.length; - truncatedFields = mockLogstashFields.slice(3); + customFields = [{ + analyzed: true, + count: 30, + filterable: true, + indexed: true, + name: 'foo', + scripted: false, + sortable: true, + type: 'number', + aggregatable: true, + searchable: false + }, + { + name: 'script number', + type: 'number', + scripted: true, + script: '1234', + lang: 'expression' + }]; + return create(indexPatternId, { _source: { customFormats: '{}', - fields: JSON.stringify(truncatedFields) + fields: JSON.stringify(customFields) } }).then(function (pattern) { indexPattern = pattern; @@ -168,8 +190,8 @@ describe('index pattern', function () { it('should fetch fields from the doc source', function () { // ensure that we don't have all the fields - expect(truncatedFields.length).to.not.equal(mockLogstashFields.length); - expect(indexPattern.fields).to.have.length(truncatedFields.length); + expect(customFields.length).to.not.equal(mockLogstashFields.length); + expect(indexPattern.fields).to.have.length(customFields.length); // ensure that all fields will be included in the returned docSource setDocsourcePayload(docSourceResponse(indexPatternId)); @@ -201,9 +223,8 @@ describe('index pattern', function () { // called to append scripted fields to the response from mapper.getFieldsForIndexPattern expect(scriptedFieldsSpy.callCount).to.equal(1); - const scripted = _.where(mockLogstashFields, { scripted: true }); - const expected = _.filter(indexPattern.fields, { scripted: true }); - expect(_.pluck(expected, 'name')).to.eql(_.pluck(scripted, 'name')); + const expected = _.filter(indexPattern.fields, {scripted: true}); + expect(_.pluck(expected, 'name')).to.eql(['script number']); }); }); }); diff --git a/src/ui/public/index_patterns/_enhance_fields_with_capabilities.js b/src/ui/public/index_patterns/_enhance_fields_with_capabilities.js new file mode 100644 index 0000000000000..6bb1980fa799c --- /dev/null +++ b/src/ui/public/index_patterns/_enhance_fields_with_capabilities.js @@ -0,0 +1,16 @@ +import chrome from 'ui/chrome'; +import _ from 'lodash'; + +export default function ($http) { + + return function (fields, indices) { + return $http.get(chrome.addBasePath(`/api/kibana/${indices}/field_capabilities`)) + .then((res) => { + const stats = _.get(res, 'data.fields', {}); + + return _.map(fields, (field) => { + return _.assign(field, stats[field.name]); + }); + }); + }; +} diff --git a/src/ui/public/index_patterns/_field.js b/src/ui/public/index_patterns/_field.js index 986d1fa14ebdd..826a6da55a6ad 100644 --- a/src/ui/public/index_patterns/_field.js +++ b/src/ui/public/index_patterns/_field.js @@ -57,6 +57,10 @@ export default function FieldObjectProvider(Private, shortDotsFilter, $rootScope obj.fact('analyzed', !!spec.analyzed); obj.fact('doc_values', !!spec.doc_values); + // stats + obj.fact('searchable', !!spec.searchable); + obj.fact('aggregatable', !!spec.aggregatable); + // usage flags, read-only and won't be saved obj.comp('format', format); obj.comp('sortable', sortable); diff --git a/src/ui/public/index_patterns/_index_pattern.js b/src/ui/public/index_patterns/_index_pattern.js index 0f7b8c8130cee..6a088619ec407 100644 --- a/src/ui/public/index_patterns/_index_pattern.js +++ b/src/ui/public/index_patterns/_index_pattern.js @@ -82,23 +82,34 @@ export default function IndexPatternFactory(Private, Notifier, config, kbnIndex, // give index pattern all of the values in _source _.assign(indexPattern, response._source); - indexFields(indexPattern); + const promise = indexFields(indexPattern); // any time index pattern in ES is updated, update index pattern object docSources .get(indexPattern) .onUpdate() .then(response => updateFromElasticSearch(indexPattern, response), notify.fatal); + + return promise; + } + + function containsFieldCapabilities(fields) { + return _.any(fields, (field) => { + return _.has(field, 'aggregatable') && _.has(field, 'searchable'); + }); } function indexFields(indexPattern) { + let promise = Promise.resolve(); + if (!indexPattern.id) { - return; + return promise; } - if (!indexPattern.fields) { - return indexPattern.refreshFields(); + + if (!indexPattern.fields || !containsFieldCapabilities(indexPattern.fields)) { + promise = indexPattern.refreshFields(); } - initFields(indexPattern); + return promise.then(() => {initFields(indexPattern);}); } function setId(indexPattern, id) { diff --git a/src/ui/public/index_patterns/_mapper.js b/src/ui/public/index_patterns/_mapper.js index 0ee8984327749..b3a0d5b4584e2 100644 --- a/src/ui/public/index_patterns/_mapper.js +++ b/src/ui/public/index_patterns/_mapper.js @@ -1,12 +1,14 @@ import { IndexPatternMissingIndices } from 'ui/errors'; import _ from 'lodash'; import moment from 'moment'; +import EnhanceFieldsWithCapabilitiesProvider from 'ui/index_patterns/_enhance_fields_with_capabilities'; import IndexPatternsTransformMappingIntoFieldsProvider from 'ui/index_patterns/_transform_mapping_into_fields'; import IndexPatternsIntervalsProvider from 'ui/index_patterns/_intervals'; import IndexPatternsPatternToWildcardProvider from 'ui/index_patterns/_pattern_to_wildcard'; import IndexPatternsLocalCacheProvider from 'ui/index_patterns/_local_cache'; export default function MapperService(Private, Promise, es, config, kbnIndex) { + let enhanceFieldsWithCapabilities = Private(EnhanceFieldsWithCapabilitiesProvider); let transformMappingIntoFields = Private(IndexPatternsTransformMappingIntoFieldsProvider); let intervals = Private(IndexPatternsIntervalsProvider); let patternToWildcard = Private(IndexPatternsPatternToWildcardProvider); @@ -49,16 +51,17 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) { }); } - let promise = Promise.resolve(id); + let indexList = id; + let promise = Promise.resolve(); if (indexPattern.intervalName) { promise = self.getIndicesForIndexPattern(indexPattern) .then(function (existing) { if (existing.matches.length === 0) throw new IndexPatternMissingIndices(); - return existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent + indexList = existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent }); } - return promise.then(function (indexList) { + return promise.then(function () { return es.indices.getFieldMapping({ index: indexList, fields: '*', @@ -69,6 +72,7 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) { }) .catch(handleMissingIndexPattern) .then(transformMappingIntoFields) + .then(fields => enhanceFieldsWithCapabilities(fields, indexList)) .then(function (fields) { fieldCache.set(id, fields); return fieldCache.get(id); diff --git a/test/unit/api/ingest/_field_capabilities.js b/test/unit/api/ingest/_field_capabilities.js new file mode 100644 index 0000000000000..e4be4a24b4930 --- /dev/null +++ b/test/unit/api/ingest/_field_capabilities.js @@ -0,0 +1,87 @@ +define(function (require) { + var Promise = require('bluebird'); + var _ = require('intern/dojo/node!lodash'); + var expect = require('intern/dojo/node!expect.js'); + + return function (bdd, scenarioManager, request) { + bdd.describe('field_capabilities API', function postIngest() { + + bdd.before(function () { + return scenarioManager.client.create({ + index: 'foo-1', + type: 'bar', + id: '1', + body: { + foo: 'bar' + } + }) + .then(function () { + return scenarioManager.client.create({ + index: 'foo-2', + type: 'bar', + id: '2', + body: { + baz: 'bar' + } + }); + }) + .then(function () { + return scenarioManager.client.indices.refresh({ + index: ['foo-1', 'foo-2'] + }); + }); + }); + + bdd.after(function () { + return scenarioManager.reload('emptyKibana') + .then(function () { + scenarioManager.client.indices.delete({ + index: 'foo*' + }); + }); + }); + + bdd.it('should return searchable/aggregatable flags for fields in the indices specified', function () { + return request.get('/kibana/foo-1/field_capabilities') + .expect(200) + .then(function (response) { + var fields = response.body.fields; + expect(fields.foo).to.eql({searchable: true, aggregatable: false}); + expect(fields['foo.keyword']).to.eql({searchable: true, aggregatable: true}); + expect(fields).to.not.have.property('baz'); + }); + }); + + bdd.it('should accept wildcards in the index name', function () { + return request.get('/kibana/foo-*/field_capabilities') + .expect(200) + .then(function (response) { + var fields = response.body.fields; + expect(fields.foo).to.eql({searchable: true, aggregatable: false}); + expect(fields.baz).to.eql({searchable: true, aggregatable: false}); + }); + }); + + bdd.it('should accept comma delimited lists of indices', function () { + return request.get('/kibana/foo-1,foo-2/field_capabilities') + .expect(200) + .then(function (response) { + var fields = response.body.fields; + expect(fields.foo).to.eql({searchable: true, aggregatable: false}); + expect(fields.baz).to.eql({searchable: true, aggregatable: false}); + }); + }); + + bdd.it('should return 404 if a pattern matches no indices', function () { + return request.post('/kibana/doesnotexist-*/field_capabilities') + .expect(404); + }); + + bdd.it('should return 404 if a concrete index does not exist', function () { + return request.post('/kibana/concrete/field_capabilities') + .expect(404); + }); + + }); + }; +}); diff --git a/test/unit/api/ingest/index.js b/test/unit/api/ingest/index.js index d83140c7dc9e6..f8b548f75db81 100644 --- a/test/unit/api/ingest/index.js +++ b/test/unit/api/ingest/index.js @@ -12,6 +12,7 @@ define(function (require) { var simulate = require('./_simulate'); var processors = require('./_processors'); var processorTypes = require('./processors/index'); + var fieldCapabilities = require('./_field_capabilities'); bdd.describe('ingest API', function () { var scenarioManager = new ScenarioManager(url.format(serverConfig.servers.elasticsearch)); @@ -31,5 +32,6 @@ define(function (require) { simulate(bdd, scenarioManager, request); processors(bdd, scenarioManager, request); processorTypes(bdd, scenarioManager, request); + fieldCapabilities(bdd, scenarioManager, request); }); });