From 4a906f30c7d0450361ce5296d3ae7c2973f598c0 Mon Sep 17 00:00:00 2001 From: Matthew Bargar Date: Wed, 21 Sep 2016 15:18:10 -0400 Subject: [PATCH] Filter non-aggregatable fields from vis editor UI Using the field_capabilities API added in the previous commit, this commit enhances the client side index pattern object with information about the searchable and aggregatable status of each field in the index pattern. We then use this information to filter out non-aggregatable fields from the vis editor so that users won't accidentally select them and get nasty errors. An example of a non-aggregatable field would be a `text` field without fielddata enabled (which is the default). I also added the searchable and aggregatable flags to the index pattern page so users can see the status of their fields. I removed the `indexed` column because it was mostly redundant with `searchable` and I needed the horizontal space. The addition of the searchable and aggregatable properties for index pattern fields would require users to manually refresh their field list when upgrading to 5.0. This commit also adds a check for those properties and if they're missing it automatically refreshes the field list for the user in a seamless manner. --- .../sections/indices/_indexed_fields.js | 15 +++++--- .../public/visualize/editor/agg_params.js | 2 +- .../__tests__/_index_pattern.js | 37 +++++++++++++++---- .../_enhance_fields_with_capabilities.js | 16 ++++++++ src/ui/public/index_patterns/_field.js | 4 ++ .../public/index_patterns/_index_pattern.js | 21 ++++++++--- src/ui/public/index_patterns/_mapper.js | 10 +++-- 7 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 src/ui/public/index_patterns/_enhance_fields_with_capabilities.js diff --git a/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js b/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js index 773c65c971ad3..6ff6548231f2a 100644 --- a/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js +++ b/src/core_plugins/kibana/public/management/sections/indices/_indexed_fields.js @@ -23,8 +23,9 @@ uiModules.get('apps/management') { title: 'name' }, { title: 'type' }, { title: 'format' }, + { title: 'searchable', info: 'These fields can be used in the filter bar' }, + { title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' }, { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' }, - { title: 'indexed', info: 'Fields that are not indexed are unavailable for search' }, { title: 'controls', sortable: false } ]; @@ -57,12 +58,16 @@ uiModules.get('apps/management') }, _.get($scope.indexPattern, ['fieldFormatMap', field.name, 'type', 'title']), { - markup: field.analyzed ? yesTemplate : noTemplate, - value: field.analyzed + markup: field.searchable ? yesTemplate : noTemplate, + value: field.searchable + }, + { + markup: field.aggregatable ? yesTemplate : noTemplate, + value: field.aggregatable }, { - markup: field.indexed ? yesTemplate : noTemplate, - value: field.indexed + markup: field.analyzed ? yesTemplate : noTemplate, + value: field.analyzed }, { markup: controlsHtml, diff --git a/src/core_plugins/kibana/public/visualize/editor/agg_params.js b/src/core_plugins/kibana/public/visualize/editor/agg_params.js index a525057c017cd..81f40941305bf 100644 --- a/src/core_plugins/kibana/public/visualize/editor/agg_params.js +++ b/src/core_plugins/kibana/public/visualize/editor/agg_params.js @@ -138,7 +138,7 @@ uiModules } function getIndexedFields(param) { - let fields = $scope.agg.vis.indexPattern.fields.raw; + let fields = _.filter($scope.agg.vis.indexPattern.fields.raw, 'aggregatable'); const fieldTypes = param.filterFieldTypes; if (fieldTypes) { diff --git a/src/ui/public/index_patterns/__tests__/_index_pattern.js b/src/ui/public/index_patterns/__tests__/_index_pattern.js index 9503deeb9e446..2b6d7ca3d9e37 100644 --- a/src/ui/public/index_patterns/__tests__/_index_pattern.js +++ b/src/ui/public/index_patterns/__tests__/_index_pattern.js @@ -43,6 +43,9 @@ describe('index pattern', function () { sinon.stub(mapper, 'getFieldsForIndexPattern', function () { return Promise.resolve(_.filter(mockLogstashFields, { scripted: false })); }); + sinon.stub(mapper, 'clearCache', function () { + return Promise.resolve(); + }); // stub mappingSetup mappingSetup = Private(UtilsMappingSetupProvider); @@ -151,15 +154,34 @@ describe('index pattern', function () { const indexPatternId = 'test-pattern'; let indexPattern; let fieldLength; - let truncatedFields; + let customFields; beforeEach(function () { fieldLength = mockLogstashFields.length; - truncatedFields = mockLogstashFields.slice(3); + customFields = [{ + analyzed: true, + count: 30, + filterable: true, + indexed: true, + name: 'foo', + scripted: false, + sortable: true, + type: 'number', + aggregatable: true, + searchable: false + }, + { + name: 'script number', + type: 'number', + scripted: true, + script: '1234', + lang: 'expression' + }]; + return create(indexPatternId, { _source: { customFormats: '{}', - fields: JSON.stringify(truncatedFields) + fields: JSON.stringify(customFields) } }).then(function (pattern) { indexPattern = pattern; @@ -168,8 +190,8 @@ describe('index pattern', function () { it('should fetch fields from the doc source', function () { // ensure that we don't have all the fields - expect(truncatedFields.length).to.not.equal(mockLogstashFields.length); - expect(indexPattern.fields).to.have.length(truncatedFields.length); + expect(customFields.length).to.not.equal(mockLogstashFields.length); + expect(indexPattern.fields).to.have.length(customFields.length); // ensure that all fields will be included in the returned docSource setDocsourcePayload(docSourceResponse(indexPatternId)); @@ -201,9 +223,8 @@ describe('index pattern', function () { // called to append scripted fields to the response from mapper.getFieldsForIndexPattern expect(scriptedFieldsSpy.callCount).to.equal(1); - const scripted = _.where(mockLogstashFields, { scripted: true }); - const expected = _.filter(indexPattern.fields, { scripted: true }); - expect(_.pluck(expected, 'name')).to.eql(_.pluck(scripted, 'name')); + const expected = _.filter(indexPattern.fields, {scripted: true}); + expect(_.pluck(expected, 'name')).to.eql(['script number']); }); }); }); diff --git a/src/ui/public/index_patterns/_enhance_fields_with_capabilities.js b/src/ui/public/index_patterns/_enhance_fields_with_capabilities.js new file mode 100644 index 0000000000000..6bb1980fa799c --- /dev/null +++ b/src/ui/public/index_patterns/_enhance_fields_with_capabilities.js @@ -0,0 +1,16 @@ +import chrome from 'ui/chrome'; +import _ from 'lodash'; + +export default function ($http) { + + return function (fields, indices) { + return $http.get(chrome.addBasePath(`/api/kibana/${indices}/field_capabilities`)) + .then((res) => { + const stats = _.get(res, 'data.fields', {}); + + return _.map(fields, (field) => { + return _.assign(field, stats[field.name]); + }); + }); + }; +} diff --git a/src/ui/public/index_patterns/_field.js b/src/ui/public/index_patterns/_field.js index 986d1fa14ebdd..826a6da55a6ad 100644 --- a/src/ui/public/index_patterns/_field.js +++ b/src/ui/public/index_patterns/_field.js @@ -57,6 +57,10 @@ export default function FieldObjectProvider(Private, shortDotsFilter, $rootScope obj.fact('analyzed', !!spec.analyzed); obj.fact('doc_values', !!spec.doc_values); + // stats + obj.fact('searchable', !!spec.searchable); + obj.fact('aggregatable', !!spec.aggregatable); + // usage flags, read-only and won't be saved obj.comp('format', format); obj.comp('sortable', sortable); diff --git a/src/ui/public/index_patterns/_index_pattern.js b/src/ui/public/index_patterns/_index_pattern.js index 0f7b8c8130cee..6a088619ec407 100644 --- a/src/ui/public/index_patterns/_index_pattern.js +++ b/src/ui/public/index_patterns/_index_pattern.js @@ -82,23 +82,34 @@ export default function IndexPatternFactory(Private, Notifier, config, kbnIndex, // give index pattern all of the values in _source _.assign(indexPattern, response._source); - indexFields(indexPattern); + const promise = indexFields(indexPattern); // any time index pattern in ES is updated, update index pattern object docSources .get(indexPattern) .onUpdate() .then(response => updateFromElasticSearch(indexPattern, response), notify.fatal); + + return promise; + } + + function containsFieldCapabilities(fields) { + return _.any(fields, (field) => { + return _.has(field, 'aggregatable') && _.has(field, 'searchable'); + }); } function indexFields(indexPattern) { + let promise = Promise.resolve(); + if (!indexPattern.id) { - return; + return promise; } - if (!indexPattern.fields) { - return indexPattern.refreshFields(); + + if (!indexPattern.fields || !containsFieldCapabilities(indexPattern.fields)) { + promise = indexPattern.refreshFields(); } - initFields(indexPattern); + return promise.then(() => {initFields(indexPattern);}); } function setId(indexPattern, id) { diff --git a/src/ui/public/index_patterns/_mapper.js b/src/ui/public/index_patterns/_mapper.js index 0ee8984327749..b3a0d5b4584e2 100644 --- a/src/ui/public/index_patterns/_mapper.js +++ b/src/ui/public/index_patterns/_mapper.js @@ -1,12 +1,14 @@ import { IndexPatternMissingIndices } from 'ui/errors'; import _ from 'lodash'; import moment from 'moment'; +import EnhanceFieldsWithCapabilitiesProvider from 'ui/index_patterns/_enhance_fields_with_capabilities'; import IndexPatternsTransformMappingIntoFieldsProvider from 'ui/index_patterns/_transform_mapping_into_fields'; import IndexPatternsIntervalsProvider from 'ui/index_patterns/_intervals'; import IndexPatternsPatternToWildcardProvider from 'ui/index_patterns/_pattern_to_wildcard'; import IndexPatternsLocalCacheProvider from 'ui/index_patterns/_local_cache'; export default function MapperService(Private, Promise, es, config, kbnIndex) { + let enhanceFieldsWithCapabilities = Private(EnhanceFieldsWithCapabilitiesProvider); let transformMappingIntoFields = Private(IndexPatternsTransformMappingIntoFieldsProvider); let intervals = Private(IndexPatternsIntervalsProvider); let patternToWildcard = Private(IndexPatternsPatternToWildcardProvider); @@ -49,16 +51,17 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) { }); } - let promise = Promise.resolve(id); + let indexList = id; + let promise = Promise.resolve(); if (indexPattern.intervalName) { promise = self.getIndicesForIndexPattern(indexPattern) .then(function (existing) { if (existing.matches.length === 0) throw new IndexPatternMissingIndices(); - return existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent + indexList = existing.matches.slice(-config.get('indexPattern:fieldMapping:lookBack')); // Grab the most recent }); } - return promise.then(function (indexList) { + return promise.then(function () { return es.indices.getFieldMapping({ index: indexList, fields: '*', @@ -69,6 +72,7 @@ export default function MapperService(Private, Promise, es, config, kbnIndex) { }) .catch(handleMissingIndexPattern) .then(transformMappingIntoFields) + .then(fields => enhanceFieldsWithCapabilities(fields, indexList)) .then(function (fields) { fieldCache.set(id, fields); return fieldCache.get(id);