diff --git a/docs/visualize.asciidoc b/docs/visualize.asciidoc index 40620bc4661e8..31656f727b018 100644 --- a/docs/visualize.asciidoc +++ b/docs/visualize.asciidoc @@ -50,6 +50,7 @@ data sets. <>:: Display words as a cloud in which the size of the word correspond to its importance <>:: Display free-form information or instructions. +<>:: Support for user-defined graphs, external data sources, images, and user-defined interactivity. . Specify a search query to retrieve the data for your visualization: ** To enter new search criteria, select the index pattern for the indices that contain the data you want to visualize. This opens the visualization builder @@ -159,3 +160,5 @@ include::visualize/tagcloud.asciidoc[] include::visualize/heatmap.asciidoc[] include::visualize/visualization-raw-data.asciidoc[] + +include::visualize/vega.asciidoc[] diff --git a/docs/visualize/vega.asciidoc b/docs/visualize/vega.asciidoc new file mode 100644 index 0000000000000..2c3dd91a77bcc --- /dev/null +++ b/docs/visualize/vega.asciidoc @@ -0,0 +1,289 @@ +[[vega-graph]] +== Vega Graphs + +__________________________________________________________________________________________________________________________________________________________________________________ +Build https://vega.github.io/vega/examples/[Vega] and +https://vega.github.io/vega-lite/examples/[VegaLite] data visualizations +into Kibana. +__________________________________________________________________________________________________________________________________________________________________________________ + +[[vega-introduction-video]] +=== Watch a short introduction video + +https://www.youtube.com/watch?v=lQGCipY3th8[image:https://i.ytimg.com/vi_webp/lQGCipY3th8/maxresdefault.webp[Kibana Vega Visualization Video]] + +[[vega-quick-demo]] +=== Quick Demo + +* In Kibana, choose Visualize, and add Vega visualization. +* You should immediately see a default graph +* Try changing `mark` from `line` to `point`, `area`, `bar`, `circle`, +`square`, ... (see +https://vega.github.io/vega-lite/docs/mark.html#mark-def[docs]) +* Try other https://vega.github.io/vega/examples/[Vega] or +https://vega.github.io/vega-lite/examples/[VegaLite] visualizations. You +may need to make URLs absolute, e.g. replace +`"url": "data/world-110m.json"` with +`"url": "https://vega.github.io/editor/data/world-110m.json"`. (see +link:#Using%20Vega%20and%20VegaLite%20examples[notes below]) +* Using https://www.npmjs.com/package/makelogs[makelogs util], generate +some logstash data and try link:public/examples/logstash[logstash +examples]. *(Do not use makelogs on a production cluster!)* + +[[vega-vs-vegalite]] +=== Vega vs VegaLite + +VegaLite is a simplified version of Vega, useful to quickly get started, +but has a number of limitations. VegaLite is automatically converted +into Vega before rendering. Compare +link:public/examples/logstash/logstash-simple_line-vega.json[logstash-simple_line-vega] +and +link:public/examples/logstash/logstash-simple_line-vegalite.json[logstash-simple_line-vegalite] +(both use the same ElasticSearch logstash data). You may use +https://vega.github.io/editor/[this editor] to convert VegaLite into +Vega. + +[[vega-querying-elasticsearch]] +== Querying ElasticSearch + +By default, Vega's https://vega.github.io/vega/docs/data/[data] element +can use embedded and external data with a `"url"` parameter. Kibana adds support for the direct ElasticSearch queries by overloading +the `"url"` value. + +Here is an example of an ES query that counts the number of documents in all indexes. The query uses *@timestamp* field to filter the time range, and break it into histogram buckets. + +[source,yaml] +---- +// An object instead of a string for the url value +// is treated as a context-aware Elasticsearch query. +url: { + // Filter the time picker (upper right corner) with this field + %timefield%: @timestamp + // Apply dashboard context filters when set + %context%: true + + // Which indexes to search + index: _all + // The body element may contain "aggs" and "query" subfields + body: { + aggs: { + time_buckets: { + date_histogram: { + // Use date histogram aggregation on @timestamp field + field: @timestamp + // interval value will depend on the daterange picker + // Use an integer to set approximate bucket count + interval: { %autointerval%: true } + // Make sure we get an entire range, even if it has no data + extended_bounds: { + min: { %timefilter%: "min" } + max: { %timefilter%: "max" } + } + // Use this for linear (e.g. line, area) graphs + // Without it, empty buckets will not show up + min_doc_count: 0 + } + } + } + // Speed up the response by only including aggregation results + size: 0 + } +} +---- + +The full ES result has this kind of structure: + +[source,yaml] +---- +{ + "aggregations": { + "time_buckets": { + "buckets": [{ + "key_as_string": "2015-11-30T22:00:00.000Z", + "key": 1448920800000, + "doc_count": 28 + }, { + "key_as_string": "2015-11-30T23:00:00.000Z", + "key": 1448924400000, + "doc_count": 330 + }, ... +---- + +Note that `"key"` is a unix timestamp, and can be used without conversions by the +Vega date expressions. + +For most graphs we only need the list of the bucket values, so we use `format: {property: "aggregations.time_buckets.buckets"}` expression to focus on just the data we need. + +Query may be specified with individual range and dashboard context as +well. This query is equivalent to `"%context%": true, "%timefield%": "@timestamp"`, +except that the timerange is shifted back by 10 minutes: + +[source,yaml] +---- +{ + body: { + query: { + bool: { + must: [ + // This string will be replaced + // with the auto-generated "MUST" clause + "%dashboard_context-must_clause%" + { + range: { + // apply timefilter (upper right corner) + // to the @timestamp variable + @timestamp: { + // "%timefilter%" will be replaced with + // the current values of the time filter + // (from the upper right corner) + "%timefilter%": true + // Only work with %timefilter% + // Shift current timefilter by 10 units back + shift: 10 + // week, day (default), hour, minute, second + unit: minute + } + } + } + ] + must_not: [ + // This string will be replaced with + // the auto-generated "MUST-NOT" clause + "%dashboard_context-must_not_clause%" + ] + } + } + } +} +---- + +The `"%timefilter%"` can also be used to specify a single min or max +value. As shown above, the date_histogram's `extended_bounds` can be set +with two values - min and max. Instead of hardcoding a value, you may +use `"min": {"%timefilter%": "min"}`, which will be replaced with the +beginning of the current time range. The `shift` and `unit` values are +also supported. The `"interval"` can also be set dynamically, depending +on the currently picked range: `"interval": {"%autointerval%": 10}` will +try to get about 10-15 data points (buckets). + +[[vega-esmfiles]] +=== Elastic Map Files + +It is possible to access Elastic Map Service's files via the same mechanism + +[source,yaml] +---- +url: { + // "type" defaults to "elasticsearch" otherwise + type: emsfile + // Name of the file, exactly as in the Region map visualization + name: World Countries +} +// The result is a geojson file, get its features to use +// this data source with the "shape" marks +// https://vega.github.io/vega/docs/marks/shape/ +format: {property: "features"} +---- + +[[vega-debugging]] +== Debugging + +[[vega-browser-debugging-console]] +=== Browser Debugging console + +Use browser debugging tools (e.g. F12 or Ctrl+Shift+J in Chrome) to +inspect the `VEGA_DEBUG` variable: +* `view` - access to the Vega View object. See https://vega.github.io/vega/docs/api/debugging/[Vega Debugging Guide] + on how to inspect data and signals at runtime. For VegaLite, `VEGA_DEBUG.view.data('source_0')` gets the main data set. + For Vega, it uses the data name as defined in your Vega spec. +* `vega_spec` - Vega JSON graph specification after some modifications by Kibana. In case +of VegaLite, this is the output of the VegaLite compiler. +* `vegalite_spec` - If this is a VegaLite graph, JSON specification of the graph before +VegaLite compilation. + +[[vega-data]] +=== Data + +If you are using ElasticSearch query, make sure your resulting data is +what you expected. The easiest way to view it is by using "networking" +tab in the browser debugging tools (e.g. F12). Modify the graph slightly +so that it makes a search request, and view the response from the +server. Another approach is to use +https://www.elastic.co/guide/en/kibana/current/console-kibana.html[Kibana +Dev Tools] tab - place the index name into the first line: +`GET /_search`, and add your query as the following lines +(just the value of the `"query"` field) + +If you need to share your graph with someone, you may want to copy the +raw data response to https://gist.github.com/[gist.github.com], possibly +with a `.json` extension, use the `[raw]` button, and use that url +directly in your graph. + +To restrict Vega from using non-ES data sources, add `vega.enableExternalUrls: false` +to your kibana.yml file. + +[[vega-notes]] +== Notes + +[[vega-useful-links]] +=== Useful Links + +* https://vega.github.io/editor/[Editor] - includes examples for Vega & +VegaLite, but does not support any Kibana-specific features like +ElasticSearch requests and interactive base maps. +* VegaLite +https://vega.github.io/vega-lite/tutorials/getting_started.html[Tutorials], +https://vega.github.io/vega-lite/docs/[docs], and +https://vega.github.io/vega-lite/examples/[examples] +* Vega https://vega.github.io/vega/tutorials/[Tutorial], +https://vega.github.io/vega/docs/[docs], +https://vega.github.io/vega/examples/[examples] + +[[vega-using-vega-and-vegalite-examples]] +==== Using Vega and VegaLite examples + +When using https://vega.github.io/vega/examples/[Vega] and +https://vega.github.io/vega-lite/examples/[VegaLite] examples, you may +need to modify the "data" section to use absolute URL. For example, +replace `"url": "data/world-110m.json"` with +`"url": "https://vega.github.io/editor/data/world-110m.json"`. Also, +regular Vega examples use `"autosize": "pad"` layout model, whereas +Kibana uses `fit`. Remove all `autosize`, `width`, and `height` +values. See link:#sizing-and-positioning[sizing and positioning] below. + +[[vega-additional-configuration-options]] +==== Additional configuration options + +These options are specific to the Kibana. + +[source,yaml] +---- +{ + config: { + kibana: { + // Placement of the Vega-defined signal bindings. + // Can be `left`, `right`, `top`, or `bottom` (default). + controlsLocation: top + // Can be `vertical` or `horizontal` (default). + controlsDirection: vertical + // If true, hides most of Vega and VegaLite warnings + hideWarnings: true + // Vega renderer to use: `svg` or `canvas` (default) + renderer: canvas + } + } + /* the rest of Vega code */ +} +---- + +[[vega-sizing-and-positioning]] +==== Sizing and positioning + +[[vega-and-vegalite]] +Vega and VegaLite + +By default, Kibana Vega graphs will use +`autosize = { type: 'fit', contains: 'padding' }` layout model for Vega +and VegaLite graphs. The `fit` model uses all available space, ignores +`width` and `height` values, but respects the padding values. You may +override this behaviour by specifying a different `autosize` value. diff --git a/package.json b/package.json index 419d0e117d57f..20e265568be70 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,8 @@ "Nicolás Bevacqua ", "Shelby Sturgis ", "Spencer Alger ", - "Tim Sullivan " + "Tim Sullivan ", + "Yuri Astrakhan " ], "scripts": { "test": "grunt test", @@ -111,6 +112,7 @@ "check-hash": "1.0.1", "color": "1.0.3", "commander": "2.8.1", + "compare-versions": "3.1.0", "css-loader": "0.28.7", "d3": "3.5.6", "d3-cloud": "1.2.1", @@ -132,6 +134,7 @@ "h2o2": "5.1.1", "handlebars": "4.0.5", "hapi": "14.2.0", + "hjson": "3.1.0", "http-proxy-agent": "1.0.0", "imports-loader": "0.7.1", "inert": "4.0.2", @@ -140,6 +143,7 @@ "joi": "10.4.1", "jquery": "3.2.1", "js-yaml": "3.4.1", + "json-stringify-pretty-compact": "1.0.4", "json-stringify-safe": "5.0.1", "jstimezonedetect": "1.0.5", "leaflet": "1.0.3", @@ -207,6 +211,9 @@ "url-loader": "0.5.9", "uuid": "3.0.1", "validate-npm-package-name": "2.2.2", + "vega": "3.0.8", + "vega-lite": "2.0.3", + "vega-schema-url-parser": "1.0.0", "vision": "4.1.0", "webpack": "3.6.0", "webpack-merge": "4.1.0", diff --git a/src/core_plugins/tests_bundle/tests_entry_template.js b/src/core_plugins/tests_bundle/tests_entry_template.js index 64f1b0dbcdbef..eb00b35877649 100644 --- a/src/core_plugins/tests_bundle/tests_entry_template.js +++ b/src/core_plugins/tests_bundle/tests_entry_template.js @@ -30,7 +30,11 @@ window.__KBN__ = { }, mapConfig: { manifestServiceUrl: 'https://staging-dot-catalogue-dot-elastic-layer.appspot.com/v1/manifest' - } + }, + vegaConfig: { + enabled: true, + enableExternalUrls: true + }, }, uiSettings: { defaults: ${JSON.stringify(defaultUiSettings, null, 2).split('\n').join('\n ')}, diff --git a/src/core_plugins/vega/index.js b/src/core_plugins/vega/index.js new file mode 100644 index 0000000000000..f5e54c30cb3eb --- /dev/null +++ b/src/core_plugins/vega/index.js @@ -0,0 +1,15 @@ +export default kibana => new kibana.Plugin({ + id: 'vega', + require: ['elasticsearch'], + + uiExports: { + visTypes: ['plugins/vega/vega_type'], + injectDefaultVars: server => ({ vegaConfig: server.config().get('vega') }), + }, + + config: (Joi) => Joi.object({ + enabled: Joi.boolean().default(true), + enableExternalUrls: Joi.boolean().default(false) + }).default(), + +}); diff --git a/src/core_plugins/vega/package.json b/src/core_plugins/vega/package.json new file mode 100644 index 0000000000000..acd6f3da128ab --- /dev/null +++ b/src/core_plugins/vega/package.json @@ -0,0 +1,6 @@ +{ + "author": "Yuri Astrakhan", + "name": "vega", + "version": "kibana" +} + diff --git a/src/core_plugins/vega/public/__tests__/vega_graph.hjson b/src/core_plugins/vega/public/__tests__/vega_graph.hjson new file mode 100644 index 0000000000000..81267cd514f1f --- /dev/null +++ b/src/core_plugins/vega/public/__tests__/vega_graph.hjson @@ -0,0 +1,76 @@ +{ + // Adapted from Vega's https://vega.github.io/vega/examples/stacked-area-chart/ + + $schema: https://vega.github.io/schema/vega/v3.0.json + data: [ + { + name: table + values: [ + {x: 0, y: 28, c: 0}, {x: 0, y: 55, c: 1}, {x: 1, y: 43, c: 0}, {x: 1, y: 91, c: 1}, + {x: 2, y: 81, c: 0}, {x: 2, y: 53, c: 1}, {x: 3, y: 19, c: 0}, {x: 3, y: 87, c: 1}, + {x: 4, y: 52, c: 0}, {x: 4, y: 48, c: 1}, {x: 5, y: 24, c: 0}, {x: 5, y: 49, c: 1}, + {x: 6, y: 87, c: 0}, {x: 6, y: 66, c: 1}, {x: 7, y: 17, c: 0}, {x: 7, y: 27, c: 1}, + {x: 8, y: 68, c: 0}, {x: 8, y: 16, c: 1}, {x: 9, y: 49, c: 0}, {x: 9, y: 15, c: 1} + ] + transform: [ + { + type: stack + groupby: ["x"] + sort: {field: "c"} + field: y + } + ] + } + ] + scales: [ + { + name: x + type: point + range: width + domain: {data: "table", field: "x"} + } + { + name: y + type: linear + range: height + nice: true + zero: true + domain: {data: "table", field: "y1"} + } + { + name: color + type: ordinal + range: category + domain: {data: "table", field: "c"} + } + ] + marks: [ + { + type: group + from: { + facet: {name: "series", data: "table", groupby: "c"} + } + marks: [ + { + type: area + from: {data: "series"} + encode: { + enter: { + interpolate: {value: "monotone"} + x: {scale: "x", field: "x"} + y: {scale: "y", field: "y0"} + y2: {scale: "y", field: "y1"} + fill: {scale: "color", field: "c"} + } + update: { + fillOpacity: {value: 1} + } + hover: { + fillOpacity: {value: 0.5} + } + } + } + ] + } + ] +} diff --git a/src/core_plugins/vega/public/__tests__/vega_image_512.png b/src/core_plugins/vega/public/__tests__/vega_image_512.png new file mode 100644 index 0000000000000..44cd0d320931f Binary files /dev/null and b/src/core_plugins/vega/public/__tests__/vega_image_512.png differ diff --git a/src/core_plugins/vega/public/__tests__/vega_visualization.js b/src/core_plugins/vega/public/__tests__/vega_visualization.js new file mode 100644 index 0000000000000..4c523df592d03 --- /dev/null +++ b/src/core_plugins/vega/public/__tests__/vega_visualization.js @@ -0,0 +1,124 @@ +import expect from 'expect.js'; +import ngMock from 'ng_mock'; +import { VegaVisualizationProvider } from '../vega_visualization'; +import LogstashIndexPatternStubProvider from 'fixtures/stubbed_logstash_index_pattern'; +import * as visModule from 'ui/vis'; +import { ImageComparator } from 'test_utils/image_comparator'; + +import vegaliteGraph from '!!raw-loader!./vegalite_graph.hjson'; +import vegaliteImage256 from './vegalite_image_256.png'; +import vegaliteImage512 from './vegalite_image_512.png'; + +import vegaGraph from '!!raw-loader!./vega_graph.hjson'; +import vegaImage512 from './vega_image_512.png'; + +import { VegaParser } from '../data_model/vega_parser'; +import { SearchCache } from '../data_model/search_cache'; + +const THRESHOLD = 0.10; +const PIXEL_DIFF = 10; + +describe('VegaVisualizations', () => { + + let domNode; + let VegaVisualization; + let Vis; + let indexPattern; + let vis; + let imageComparator; + + beforeEach(ngMock.module('kibana')); + beforeEach(ngMock.inject((Private) => { + + Vis = Private(visModule.VisProvider); + VegaVisualization = Private(VegaVisualizationProvider); + indexPattern = Private(LogstashIndexPatternStubProvider); + + })); + + + describe('VegaVisualization - basics', () => { + + beforeEach(async function () { + setupDOM('512px', '512px'); + imageComparator = new ImageComparator(); + vis = new Vis(indexPattern, { type: 'vega' }); + }); + + afterEach(function () { + teardownDOM(); + imageComparator.destroy(); + }); + + it('should show vegalite graph and update on resize', async function () { + + let vegaVis; + try { + vegaVis = new VegaVisualization(domNode, vis); + const vegaParser = new VegaParser(vegaliteGraph, new SearchCache()); + await vegaParser.parseAsync(); + + await vegaVis.render(vegaParser, { data: true }); + const mismatchedPixels1 = await compareImage(vegaliteImage512); + expect(mismatchedPixels1).to.be.lessThan(PIXEL_DIFF); + + domNode.style.width = '256px'; + domNode.style.height = '256px'; + + await vegaVis.render(vegaParser, { resize: true }); + const mismatchedPixels2 = await compareImage(vegaliteImage256); + expect(mismatchedPixels2).to.be.lessThan(PIXEL_DIFF); + + } finally { + vegaVis.destroy(); + } + + }); + + it('should show vega graph', async function () { + + let vegaVis; + try { + + vegaVis = new VegaVisualization(domNode, vis); + const vegaParser = new VegaParser(vegaGraph, new SearchCache()); + await vegaParser.parseAsync(); + + await vegaVis.render(vegaParser, { data: true }); + const mismatchedPixels = await compareImage(vegaImage512); + expect(mismatchedPixels).to.be.lessThan(PIXEL_DIFF); + + } finally { + vegaVis.destroy(); + } + + }); + + }); + + + async function compareImage(expectedImageSource) { + const elementList = domNode.querySelectorAll('canvas'); + expect(elementList.length).to.equal(1); + const firstCanvasOnMap = elementList[0]; + return imageComparator.compareImage(firstCanvasOnMap, expectedImageSource, THRESHOLD); + } + + function setupDOM(width, height) { + domNode = document.createElement('div'); + domNode.style.top = '0'; + domNode.style.left = '0'; + domNode.style.width = width; + domNode.style.height = height; + domNode.style.position = 'fixed'; + domNode.style.border = '1px solid blue'; + domNode.style['pointer-events'] = 'none'; + document.body.appendChild(domNode); + } + + function teardownDOM() { + domNode.innerHTML = ''; + document.body.removeChild(domNode); + } + +}); diff --git a/src/core_plugins/vega/public/__tests__/vegalite_graph.hjson b/src/core_plugins/vega/public/__tests__/vegalite_graph.hjson new file mode 100644 index 0000000000000..ba39dfb97b0aa --- /dev/null +++ b/src/core_plugins/vega/public/__tests__/vegalite_graph.hjson @@ -0,0 +1,45 @@ +{ + $schema: https://vega.github.io/schema/vega-lite/v2.json + data: { + format: {property: "aggregations.time_buckets.buckets"} + values: { + aggregations: { + time_buckets: { + buckets: [ + {key: 1512950400000, doc_count: 0} + {key: 1513036800000, doc_count: 0} + {key: 1513123200000, doc_count: 0} + {key: 1513209600000, doc_count: 4545} + {key: 1513296000000, doc_count: 4667} + {key: 1513382400000, doc_count: 4660} + {key: 1513468800000, doc_count: 133} + {key: 1513555200000, doc_count: 0} + {key: 1513641600000, doc_count: 0} + {key: 1513728000000, doc_count: 0} + ] + } + } + status: 200 + } + } + mark: line + encoding: { + x: { + field: key + type: temporal + axis: null + } + y: { + field: doc_count + type: quantitative + axis: null + } + } + config: { + range: { + category: {scheme: "elastic"} + } + mark: {color: "#00A69B"} + } + autosize: {type: "fit", contains: "padding"} +} diff --git a/src/core_plugins/vega/public/__tests__/vegalite_image_256.png b/src/core_plugins/vega/public/__tests__/vegalite_image_256.png new file mode 100644 index 0000000000000..3f247b57905d4 Binary files /dev/null and b/src/core_plugins/vega/public/__tests__/vegalite_image_256.png differ diff --git a/src/core_plugins/vega/public/__tests__/vegalite_image_512.png b/src/core_plugins/vega/public/__tests__/vegalite_image_512.png new file mode 100644 index 0000000000000..c387c3ec789d3 Binary files /dev/null and b/src/core_plugins/vega/public/__tests__/vegalite_image_512.png differ diff --git a/src/core_plugins/vega/public/data_model/__tests__/es_query_parser.js b/src/core_plugins/vega/public/data_model/__tests__/es_query_parser.js new file mode 100644 index 0000000000000..44752e7736566 --- /dev/null +++ b/src/core_plugins/vega/public/data_model/__tests__/es_query_parser.js @@ -0,0 +1,131 @@ +import _ from 'lodash'; +import expect from 'expect.js'; +import { EsQueryParser } from '../es_query_parser'; + +const second = 1000; +const minute = 60 * second; +const hour = 60 * minute; +const day = 24 * hour; + +const rangeStart = 10 * day; +const rangeEnd = 12 * day; +const ctxArr = { bool: { must: [{ match_all: { c: 3 } }], must_not: [{ d: 4 }] } }; +const ctxObj = { bool: { must: { match_all: { a: 1 } }, must_not: { b: 2 } } }; + +function create(min, max, dashboardCtx) { + const inst = new EsQueryParser( + { + getTimeBounds: () => ({ min, max }) + }, + () => {}, + () => _.cloneDeep(dashboardCtx), + () => (inst.$$$warnCount = (inst.$$$warnCount || 0) + 1) + ); + return inst; +} + +describe(`EsQueryParser time`, () => { + + it(`roundInterval(4s)`, () => expect(EsQueryParser._roundInterval(4 * second)).to.be(`1s`)); + it(`roundInterval(4hr)`, () => expect(EsQueryParser._roundInterval(4 * hour)).to.be(`3h`)); + it(`getTimeBound`, () => expect(create(1000, 2000)._getTimeBound({}, `min`)).to.be(1000)); + it(`getTimeBound(shift 2d)`, () => expect(create(5, 2000)._getTimeBound({ shift: 2 }, `min`)).to.be(5 + 2 * day)); + it(`getTimeBound(shift -2hr)`, () => expect(create(10 * day, 20 * day) + ._getTimeBound({ shift: -2, unit: `h` }, `min`)) + .to.be(10 * day - 2 * hour)); + it(`createRangeFilter({})`, () => { + const obj = {}; + expect(create(1000, 2000)._createRangeFilter(obj)) + .to.eql({ format: `epoch_millis`, gte: 1000, lte: 2000 }).and.to.be(obj); + }); + it(`createRangeFilter(shift 1s)`, () => { + const obj = { shift: 5, unit: 's' }; + expect(create(1000, 2000)._createRangeFilter(obj)) + .to.eql({ format: `epoch_millis`, gte: 6000, lte: 7000 }).and.to.be(obj); + }); + +}); + +describe(`EsQueryParser.injectQueryContextVars`, () => { + + function test(obj, expected, ctx) { + return () => { + create(rangeStart, rangeEnd, ctx)._injectContextVars(obj, true); + expect(obj).to.eql(expected); + }; + } + + it(`empty`, test({}, {})); + it(`simple`, () => { + const obj = { a: { c: 10 }, b: [{ d: 2 }, 4, 5], c: [], d: {} }; + test(obj, _.cloneDeep(obj)); + }); + it(`must clause empty`, test({ arr: ['%dashboard_context-must_clause%'] }, { arr: [] }, {})); + it(`must clause arr`, test({ arr: ['%dashboard_context-must_clause%'] }, { arr: [...ctxArr.bool.must] }, ctxArr)); + it(`must clause obj`, test({ arr: ['%dashboard_context-must_clause%'] }, { arr: [ctxObj.bool.must] }, ctxObj)); + it(`mixed clause arr`, test( + { arr: [1, '%dashboard_context-must_clause%', 2, '%dashboard_context-must_not_clause%'] }, + { arr: [1, ...ctxArr.bool.must, 2, ...ctxArr.bool.must_not] }, ctxArr)); + it(`mixed clause obj`, test( + { arr: ['%dashboard_context-must_clause%', 1, '%dashboard_context-must_not_clause%', 2] }, + { arr: [ctxObj.bool.must, 1, ctxObj.bool.must_not, 2] }, ctxObj)); + it(`%autointerval% = true`, test({ interval: { '%autointerval%': true } }, { interval: `1h` }, ctxObj)); + it(`%autointerval% = 10`, test({ interval: { '%autointerval%': 10 } }, { interval: `3h` }, ctxObj)); + it(`%timefilter% = min`, test({ a: { '%timefilter%': 'min' } }, { a: rangeStart })); + it(`%timefilter% = max`, test({ a: { '%timefilter%': 'max' } }, { a: rangeEnd })); + it(`%timefilter% = true`, test( + { a: { '%timefilter%': true } }, + { a: { format: `epoch_millis`, gte: rangeStart, lte: rangeEnd } })); +}); + +describe(`EsQueryParser.parseEsRequest`, () => { + function test(req, ctx, expected) { + return () => { + create(rangeStart, rangeEnd, ctx).parseUrl({}, req); + expect(req).to.eql(expected); + }; + } + + it(`%context_query%=true`, test( + { index: '_all', '%context_query%': true }, ctxArr, + { index: '_all', body: { query: ctxArr } })); + + it(`%context%=true`, test( + { index: '_all', '%context%': true }, ctxArr, + { index: '_all', body: { query: ctxArr } })); + + const expectedForCtxAndTimefield = { + index: '_all', + body: { + query: { + bool: { + must: [ + { match_all: { c: 3 } }, + { range: { abc: { format: 'epoch_millis', gte: rangeStart, lte: rangeEnd } } } + ], + must_not: [{ 'd': 4 }] + } + } + } + }; + + it(`%context_query%='abc'`, test( + { index: '_all', '%context_query%': 'abc' }, ctxArr, expectedForCtxAndTimefield)); + + it(`%context%=true, %timefield%='abc'`, test( + { index: '_all', '%context%': true, '%timefield%': 'abc' }, ctxArr, expectedForCtxAndTimefield)); + + it(`%timefield%='abc'`, test({ index: '_all', '%timefield%': 'abc' }, ctxArr, + { + index: '_all', + body: { query: { range: { abc: { format: 'epoch_millis', gte: rangeStart, lte: rangeEnd } } } } + } + )); + + it(`no esRequest`, test({ index: '_all' }, ctxArr, { index: '_all', body: {} })); + + it(`esRequest`, test({ index: '_all', body: { query: 2 } }, ctxArr, { + index: '_all', + body: { query: 2 } + })); +}); diff --git a/src/core_plugins/vega/public/data_model/__tests__/search_cache.js b/src/core_plugins/vega/public/data_model/__tests__/search_cache.js new file mode 100644 index 0000000000000..e5a1c165454a0 --- /dev/null +++ b/src/core_plugins/vega/public/data_model/__tests__/search_cache.js @@ -0,0 +1,54 @@ +import expect from 'expect.js'; +import { SearchCache } from '../search_cache'; + +describe(`SearchCache`, () => { + + class FauxEs { + constructor() { + // contains all request batches, separated by 0 + this.searches = []; + } + + async search(request) { + this.searches.push(request); + return { req: request }; + } + } + + const request1 = { body: 'b1' }; + const expected1 = { req: { body: 'b1' } }; + const request2 = { body: 'b2' }; + const expected2 = { req: { body: 'b2' } }; + const request3 = { body: 'b3' }; + const expected3 = { req: { body: 'b3' } }; + + it(`sequence`, async () => { + const sc = new SearchCache(new FauxEs()); + + // empty request + let res = await sc.search([]); + expect(res).to.eql([]); + expect(sc._es.searches).to.eql([]); + + // single request + res = await sc.search([request1]); + expect(res).to.eql([expected1]); + expect(sc._es.searches).to.eql([request1]); + + // repeat the same search, use array notation + res = await sc.search([request1]); + expect(res).to.eql([expected1]); + expect(sc._es.searches).to.eql([request1]); // no new entries + + // new single search + res = await sc.search([request2]); + expect(res).to.eql([expected2]); + expect(sc._es.searches).to.eql([request1, request2]); + + // multiple search, some new, some old + res = await sc.search([request1, request3, request2]); + expect(res).to.eql([expected1, expected3, expected2]); + expect(sc._es.searches).to.eql([request1, request2, request3]); + }); + +}); diff --git a/src/core_plugins/vega/public/data_model/__tests__/time_cache.js b/src/core_plugins/vega/public/data_model/__tests__/time_cache.js new file mode 100644 index 0000000000000..6684a12c075cd --- /dev/null +++ b/src/core_plugins/vega/public/data_model/__tests__/time_cache.js @@ -0,0 +1,82 @@ +import expect from 'expect.js'; +import { TimeCache } from '../time_cache'; + +describe(`TimeCache`, () => { + + class FauxTimefilter { + constructor(min, max) { + // logs all requests + this.searches = []; + this.time = {}; + this.setTime(min, max); + this._accessCount = 0; + } + + setTime(min, max) { + this._min = min; + this._max = max; + } + + getBounds() { + this._accessCount++; + return { + min: { valueOf: () => this._min }, + max: { valueOf: () => this._max }, + }; + } + } + + class FauxTime { + constructor() { + this._time = 10000; + this._accessCount = 0; + } + + now() { + this._accessCount++; + return this._time; + } + + increment(inc) { + this._time += inc; + } + } + + it(`sequence`, async () => { + const timefilter = new FauxTimefilter(10000, 20000, 'quick'); + const tc = new TimeCache(timefilter, 5000); + const time = new FauxTime(); + tc._now = () => time.now(); + + let timeAccess = 0; + let filterAccess = 0; + + // first call - gets bounds + expect(tc.getTimeBounds()).to.eql({ min: 10000, max: 20000 }); + expect(time._accessCount).to.be(++timeAccess); + expect(timefilter._accessCount).to.be(++filterAccess); + + // short diff, same result + time.increment(10); + timefilter.setTime(10010, 20010); + expect(tc.getTimeBounds()).to.eql({ min: 10000, max: 20000 }); + expect(time._accessCount).to.be(++timeAccess); + expect(timefilter._accessCount).to.be(filterAccess); + + // longer diff, gets bounds but returns original + time.increment(200); + timefilter.setTime(10210, 20210); + expect(tc.getTimeBounds()).to.eql({ min: 10000, max: 20000 }); + expect(time._accessCount).to.be(++timeAccess); + expect(timefilter._accessCount).to.be(++filterAccess); + + // long diff, new result + time.increment(10000); + timefilter.setTime(20220, 30220); + expect(tc.getTimeBounds()).to.eql({ min: 20220, max: 30220 }); + expect(time._accessCount).to.be(++timeAccess); + expect(timefilter._accessCount).to.be(++filterAccess); + + }); + +}); diff --git a/src/core_plugins/vega/public/data_model/__tests__/vega_parser.js b/src/core_plugins/vega/public/data_model/__tests__/vega_parser.js new file mode 100644 index 0000000000000..3d52a11c08e70 --- /dev/null +++ b/src/core_plugins/vega/public/data_model/__tests__/vega_parser.js @@ -0,0 +1,192 @@ +import _ from 'lodash'; +import expect from 'expect.js'; +import { VegaParser } from '../vega_parser'; + +describe(`VegaParser._setDefaultValue`, () => { + + function test(spec, expected, ...params) { + return () => { + const vp = new VegaParser(spec); + vp._setDefaultValue(...params); + expect(vp.spec).to.eql(expected); + expect(vp.warnings).to.have.length(0); + }; + } + + it(`empty`, test({}, { config: { test: 42 } }, 42, 'config', 'test')); + it(`exists`, test({ config: { test: 42 } }, { config: { test: 42 } }, 1, 'config', 'test')); + it(`exists non-obj`, test({ config: false }, { config: false }, 42, 'config', 'test')); + +}); + +describe(`VegaParser._setDefaultColors`, () => { + + function test(spec, isVegaLite, expected) { + return () => { + const vp = new VegaParser(spec); + vp.isVegaLite = isVegaLite; + vp._setDefaultColors(); + expect(vp.spec).to.eql(expected); + expect(vp.warnings).to.have.length(0); + }; + } + + it(`vegalite`, test({}, true, { + config: { + range: { category: { scheme: 'elastic' } }, + mark: { color: '#00A69B' } + } + })); + + it(`vega`, test({}, false, { + config: { + range: { category: { scheme: 'elastic' } }, + arc: { fill: '#00A69B' }, + area: { fill: '#00A69B' }, + line: { stroke: '#00A69B' }, + path: { stroke: '#00A69B' }, + rect: { fill: '#00A69B' }, + rule: { stroke: '#00A69B' }, + shape: { stroke: '#00A69B' }, + symbol: { fill: '#00A69B' }, + trail: { fill: '#00A69B' } + } + })); + +}); + +describe('VegaParser._resolveEsQueries', () => { + function test(spec, expected, warnCount) { + return async () => { + const vp = new VegaParser(spec, { search: async () => [[42]] }, 0, 0, { + getFileLayers: async () => [{ name: 'file1', url: 'url1' }] + }); + await vp._resolveDataUrls(); + expect(vp.spec).to.eql(expected); + expect(vp.warnings).to.have.length(warnCount || 0); + }; + } + + it('no data', test({}, {})); + it('no data2', test({ a: 1 }, { a: 1 })); + it('non-es data', test({ data: { a: 10 } }, { data: { a: 10 } })); + it('es', test({ data: { url: { index: 'a' }, x: 1 } }, { data: { values: [42], x: 1 } })); + it('es', test({ data: { url: { '%type%': 'elasticsearch', index: 'a' } } }, { data: { values: [42] } })); + it('es arr', test({ arr: [{ data: { url: { index: 'a' }, x: 1 } }] }, { arr: [{ data: { values: [42], x: 1 } }] })); + it('emsfile', test({ data: { url: { '%type%': 'emsfile', name: 'file1' } } }, { data: { url: 'url1' } })); +}); + +describe('VegaParser._parseSchema', () => { + function test(schema, isVegaLite, warningCount) { + return () => { + const vp = new VegaParser({ $schema: schema }); + expect(vp._parseSchema()).to.be(isVegaLite); + expect(vp.spec).to.eql({ $schema: schema }); + expect(vp.warnings).to.have.length(warningCount); + }; + } + + it('no schema', () => { + const vp = new VegaParser({}); + expect(vp._parseSchema()).to.be(false); + expect(vp.spec).to.eql({ $schema: 'https://vega.github.io/schema/vega/v3.0.json' }); + expect(vp.warnings).to.have.length(1); + }); + it('vega', test('https://vega.github.io/schema/vega/v3.0.json', false, 0)); + it('vega old', test('https://vega.github.io/schema/vega/v4.0.json', false, 1)); + it('vega-lite', test('https://vega.github.io/schema/vega-lite/v2.0.json', true, 0)); + it('vega-lite old', test('https://vega.github.io/schema/vega-lite/v3.0.json', true, 1)); +}); + +describe('VegaParser._parseMapConfig', () => { + function test(config, expected, warnCount) { + return () => { + const vp = new VegaParser(); + vp._config = config; + expect(vp._parseMapConfig()).to.eql(expected); + expect(vp.warnings).to.have.length(warnCount); + }; + } + + it('empty', test({}, { + delayRepaint: true, + latitude: 0, + longitude: 0, + mapStyle: 'default', + zoomControl: true + }, 0)); + + it('filled', test({ + delayRepaint: true, + latitude: 0, + longitude: 0, + mapStyle: 'default', + zoomControl: true, + maxBounds: [1, 2, 3, 4], + }, { + delayRepaint: true, + latitude: 0, + longitude: 0, + mapStyle: 'default', + zoomControl: true, + maxBounds: [1, 2, 3, 4], + }, 0)); + + it('warnings', test({ + delayRepaint: true, + latitude: 0, + longitude: 0, + zoom: 'abc', // ignored + mapStyle: 'abc', + zoomControl: 'abc', + maxBounds: [2, 3, 4], + }, { + delayRepaint: true, + latitude: 0, + longitude: 0, + mapStyle: 'default', + zoomControl: true, + }, 4)); +}); + +describe('VegaParser._parseConfig', () => { + function test(spec, expectedConfig, expectedSpec, warnCount) { + return async () => { + expectedSpec = expectedSpec || _.cloneDeep(spec); + const vp = new VegaParser(spec); + const config = await vp._parseConfig(); + expect(config).to.eql(expectedConfig); + expect(vp.spec).to.eql(expectedSpec); + expect(vp.warnings).to.have.length(warnCount || 0); + }; + } + + it('no config', test({}, {}, {})); + it('simple config', test({ config: { a: 1 } }, {})); + it('kibana config', test({ config: { kibana: { a: 1 } } }, { a: 1 }, { config: {} })); + it('_hostConfig', test({ _hostConfig: { a: 1 } }, { a: 1 }, {}, 1)); +}); + +describe('VegaParser._calcSizing', () => { + function test(spec, useResize, paddingWidth, paddingHeight, isVegaLite, expectedSpec, warnCount) { + return async () => { + expectedSpec = expectedSpec || _.cloneDeep(spec); + const vp = new VegaParser(spec); + vp.isVegaLite = !!isVegaLite; + vp._calcSizing(); + expect(vp.useResize).to.eql(useResize); + expect(vp.paddingWidth).to.eql(paddingWidth); + expect(vp.paddingHeight).to.eql(paddingHeight); + expect(vp.spec).to.eql(expectedSpec); + expect(vp.warnings).to.have.length(warnCount || 0); + }; + } + + it('no size', test({ autosize: {} }, false, 0, 0)); + it('fit', test({ autosize: 'fit' }, true, 0, 0)); + it('fit obj', test({ autosize: { type: 'fit' } }, true, 0, 0)); + it('padding const', test({ autosize: 'fit', padding: 10 }, true, 20, 20)); + it('padding obj', test({ autosize: 'fit', padding: { left: 5, bottom: 7, right: 6, top: 8 } }, true, 11, 15)); + it('width height', test({ autosize: 'fit', width: 1, height: 2 }, true, 0, 0, false, false, 1)); + it('VL width height', test({ autosize: 'fit', width: 1, height: 2 }, true, 0, 0, true, { autosize: 'fit' }, 0)); +}); diff --git a/src/core_plugins/vega/public/data_model/ems_file_parser.js b/src/core_plugins/vega/public/data_model/ems_file_parser.js new file mode 100644 index 0000000000000..f576d1788cf0a --- /dev/null +++ b/src/core_plugins/vega/public/data_model/ems_file_parser.js @@ -0,0 +1,45 @@ +/** + * This class processes all Vega spec customizations, + * converting url object parameters into query results. + */ +export class EmsFileParser { + + constructor(serviceSettings) { + this._serviceSettings = serviceSettings; + } + + // noinspection JSMethodCanBeStatic + /** + * Update request object, expanding any context-aware keywords + */ + parseUrl(obj, url) { + if (typeof url.name !== 'string') { + throw new Error(`data.url with {"%type%": "emsfile"} is missing the "name" of the file`); + } + // Optimization: so initiate remote request as early as we know that we will need it + if (!this._fileLayersP) { + this._fileLayersP = this._serviceSettings.getFileLayers(); + } + return { obj, name: url.name }; + } + + /** + * Process items generated by parseUrl() + * @param {object[]} requests each object is generated by parseUrl() + * @returns {Promise} + */ + async populateData(requests) { + if (requests.length === 0) return; + + const layers = await this._fileLayersP; + + for (const { obj, name } of requests) { + const foundLayer = layers.find(v => v.name === name); + if (!foundLayer) { + throw new Error(`emsfile ${JSON.stringify(name)} does not exist`); + } + obj.url = foundLayer.url; + } + } + +} diff --git a/src/core_plugins/vega/public/data_model/es_query_parser.js b/src/core_plugins/vega/public/data_model/es_query_parser.js new file mode 100644 index 0000000000000..77b7ba04a2058 --- /dev/null +++ b/src/core_plugins/vega/public/data_model/es_query_parser.js @@ -0,0 +1,286 @@ +import _ from 'lodash'; + +const TIMEFILTER = '%timefilter%'; +const AUTOINTERVAL = '%autointerval%'; +const MUST_CLAUSE = '%dashboard_context-must_clause%'; +const MUST_NOT_CLAUSE = '%dashboard_context-must_not_clause%'; + +// These values may appear in the 'url': { ... } object +const LEGACY_CONTEXT = '%context_query%'; +const CONTEXT = '%context%'; +const TIMEFIELD = '%timefield%'; + +/** + * This class parses ES requests specified in the data.url objects. + */ +export class EsQueryParser { + + constructor(timeCache, searchCache, dashboardContext, onWarning) { + this._timeCache = timeCache; + this._searchCache = searchCache; + this._dashboardContext = dashboardContext; + this._onWarning = onWarning; + } + + // noinspection JSMethodCanBeStatic + /** + * Update request object, expanding any context-aware keywords + */ + parseUrl(dataObject, url) { + let body = url.body; + let context = url[CONTEXT]; + delete url[CONTEXT]; + let timefield = url[TIMEFIELD]; + delete url[TIMEFIELD]; + let usesContext = context !== undefined || timefield !== undefined; + + if (body === undefined) { + url.body = body = {}; + } else if (!_.isPlainObject(body)) { + throw new Error('url.body must be an object'); + } + + // Migrate legacy %context_query% into context & timefield values + const legacyContext = url[LEGACY_CONTEXT]; + delete url[LEGACY_CONTEXT]; + if (legacyContext !== undefined) { + if (body.query !== undefined) { + throw new Error(`Data url must not have legacy "${LEGACY_CONTEXT}" and "body.query" values at the same time`); + } else if (usesContext) { + throw new Error(`Data url must not have "${LEGACY_CONTEXT}" together with "${CONTEXT}" or "${TIMEFIELD}"`); + } else if (legacyContext !== true && (typeof legacyContext !== 'string' || legacyContext.length === 0)) { + throw new Error(`Legacy "${LEGACY_CONTEXT}" can either be true (ignores time range picker), ` + + 'or it can be the name of the time field, e.g. "@timestamp"'); + } + + usesContext = true; + context = true; + let result = `"url": {"${CONTEXT}": true`; + if (typeof legacyContext === 'string') { + timefield = legacyContext; + result += `, "${TIMEFIELD}": ${JSON.stringify(timefield)}`; + } + result += '}'; + + this._onWarning( + `Legacy "url": {"${LEGACY_CONTEXT}": ${JSON.stringify(legacyContext)}} should change to ${result}`); + } + + if (body.query !== undefined) { + if (usesContext) { + throw new Error(`url.${CONTEXT} and url.${TIMEFIELD} must not be used when url.body.query is set`); + } + this._injectContextVars(body.query, true); + } else if (usesContext) { + + if (timefield) { + // Inject range filter based on the timefilter values + body.query = { range: { [timefield]: this._createRangeFilter({ [TIMEFILTER]: true }) } }; + } + + if (context) { + // Use dashboard context + const newQuery = this._dashboardContext(); + if (timefield) { + newQuery.bool.must.push(body.query); + } + body.query = newQuery; + } + } + + this._injectContextVars(body.aggs, false); + return { dataObject, url }; + } + + /** + * Process items generated by parseUrl() + * @param {object[]} requests each object is generated by parseUrl() + * @returns {Promise} + */ + async populateData(requests) { + const esSearches = requests.map((r) => r.url); + + const results = await this._searchCache.search(esSearches); + + for (let i = 0; i < requests.length; i++) { + requests[i].dataObject.values = results[i]; + } + } + + /** + * Modify ES request by processing magic keywords + * @param {*} obj + * @param {boolean} isQuery - if true, the `obj` belongs to the req's query portion + */ + _injectContextVars(obj, isQuery) { + if (obj && typeof obj === 'object') { + if (Array.isArray(obj)) { + // For arrays, replace MUST_CLAUSE and MUST_NOT_CLAUSE string elements + for (let pos = 0; pos < obj.length;) { + const item = obj[pos]; + if (isQuery && (item === MUST_CLAUSE || item === MUST_NOT_CLAUSE)) { + const ctxTag = item === MUST_CLAUSE ? 'must' : 'must_not'; + const ctx = this._dashboardContext(); + if (ctx && ctx.bool && ctx.bool[ctxTag]) { + if (Array.isArray(ctx.bool[ctxTag])) { + // replace one value with an array of values + obj.splice(pos, 1, ...ctx.bool[ctxTag]); + pos += ctx.bool[ctxTag].length; + } else { + obj[pos++] = ctx.bool[ctxTag]; + } + } else { + obj.splice(pos, 1); // remove item, keep pos at the same position + } + } else { + this._injectContextVars(item, isQuery); + pos++; + } + } + } else { + for (const prop of Object.keys(obj)) { + const subObj = obj[prop]; + if (!subObj || typeof obj !== 'object') continue; + + // replace "interval": { "%autointerval%": true|integer } with + // auto-generated range based on the timepicker + if (prop === 'interval' && subObj[AUTOINTERVAL]) { + let size = subObj[AUTOINTERVAL]; + if (size === true) { + size = 50; // by default, try to get ~80 values + } else if (typeof size !== 'number') { + throw new Error(`"${AUTOINTERVAL}" must be either true or a number`); + } + const bounds = this._timeCache.getTimeBounds(); + obj.interval = EsQueryParser._roundInterval((bounds.max - bounds.min) / size); + continue; + } + + // handle %timefilter% + switch (subObj[TIMEFILTER]) { + case 'min': + case 'max': + // Replace {"%timefilter%": "min|max", ...} object with a timestamp + obj[prop] = this._getTimeBound(subObj, subObj[TIMEFILTER]); + continue; + case true: + // Replace {"%timefilter%": true, ...} object with the "range" object + this._createRangeFilter(subObj); + continue; + case undefined: + this._injectContextVars(subObj, isQuery); + continue; + default: + throw new Error(`"${TIMEFILTER}" property must be set to true, "min", or "max"`); + } + } + } + } + } + + /** + * replaces given object that contains `%timefilter%` key with the timefilter bounds and optional shift & unit parameters + * @param {object} obj + * @return {object} + */ + _createRangeFilter(obj) { + obj.gte = this._getTimeBound(obj, 'min'); + obj.lte = this._getTimeBound(obj, 'max'); + obj.format = 'epoch_millis'; + delete obj[TIMEFILTER]; + delete obj.shift; + delete obj.unit; + return obj; + } + + /** + * + * @param {object} opts + * @param {number} [opts.shift] + * @param {string} [opts.unit] + * @param {'min'|'max'} type + * @returns {*} + */ + _getTimeBound(opts, type) { + const bounds = this._timeCache.getTimeBounds(); + let result = bounds[type]; + + if (opts.shift) { + const shift = opts.shift; + if (typeof shift !== 'number') { + throw new Error('shift must be a numeric value'); + } + let multiplier; + switch (opts.unit || 'd') { + case 'w': + case 'week': + multiplier = 1000 * 60 * 60 * 24 * 7; + break; + case 'd': + case 'day': + multiplier = 1000 * 60 * 60 * 24; + break; + case 'h': + case 'hour': + multiplier = 1000 * 60 * 60; + break; + case 'm': + case 'minute': + multiplier = 1000 * 60; + break; + case 's': + case 'second': + multiplier = 1000; + break; + default: + throw new Error('Unknown unit value. Must be one of: [week, day, hour, minute, second]'); + } + result += shift * multiplier; + } + + return result; + } + + /** + * Adapted from src/core_plugins/timelion/common/lib/calculate_interval.js + * @param interval (ms) + * @returns {string} + */ + static _roundInterval(interval) { + switch (true) { + case (interval <= 500): // <= 0.5s + return '100ms'; + case (interval <= 5000): // <= 5s + return '1s'; + case (interval <= 7500): // <= 7.5s + return '5s'; + case (interval <= 15000): // <= 15s + return '10s'; + case (interval <= 45000): // <= 45s + return '30s'; + case (interval <= 180000): // <= 3m + return '1m'; + case (interval <= 450000): // <= 9m + return '5m'; + case (interval <= 1200000): // <= 20m + return '10m'; + case (interval <= 2700000): // <= 45m + return '30m'; + case (interval <= 7200000): // <= 2h + return '1h'; + case (interval <= 21600000): // <= 6h + return '3h'; + case (interval <= 86400000): // <= 24h + return '12h'; + case (interval <= 604800000): // <= 1w + return '24h'; + case (interval <= 1814400000): // <= 3w + return '1w'; + case (interval < 3628800000): // < 2y + return '30d'; + default: + return '1y'; + } + } + +} diff --git a/src/core_plugins/vega/public/data_model/search_cache.js b/src/core_plugins/vega/public/data_model/search_cache.js new file mode 100644 index 0000000000000..ad9f6653c7dac --- /dev/null +++ b/src/core_plugins/vega/public/data_model/search_cache.js @@ -0,0 +1,30 @@ +import LruCache from 'lru-cache'; + +export class SearchCache { + + constructor(es, cacheOpts) { + this._es = es; + this._cache = new LruCache(cacheOpts); + } + + /** + * Execute multiple searches, possibly combining the results of the cached searches + * with the new ones already in cache + * @param {object[]} requests array of search requests + */ + search(requests) { + const promises = []; + + for (const request of requests) { + const key = JSON.stringify(request); + let pending = this._cache.get(key); + if (pending === undefined) { + pending = this._es.search(request); + this._cache.set(key, pending); + } + promises.push(pending); + } + + return Promise.all(promises); + } +} diff --git a/src/core_plugins/vega/public/data_model/time_cache.js b/src/core_plugins/vega/public/data_model/time_cache.js new file mode 100644 index 0000000000000..df82b90a4bfda --- /dev/null +++ b/src/core_plugins/vega/public/data_model/time_cache.js @@ -0,0 +1,73 @@ +/** + * Optimization caching - always return the same value if queried within this time + * @type {number} + */ +const AlwaysCacheMaxAge = 40; + +/** + * This class caches timefilter's bounds to minimize number of server requests + */ +export class TimeCache { + + constructor(timefilter, maxAge) { + this._timefilter = timefilter; + this._maxAge = maxAge; + this._cachedBounds = null; + this._cacheTS = 0; + } + + // Simplifies unit testing + // noinspection JSMethodCanBeStatic + _now() { + return Date.now(); + } + + /** + * Get cached time range values + * @returns {{min: number, max: number}} + */ + getTimeBounds() { + const ts = this._now(); + + let bounds; + if (this._cachedBounds) { + const diff = ts - this._cacheTS; + + // For very rapid usage (multiple calls within a few milliseconds) + // Avoids expensive time parsing + if (diff < AlwaysCacheMaxAge) { + return this._cachedBounds; + } + + // If the time is relative, mode hasn't changed, and time hasn't changed more than maxAge, + // return old time to avoid multiple near-identical server calls + if (diff < this._maxAge) { + bounds = this._getBounds(); + if ( + (Math.abs(bounds.min - this._cachedBounds.min) < this._maxAge) && + (Math.abs(bounds.max - this._cachedBounds.max) < this._maxAge) + ) { + return this._cachedBounds; + } + } + } + + this._cacheTS = ts; + this._cachedBounds = bounds || this._getBounds(); + + return this._cachedBounds; + } + + /** + * Get parsed min/max values + * @returns {{min: number, max: number}} + * @private + */ + _getBounds() { + const bounds = this._timefilter.getBounds(); + return { + min: bounds.min.valueOf(), + max: bounds.max.valueOf() + }; + } +} diff --git a/src/core_plugins/vega/public/data_model/url_parser.js b/src/core_plugins/vega/public/data_model/url_parser.js new file mode 100644 index 0000000000000..629e8d3468e6c --- /dev/null +++ b/src/core_plugins/vega/public/data_model/url_parser.js @@ -0,0 +1,39 @@ +import $ from 'jquery'; + +/** + * This class processes all Vega spec customizations, + * converting url object parameters into query results. + */ +export class UrlParser { + + constructor(onWarning) { + this._onWarning = onWarning; + } + + // noinspection JSMethodCanBeStatic + /** + * Update request object + */ + parseUrl(obj, urlObj) { + let url = urlObj.url; + if (!url) { + throw new Error(`data.url requires a url parameter in a form 'https://example.org/path/subpath'`); + } + + const query = urlObj.query; + if (!query) { + this._onWarning(`Using a "url": {"%type%": "url", "url": ...} should have a "query" sub-object`); + } else { + url += (url.includes('?') ? '&' : '?') + $.param(query); + } + + obj.url = url; + } + + /** + * No-op - the url is already set during the parseUrl + */ + populateData() { + } + +} diff --git a/src/core_plugins/vega/public/data_model/utils.js b/src/core_plugins/vega/public/data_model/utils.js new file mode 100644 index 0000000000000..2d8b686e93f9f --- /dev/null +++ b/src/core_plugins/vega/public/data_model/utils.js @@ -0,0 +1,32 @@ +import compactStringify from 'json-stringify-pretty-compact'; + +export class Utils { + + /** + * If the 2nd array parameter in args exists, append it to the warning/error string value + */ + static formatWarningToStr(value) { + if (arguments.length >= 2) { + try { + if (typeof arguments[1] === 'string') { + value += `\n${arguments[1]}`; + } else { + value += '\n' + compactStringify(arguments[1], { maxLength: 70 }); + } + } catch (err) { + // ignore + } + } + return value; + } + + static formatErrorToStr(error) { + if (!error) { + error = 'ERR'; + } else if (error instanceof Error) { + error = error.message; + } + return Utils.formatWarningToStr(error, ...Array.from(arguments).slice(1)); + } + +} diff --git a/src/core_plugins/vega/public/data_model/vega_parser.js b/src/core_plugins/vega/public/data_model/vega_parser.js new file mode 100644 index 0000000000000..e56bf652e6e58 --- /dev/null +++ b/src/core_plugins/vega/public/data_model/vega_parser.js @@ -0,0 +1,390 @@ +import _ from 'lodash'; +import * as vega from 'vega'; +import * as vegaLite from 'vega-lite'; +import schemaParser from 'vega-schema-url-parser'; +import versionCompare from 'compare-versions'; +import { EsQueryParser } from './es_query_parser'; +import hjson from 'hjson'; +import { Utils } from './utils'; +import { EmsFileParser } from './ems_file_parser'; +import { UrlParser } from './url_parser'; + +const DEFAULT_SCHEMA = 'https://vega.github.io/schema/vega/v3.0.json'; + +const locToDirMap = { + left: 'row-reverse', + right: 'row', + top: 'column-reverse', + bottom: 'column' +}; + +// If there is no "%type%" parameter, use this parser +const DEFAULT_PARSER = 'elasticsearch'; + +export class VegaParser { + + constructor(spec, searchCache, timeCache, dashboardContext, serviceSettings) { + this.spec = spec; + this.hideWarnings = false; + this.error = undefined; + this.warnings = []; + + const onWarn = this._onWarning.bind(this); + this._urlParsers = { + elasticsearch: new EsQueryParser(timeCache, searchCache, dashboardContext, onWarn), + emsfile: new EmsFileParser(serviceSettings), + url: new UrlParser(onWarn), + }; + } + + async parseAsync() { + try { + await this._parseAsync(); + } catch (err) { + // if we reject current promise, it will use the standard Kibana error handling + this.error = Utils.formatErrorToStr(err); + } + return this; + } + + async _parseAsync() { + if (this.isVegaLite !== undefined) throw new Error(); + + if (typeof this.spec === 'string') { + this.spec = hjson.parse(this.spec, { legacyRoot: false }); + } + if (!_.isPlainObject(this.spec)) { + throw new Error('Invalid Vega spec'); + } + this.isVegaLite = this._parseSchema(); + this.useHover = !this.isVegaLite; + + this._config = this._parseConfig(); + this.hideWarnings = !!this._config.hideWarnings; + this.useMap = this._config.type === 'map'; + this.renderer = this._config.renderer === 'svg' ? 'svg' : 'canvas'; + + this._setDefaultColors(); + this._parseControlPlacement(this._config); + if (this.useMap) { + this.mapConfig = this._parseMapConfig(); + } else if (this.spec.autosize === undefined) { + // Default autosize should be fit, unless it's a map (leaflet-vega handles that) + this.spec.autosize = { type: 'fit', contains: 'padding' }; + } + + await this._resolveDataUrls(); + + if (this.isVegaLite) { + this._compileVegaLite(); + } + + this._calcSizing(); + } + + /** + * Convert VegaLite to Vega spec + * @private + */ + _compileVegaLite() { + if (this.useMap) { + throw new Error('"_map" configuration is not compatible with vega-lite spec'); + } + this.vlspec = this.spec; + + const logger = vega.logger(vega.Warn); + logger.warn = this._onWarning.bind(this); + this.spec = vegaLite.compile(this.vlspec, logger).spec; + } + + /** + * Process graph size and padding + * @private + */ + _calcSizing() { + this.useResize = !this.useMap && (this.spec.autosize === 'fit' || this.spec.autosize.type === 'fit'); + + // Padding is not included in the width/height by default + this.paddingWidth = 0; + this.paddingHeight = 0; + if (this.useResize && this.spec.padding && this.spec.autosize.contains !== 'padding') { + if (typeof this.spec.padding === 'object') { + this.paddingWidth += (+this.spec.padding.left || 0) + (+this.spec.padding.right || 0); + this.paddingHeight += (+this.spec.padding.top || 0) + (+this.spec.padding.bottom || 0); + } else { + this.paddingWidth += 2 * (+this.spec.padding || 0); + this.paddingHeight += 2 * (+this.spec.padding || 0); + } + } + + if (this.useResize && (this.spec.width || this.spec.height)) { + if (this.isVegaLite) { + delete this.spec.width; + delete this.spec.height; + } else { + this._onWarning(`The 'width' and 'height' params are ignored with autosize=fit`); + } + } + } + + /** + * Calculate container-direction CSS property for binding placement + * @private + */ + _parseControlPlacement() { + this.containerDir = locToDirMap[this._config.controlsLocation]; + if (this.containerDir === undefined) { + if (this._config.controlsLocation === undefined) { + this.containerDir = 'column'; + } else { + throw new Error('Unrecognized controlsLocation value. Expecting one of ["' + + locToDirMap.keys().join('", "') + '"]'); + } + } + const dir = this._config.controlsDirection; + if (dir !== undefined && dir !== 'horizontal' && dir !== 'vertical') { + throw new Error('Unrecognized dir value. Expecting one of ["horizontal", "vertical"]'); + } + this.controlsDir = dir === 'horizontal' ? 'row' : 'column'; + } + + /** + * Parse {config: kibana: {...}} portion of the Vega spec (or root-level _hostConfig for backward compat) + * @returns {object} kibana config + * @private + */ + _parseConfig() { + let result; + if (this.spec._hostConfig !== undefined) { + result = this.spec._hostConfig; + delete this.spec._hostConfig; + if (!_.isPlainObject(result)) { + throw new Error('If present, _hostConfig must be an object'); + } + this._onWarning('_hostConfig has been deprecated. Use config.kibana instead.'); + } + if (_.isPlainObject(this.spec.config) && this.spec.config.kibana !== undefined) { + result = this.spec.config.kibana; + delete this.spec.config.kibana; + if (!_.isPlainObject(result)) { + throw new Error('If present, config.kibana must be an object'); + } + } + return result || {}; + } + + /** + * Parse map-specific configuration + * @returns {{mapStyle: *|string, delayRepaint: boolean, latitude: number, longitude: number, zoom, minZoom, maxZoom, zoomControl: *|boolean, maxBounds: *}} + * @private + */ + _parseMapConfig() { + const res = { + delayRepaint: this._config.delayRepaint === undefined ? true : this._config.delayRepaint, + }; + + const validate = (name, isZoom) => { + const val = this._config[name]; + if (val !== undefined) { + const parsed = Number.parseFloat(val); + if (Number.isFinite(parsed) && (!isZoom || (parsed >= 0 && parsed <= 30))) { + res[name] = parsed; + return; + } + this._onWarning(`config.kibana.${name} is not valid`); + } + if (!isZoom) res[name] = 0; + }; + + validate(`latitude`, false); + validate(`longitude`, false); + validate(`zoom`, true); + validate(`minZoom`, true); + validate(`maxZoom`, true); + + // `false` is a valid value + res.mapStyle = this._config.mapStyle === undefined ? `default` : this._config.mapStyle; + if (res.mapStyle !== `default` && res.mapStyle !== false) { + this._onWarning(`config.kibana.mapStyle may either be false or "default"`); + res.mapStyle = `default`; + } + + const zoomControl = this._config.zoomControl; + if (zoomControl === undefined) { + res.zoomControl = true; + } else if (typeof zoomControl !== 'boolean') { + this._onWarning('config.kibana.zoomControl must be a boolean value'); + res.zoomControl = true; + } else { + res.zoomControl = zoomControl; + } + + const maxBounds = this._config.maxBounds; + if (maxBounds !== undefined) { + if (!Array.isArray(maxBounds) || maxBounds.length !== 4 || + !maxBounds.every(v => typeof v === 'number' && Number.isFinite(v)) + ) { + this._onWarning(`config.kibana.maxBounds must be an array with four numbers`); + } else { + res.maxBounds = maxBounds; + } + } + + return res; + } + + /** + * Parse Vega schema element + * @returns {boolean} is this a VegaLite schema? + * @private + */ + _parseSchema() { + if (!this.spec.$schema) { + this._onWarning(`The input spec does not specify a "$schema", defaulting to "${DEFAULT_SCHEMA}"`); + this.spec.$schema = DEFAULT_SCHEMA; + } + + const schema = schemaParser(this.spec.$schema); + const isVegaLite = schema.library === 'vega-lite'; + const libVersion = isVegaLite ? vegaLite.version : vega.version; + + if (versionCompare(schema.version, libVersion) > 0) { + this._onWarning( + `The input spec uses ${schema.library} ${schema.version}, but ` + + `current version of ${schema.library} is ${libVersion}.` + ); + } + + return isVegaLite; + } + + /** + * Replace all instances of ES requests with raw values. + * Also handle any other type of url: {type: xxx, ...} + * @private + */ + async _resolveDataUrls() { + const pending = {}; + + this._findObjectDataUrls(this.spec, (obj) => { + const url = obj.url; + delete obj.url; + let type = url['%type%']; + delete url['%type%']; + if (type === undefined) { + type = DEFAULT_PARSER; + } + + const parser = this._urlParsers[type]; + if (parser === undefined) { + throw new Error(`url: {"%type%": "${type}"} is not supported`); + } + + let pendingArr = pending[type]; + if (pendingArr === undefined) { + pending[type] = pendingArr = []; + } + + pendingArr.push(parser.parseUrl(obj, url)); + }); + + const pendingParsers = Object.keys(pending); + if (pendingParsers.length > 0) { + // let each parser populate its data in parallel + await Promise.all(pendingParsers.map(type => this._urlParsers[type].populateData(pending[type]))); + } + } + + /** + * Recursively find and callback every instance of the data.url as an object + * @param {*} obj current location in the object tree + * @param {function({object})} onFind Call this function for all url objects + * @param {string} [key] field name of the current object + * @private + */ + _findObjectDataUrls(obj, onFind, key) { + if (Array.isArray(obj)) { + for (const elem of obj) { + this._findObjectDataUrls(elem, onFind, key); + } + } else if (_.isPlainObject(obj)) { + if (key === 'data' && _.isPlainObject(obj.url)) { + // Assume that any "data": {"url": {...}} is a request for data + if (obj.values !== undefined || obj.source !== undefined) { + throw new Error('Data must not have more than one of "url", "values", and "source"'); + } + onFind(obj); + } else { + for (const k of Object.keys(obj)) { + this._findObjectDataUrls(obj[k], onFind, k); + } + } + } + } + + /** + * Inject default colors into the spec.config + * @private + */ + _setDefaultColors() { + // Default category coloring to the Elastic color scheme + this._setDefaultValue({ scheme: 'elastic' }, 'config', 'range', 'category'); + + // Set default single color to match other Kibana visualizations + const defaultColor = '#00A69B'; + if (this.isVegaLite) { + // Vega-Lite: set default color, works for fill and strike -- config: { mark: { color: '#00A69B' }} + this._setDefaultValue(defaultColor, 'config', 'mark', 'color'); + } else { + // Vega - global mark has very strange behavior, must customize each mark type individually + // https://github.com/vega/vega/issues/1083 + // Don't set defaults if spec.config.mark.color or fill are set + if (!this.spec.config.mark || (this.spec.config.mark.color === undefined && this.spec.config.mark.fill === undefined)) { + this._setDefaultValue(defaultColor, 'config', 'arc', 'fill'); + this._setDefaultValue(defaultColor, 'config', 'area', 'fill'); + this._setDefaultValue(defaultColor, 'config', 'line', 'stroke'); + this._setDefaultValue(defaultColor, 'config', 'path', 'stroke'); + this._setDefaultValue(defaultColor, 'config', 'rect', 'fill'); + this._setDefaultValue(defaultColor, 'config', 'rule', 'stroke'); + this._setDefaultValue(defaultColor, 'config', 'shape', 'stroke'); + this._setDefaultValue(defaultColor, 'config', 'symbol', 'fill'); + this._setDefaultValue(defaultColor, 'config', 'trail', 'fill'); + } + } + } + + /** + * Set default value if it doesn't exist. + * Given an object, and an array of fields, ensure that obj.fld1.fld2. ... .fldN is set to value if it doesn't exist. + * @param {*} value + * @param {string} fields + * @private + */ + _setDefaultValue(value, ...fields) { + let o = this.spec; + for (let i = 0; i < fields.length - 1; i++) { + const field = fields[i]; + const subObj = o[field]; + if (subObj === undefined) { + o[field] = {}; + } else if (!_.isPlainObject(subObj)) { + return; + } + o = o[field]; + } + const lastField = fields[fields.length - 1]; + if (o[lastField] === undefined) { + o[lastField] = value; + } + } + + /** + * Add a warning to the warnings array + * @private + */ + _onWarning() { + if (!this.hideWarnings) { + this.warnings.push(Utils.formatWarningToStr(...arguments)); + } + } +} diff --git a/src/core_plugins/vega/public/default.spec.hjson b/src/core_plugins/vega/public/default.spec.hjson new file mode 100644 index 0000000000000..cc60832619290 --- /dev/null +++ b/src/core_plugins/vega/public/default.spec.hjson @@ -0,0 +1,100 @@ +{ +/* + +Welcome to Vega visualizations. Here you can design your own dataviz from scratch using a declarative language called Vega, or its simpler form Vega-Lite. In Vega, you have the full control of what data is loaded, even from multiple sources, how that data is transformed, and what visual elements are used to show it. Use help icon to view Vega examples, tutorials, and other docs. Use the wrench icon to reformat this text, or to remove comments. + +This example graph shows the document count in all indexes in the current time range. You might need to adjust the time filter in the upper right corner. +*/ + + $schema: https://vega.github.io/schema/vega-lite/v2.json + title: Event counts from all indexes + + // Define the data source + data: { + url: { +/* +An object instead of a string for the "url" param is treated as an Elasticsearch query. Anything inside this object is not part of the Vega language, but only understood by Kibana and Elasticsearch server. This query counts the number of documents per time interval, assuming you have a @timestamp field in your data. + +Kibana has a special handling for the fields surrounded by "%". They are processed before the the query is sent to Elasticsearch. This way the query becomes context aware, and can use the time range and the dashboard filters. +*/ + + // Apply dashboard context filters when set + %context%: true + // Filter the time picker (upper right corner) with this field + %timefield%: @timestamp + +/* +See .search() documentation for : https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/api-reference.html#api-search +*/ + + // Which index to search + index: _all + // Aggregate data by the time field into time buckets, counting the number of documents in each bucket. + body: { + aggs: { + time_buckets: { + date_histogram: { + // Use date histogram aggregation on @timestamp field + field: @timestamp + // The interval value will depend on the daterange picker (true), or use an integer to set an approximate bucket count + interval: {%autointerval%: true} + // Make sure we get an entire range, even if it has no data + extended_bounds: { + // Use the current time range's start and end + min: {%timefilter%: "min"} + max: {%timefilter%: "max"} + } + // Use this for linear (e.g. line, area) graphs. Without it, empty buckets will not show up + min_doc_count: 0 + } + } + } + // Speed up the response by only including aggregation results + size: 0 + } + } +/* +Elasticsearch will return results in this format: + +aggregations: { + time_buckets: { + buckets: [ + { + key_as_string: 2015-11-30T22:00:00.000Z + key: 1448920800000 + doc_count: 0 + }, + { + key_as_string: 2015-11-30T23:00:00.000Z + key: 1448924400000 + doc_count: 0 + } + ... + ] + } +} + +For our graph, we only need the list of bucket values. Use the format.property to discard everything else. +*/ + format: {property: "aggregations.time_buckets.buckets"} + } + + // "mark" is the graphics element used to show our data. Other mark values are: area, bar, circle, line, point, rect, rule, square, text, and tick. See https://vega.github.io/vega-lite/docs/mark.html + mark: line + + // "encoding" tells the "mark" what data to use and in what way. See https://vega.github.io/vega-lite/docs/encoding.html + encoding: { + x: { + // The "key" value is the timestamp in milliseconds. Use it for X axis. + field: key + type: temporal + axis: {title: false} // Customize X axis format + } + y: { + // The "doc_count" is the count per bucket. Use it for Y axis. + field: doc_count + type: quantitative + axis: {title: "Document count"} + } + } +} diff --git a/src/core_plugins/vega/public/vega.less b/src/core_plugins/vega/public/vega.less new file mode 100644 index 0000000000000..0b2a8e3b0040c --- /dev/null +++ b/src/core_plugins/vega/public/vega.less @@ -0,0 +1,79 @@ +.vega-main { + display: flex; + flex: 1 1 100%; + position: relative; + // align-self: flex-start; + + > .vega-messages { + position: absolute; + top: 0; + width: 100%; + margin: auto; + opacity: 0.8; + z-index: 1; + list-style: none; + + li { + margin: 0.5em; + } + + .vega-message-warn pre { + background: #ffff83; + } + + .vega-message-err pre { + background: #ffdddd; + } + + pre { + white-space: pre-wrap; + padding: 0.7em; + } + } + + .vega-view-container { + z-index: 0; + flex: 1 1 100%; + + display: block; + max-width: 100%; + max-height: 100%; + width: 100%; + height: 100%; + overflow: hidden; + } + + .vega-controls-container { + display: flex; + font-size: 14px; + line-height: 20px; + + > .vega-bind { + flex: 1; /*grow*/ + + .vega-bind-name { + display: inline-block; + width: 105px; + } + + input[type="range"] { + width: 120px; + color: rgb(157, 150, 142); + display: inline-block; + vertical-align: middle; + } + + label { + margin: 0 0 0 0.6em; + } + + select { + max-width: 200px; + } + + .vega-bind-radio label { + margin: 0 0.6em 0 0.3em; + } + } + } +} diff --git a/src/core_plugins/vega/public/vega_editor.less b/src/core_plugins/vega/public/vega_editor.less new file mode 100644 index 0000000000000..72755b7218af0 --- /dev/null +++ b/src/core_plugins/vega/public/vega_editor.less @@ -0,0 +1,42 @@ +@import (reference) "~ui/styles/variables"; + +.vis-type-vega { + .vis-editor-config { + // Vega controls its own padding, no need for additional padding + padding: 0 !important; + } +} + +visualization-editor { + + .vis-editor-canvas { + padding-left: 0; + } + + .vegaEditor { + display: flex; + flex: 1 1 auto; + position: relative; + + // If the screen too small, make sure the editor has some height + min-height: 18em; + @media (min-width: @screen-md-min) { + min-height: unset; + } + + #editor_actions { + position: absolute; + margin: 0.6em 1.3em; + z-index: 80; + + ul.dropdown-menu { + left: auto; + right: 0; + } + } + + #vegaAceEditor { + flex: 1 1 auto; + } + } +} diff --git a/src/core_plugins/vega/public/vega_editor_controller.js b/src/core_plugins/vega/public/vega_editor_controller.js new file mode 100644 index 0000000000000..86e00e16b7edd --- /dev/null +++ b/src/core_plugins/vega/public/vega_editor_controller.js @@ -0,0 +1,70 @@ +import compactStringify from 'json-stringify-pretty-compact'; +import hjson from 'hjson'; +import { Notifier } from 'ui/notify'; +import { uiModules } from 'ui/modules'; + +import 'ui/accessibility/kbn_ui_ace_keyboard_mode'; + +const module = uiModules.get('kibana/vega', ['kibana']); +module.controller('VegaEditorController', ($scope /*, kbnUiAceKeyboardModeService*/) => { + + const notify = new Notifier({ location: 'Vega' }); + + return new (class VegaEditorController { + constructor() { + $scope.aceLoaded = (editor) => { + editor.$blockScrolling = Infinity; + + const session = editor.getSession(); + session.setTabSize(2); + session.setUseSoftTabs(true); + + this.aceEditor = editor; + }; + + $scope.formatJson = (event) => { + this._format(event, compactStringify, { + maxLength: this._getCodeWidth(), + }); + }; + + $scope.formatHJson = (event) => { + this._format(event, hjson.stringify, { + condense: this._getCodeWidth(), + bracesSameLine: true, + keepWsc: true, + }); + }; + } + + _getCodeWidth() { + return this.aceEditor.getSession().getWrapLimit(); + } + + _format(event, stringify, opts) { + event.preventDefault(); + + let newSpec; + try { + const spec = hjson.parse(this.aceEditor.getSession().doc.getValue(), { legacyRoot: false, keepWsc: true }); + newSpec = stringify(spec, opts); + } catch (err) { + // This is a common case - user tries to format an invalid HJSON text + notify.error(err); + return; + } + + // ui-ace only accepts changes from the editor when they + // happen outside of a digest cycle + // Per @spalger, we used $$postDigest() instead of setTimeout(() => {}, 0) + // because it better described the intention. + $scope.$$postDigest(() => { + // set the new value to the session doc so that it + // is treated as an edit by ace: ace adds it to the + // undo stack and emits it as a change like all + // other edits + this.aceEditor.getSession().doc.setValue(newSpec); + }); + } + })(); +}); diff --git a/src/core_plugins/vega/public/vega_editor_template.html b/src/core_plugins/vega/public/vega_editor_template.html new file mode 100644 index 0000000000000..99241f6cab4be --- /dev/null +++ b/src/core_plugins/vega/public/vega_editor_template.html @@ -0,0 +1,76 @@ +
+
+ +
+ + + + + + + + +
+ +
diff --git a/src/core_plugins/vega/public/vega_request_handler.js b/src/core_plugins/vega/public/vega_request_handler.js new file mode 100644 index 0000000000000..20108de5cebcd --- /dev/null +++ b/src/core_plugins/vega/public/vega_request_handler.js @@ -0,0 +1,22 @@ +import { VegaParser } from './data_model/vega_parser'; +import { dashboardContextProvider } from 'plugins/kibana/dashboard/dashboard_context'; +import { SearchCache } from './data_model/search_cache'; +import { TimeCache } from './data_model/time_cache'; + +export function VegaRequestHandlerProvider(Private, es, timefilter, serviceSettings) { + + const dashboardContext = Private(dashboardContextProvider); + const searchCache = new SearchCache(es, { max: 10, maxAge: 4 * 1000 }); + const timeCache = new TimeCache(timefilter, 3 * 1000); + + return { + + name: 'vega', + + handler(vis) { + const vp = new VegaParser(vis.params.spec, searchCache, timeCache, dashboardContext, serviceSettings); + return vp.parseAsync(); + } + + }; +} diff --git a/src/core_plugins/vega/public/vega_type.js b/src/core_plugins/vega/public/vega_type.js new file mode 100644 index 0000000000000..ed96dbb11240e --- /dev/null +++ b/src/core_plugins/vega/public/vega_type.js @@ -0,0 +1,42 @@ +import { VisTypesRegistryProvider } from 'ui/registry/vis_types'; +import { VisFactoryProvider } from 'ui/vis/vis_factory'; +import { CATEGORY } from 'ui/vis/vis_category'; +import { DefaultEditorSize } from 'ui/vis/editor_size'; + +import { VegaRequestHandlerProvider } from './vega_request_handler'; +import { VegaVisualizationProvider } from './vega_visualization'; + +import './vega.less'; + +// Editor-specific code +import 'brace/mode/hjson'; +import 'brace/ext/searchbox'; +import './vega_editor.less'; +import './vega_editor_controller'; +import vegaEditorTemplate from './vega_editor_template.html'; +import defaultSpec from '!!raw-loader!./default.spec.hjson'; + +VisTypesRegistryProvider.register((Private) => { + const VisFactory = Private(VisFactoryProvider); + const vegaRequestHandler = Private(VegaRequestHandlerProvider).handler; + const VegaVisualization = Private(VegaVisualizationProvider); + + return VisFactory.createBaseVisualization({ + name: 'vega', + title: 'Vega', + description: 'Create custom visualizations using Vega and VegaLite', + icon: 'fa-code', + category: CATEGORY.OTHER, + visConfig: { defaults: { spec: defaultSpec } }, + editorConfig: { + optionsTemplate: vegaEditorTemplate, + enableAutoApply: true, + defaultSize: DefaultEditorSize.MEDIUM, + }, + visualization: VegaVisualization, + requestHandler: vegaRequestHandler, + responseHandler: 'none', + options: { showIndexSelection: false }, + stage: 'lab', + }); +}); diff --git a/src/core_plugins/vega/public/vega_view/vega_base_view.js b/src/core_plugins/vega/public/vega_view/vega_base_view.js new file mode 100644 index 0000000000000..350ceb34170f0 --- /dev/null +++ b/src/core_plugins/vega/public/vega_view/vega_base_view.js @@ -0,0 +1,178 @@ +import $ from 'jquery'; +import * as vega from 'vega'; +import * as vegaLite from 'vega-lite'; +import { Utils } from '../data_model/utils'; + +//https://github.com/elastic/kibana/issues/13327 +vega.scheme('elastic', + ['#00B3A4', '#3185FC', '#DB1374', '#490092', '#FEB6DB', '#F98510', '#E6C220', '#BFA180', '#920000', '#461A0A'] +); + +export class VegaBaseView { + constructor(vegaConfig, editorMode, parentEl, vegaParser, serviceSettings) { + this._vegaConfig = vegaConfig; + this._editorMode = editorMode; + this._$parentEl = $(parentEl); + this._parser = vegaParser; + this._serviceSettings = serviceSettings; + this._view = null; + this._vegaViewConfig = null; + this._$messages = null; + this._destroyHandlers = []; + this._initialized = false; + } + + async init() { + if (this._initialized) throw new Error(); // safety + this._initialized = true; + + try { + this._$parentEl.empty() + .addClass('vega-main') + .css('flex-direction', this._parser.containerDir); + + // bypass the onWarn warning checks - in some cases warnings may still need to be shown despite being disabled + for (const warn of this._parser.warnings) { + this._addMessage('warn', warn); + } + + if (this._parser.error) { + this._addMessage('err', this._parser.error); + return; + } + + this._$container = $('
') + .appendTo(this._$parentEl); + this._$controls = $('
') + .css('flex-direction', this._parser.controlsDir) + .appendTo(this._$parentEl); + + this._addDestroyHandler(() => { + this._$container.remove(); + this._$container = null; + this._$controls.remove(); + this._$controls = null; + if (this._$messages) { + this._$messages.remove(); + this._$messages = null; + } + }); + + this._vegaViewConfig = { + logLevel: vega.Warn, + renderer: this._parser.renderer, + }; + if (!this._vegaConfig.enableExternalUrls) { + // Override URL loader and sanitizer to disable all URL-based requests + const errorFunc = () => { + throw new Error('External URLs are not enabled. Add "vega": {"enableExternalUrls": true} to kibana.yml'); + }; + const loader = vega.loader(); + loader.load = errorFunc; + loader.sanitize = errorFunc; + this._vegaViewConfig.loader = loader; + } + + // The derived class should create this method + await this._initViewCustomizations(); + } catch (err) { + this.onError(err); + } + } + + onError() { + this._addMessage('err', Utils.formatErrorToStr(...arguments)); + } + + onWarn() { + if (!this._parser || !this._parser.hideWarnings) { + this._addMessage('warn', Utils.formatWarningToStr(...arguments)); + } + } + + _addMessage(type, text) { + if (!this._$messages) { + this._$messages = $(`
    `).appendTo(this._$parentEl); + } + this._$messages.append( + $(`
  • `).append( + $(`
    `).text(text)
+      )
+    );
+  }
+
+  resize() {
+    if (this._parser.useResize && this._view && this.updateVegaSize(this._view)) {
+      return this._view.runAsync();
+    }
+  }
+
+  updateVegaSize(view) {
+    // For some reason the object is slightly scrollable without the extra padding.
+    // This might be due to https://github.com/jquery/jquery/issues/3808
+    // Which is being fixed as part of jQuery 3.3.0
+    const heightExtraPadding = 6;
+    const width = Math.max(0, this._$container.width() - this._parser.paddingWidth);
+    const height = Math.max(0, this._$container.height() - this._parser.paddingHeight) - heightExtraPadding;
+    if (view.width() !== width || view.height() !== height) {
+      view.width(width).height(height);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Set global debug variable to simplify vega debugging in console. Show info message first time
+   */
+  setDebugValues(view, spec, vlspec) {
+    if (!this._editorMode) {
+      // VEGA_DEBUG should only be enabled in the editor mode
+      return;
+    }
+
+    if (window) {
+      if (!view) {
+        // disposing, get rid of the stale debug info
+        delete window.VEGA_DEBUG;
+      } else {
+        if (window.VEGA_DEBUG === undefined && console) {
+          console.log('%cWelcome to Kibana Vega Plugin!', 'font-size: 16px; font-weight: bold;');
+          console.log('You can access the Vega view with VEGA_DEBUG. ' +
+            'Learn more at https://vega.github.io/vega/docs/api/debugging/.');
+        }
+
+        window.VEGA_DEBUG = window.VEGA_DEBUG || {};
+        window.VEGA_DEBUG.VEGA_VERSION = vega.version;
+        window.VEGA_DEBUG.VEGA_LITE_VERSION = vegaLite.version;
+        window.VEGA_DEBUG.view = view;
+        window.VEGA_DEBUG.vega_spec = spec;
+        window.VEGA_DEBUG.vegalite_spec = vlspec;
+      }
+    }
+  }
+
+  destroy() {
+    // properly handle multiple destroy() calls by converting this._destroyHandlers
+    // into the _ongoingDestroy promise, while handlers are being disposed
+    if (this._destroyHandlers) {
+      // If no destroy is yet running, execute all handlers and wait for all of them to resolve.
+      // Once done, keep the resolved promise, and get rid of any values returned from handlers.
+      this._ongoingDestroy = Promise.all(this._destroyHandlers.map(v => v())).then(() => 0);
+      this._destroyHandlers = null;
+    }
+    return this._ongoingDestroy;
+  }
+
+  _addDestroyHandler(handler) {
+    // If disposing hasn't started yet, enqueue it, otherwise dispose right away
+    // This creates a minor issue - if disposing has started but not yet finished,
+    // and we dispose the new handler right away, the destroy() does not wait for it.
+    // This behavior is no different from the case when disposing has already completed,
+    // so it shouldn't create any issues.
+    if (this._destroyHandlers) {
+      this._destroyHandlers.push(handler);
+    } else {
+      handler();
+    }
+  }
+}
diff --git a/src/core_plugins/vega/public/vega_view/vega_view.js b/src/core_plugins/vega/public/vega_view/vega_view.js
new file mode 100644
index 0000000000000..b90287c690f02
--- /dev/null
+++ b/src/core_plugins/vega/public/vega_view/vega_view.js
@@ -0,0 +1,27 @@
+import * as vega from 'vega';
+import { VegaBaseView } from './vega_base_view';
+
+export class VegaView extends VegaBaseView {
+  async _initViewCustomizations() {
+    // In some cases, Vega may be initialized twice... TBD
+    if (!this._$container) return;
+
+    const view = new vega.View(vega.parse(this._parser.spec), this._vegaViewConfig);
+    this.setDebugValues(view, this._parser.spec, this._parser.vlspec);
+
+    view.warn = this.onWarn.bind(this);
+    view.error = this.onError.bind(this);
+    if (this._parser.useResize) this.updateVegaSize(view);
+    view.initialize(this._$container.get(0), this._$controls.get(0));
+
+    if (this._parser.useHover) view.hover();
+
+    this._addDestroyHandler(() => {
+      this._view = null;
+      view.finalize();
+    });
+
+    await view.runAsync();
+    this._view = view;
+  }
+}
diff --git a/src/core_plugins/vega/public/vega_visualization.js b/src/core_plugins/vega/public/vega_visualization.js
new file mode 100644
index 0000000000000..a7af21a8a41ce
--- /dev/null
+++ b/src/core_plugins/vega/public/vega_visualization.js
@@ -0,0 +1,67 @@
+import { Notifier } from 'ui/notify';
+import { VegaView } from './vega_view/vega_view';
+
+export function VegaVisualizationProvider(vegaConfig, serviceSettings) {
+
+  const notify = new Notifier({ location: 'Vega' });
+
+  return class VegaVisualization {
+    constructor(el, vis) {
+      this._el = el;
+      this._vis = vis;
+    }
+
+    /**
+     *
+     * @param {VegaParser} visData
+     * @param {*} status
+     * @returns {Promise}
+     */
+    async render(visData, status) {
+      if (!visData && !this._vegaView) {
+        notify.warning('Unable to render without data');
+        return;
+      }
+
+      try {
+
+        await this._render(visData, status);
+
+      } catch (error) {
+        if (this._vegaView) {
+          this._vegaView.onError(error);
+        } else {
+          notify.error(error);
+        }
+      }
+    }
+
+    async _render(vegaParser, status) {
+      if (vegaParser && (status.data || !this._vegaView)) {
+
+        // New data received, rebuild the graph
+        if (this._vegaView) {
+          await this._vegaView.destroy();
+          this._vegaView = null;
+        }
+
+        if (vegaParser.useMap) {
+          throw new Error('Map mode is not yet supported in Kibana Core. You must use Kibana Vega plugin');
+        } else {
+          this._vegaView = new VegaView(vegaConfig, this._vis.editorMode, this._el, vegaParser, serviceSettings);
+        }
+        await this._vegaView.init();
+
+      } else if (status.resize) {
+
+        // the graph has been resized
+        await this._vegaView.resize();
+
+      }
+    }
+
+    destroy() {
+      return this._vegaView && this._vegaView.destroy();
+    }
+  };
+}
diff --git a/test/functional/apps/visualize/_chart_types.js b/test/functional/apps/visualize/_chart_types.js
index 1379ebdd7768f..556f2ea32e918 100644
--- a/test/functional/apps/visualize/_chart_types.js
+++ b/test/functional/apps/visualize/_chart_types.js
@@ -32,6 +32,7 @@ export default function ({ getService, getPageObjects }) {
           'Controls',
           'Markdown',
           'Tag Cloud',
+          'Vega',
         ];
 
         // find all the chart types and make sure there all there
diff --git a/test/functional/apps/visualize/_vega_chart.js b/test/functional/apps/visualize/_vega_chart.js
new file mode 100644
index 0000000000000..a9bf6c3f4e63c
--- /dev/null
+++ b/test/functional/apps/visualize/_vega_chart.js
@@ -0,0 +1,41 @@
+import expect from 'expect.js';
+
+export default function ({ getService, getPageObjects }) {
+  const log = getService('log');
+  const PageObjects = getPageObjects(['common', 'visualize', 'header']);
+
+  describe('visualize app', () => {
+    before(async () => {
+      log.debug('navigateToApp visualize');
+      await PageObjects.common.navigateToUrl('visualize', 'new');
+      log.debug('clickVega');
+      await PageObjects.visualize.clickVega();
+    });
+
+    describe('vega chart', () => {
+      it('should not display spy panel toggle button', async function () {
+        const spyToggleExists = await PageObjects.visualize.getSpyToggleExists();
+        expect(spyToggleExists).to.be(false);
+      });
+
+      it('should have some initial vega spec text', async function () {
+        const vegaSpec = await PageObjects.visualize.getVegaSpec();
+        expect(vegaSpec).to.contain('{').and.to.contain('data');
+        expect(vegaSpec.length).to.be.above(500);
+      });
+
+      it('should have view and control containers', async function () {
+        const view = await PageObjects.visualize.getVegaViewContainer();
+        expect(view).to.be.ok();
+        const size = await view.getSize();
+        expect(size).to.have.property('width').and.to.have.property('height');
+        expect(size.width).to.be.above(0);
+        expect(size.height).to.be.above(0);
+
+        const controls = await PageObjects.visualize.getVegaControlContainer();
+        expect(controls).to.be.ok();
+      });
+
+    });
+  });
+}
diff --git a/test/functional/apps/visualize/index.js b/test/functional/apps/visualize/index.js
index 60c96e6ae017a..d5d3f29f18d7b 100644
--- a/test/functional/apps/visualize/index.js
+++ b/test/functional/apps/visualize/index.js
@@ -32,5 +32,6 @@ export default function ({ getService, loadTestFile }) {
     loadTestFile(require.resolve('./_shared_item'));
     loadTestFile(require.resolve('./_input_control_vis'));
     loadTestFile(require.resolve('./_histogram_request_start'));
+    loadTestFile(require.resolve('./_vega_chart'));
   });
 }
diff --git a/test/functional/page_objects/visualize_page.js b/test/functional/page_objects/visualize_page.js
index e516f53b1e4d3..0beb78a55e70e 100644
--- a/test/functional/page_objects/visualize_page.js
+++ b/test/functional/page_objects/visualize_page.js
@@ -1,5 +1,6 @@
 import { VisualizeConstants } from '../../../src/core_plugins/kibana/public/visualize/visualize_constants';
 import Keys from 'leadfoot/keys';
+import Bluebird from 'bluebird';
 
 export function VisualizePageProvider({ getService, getPageObjects }) {
   const remote = getService('remote');
@@ -61,6 +62,10 @@ export function VisualizePageProvider({ getService, getPageObjects }) {
       await find.clickByPartialLinkText('Tag Cloud');
     }
 
+    async clickVega() {
+      await find.clickByPartialLinkText('Vega');
+    }
+
     async clickVisualBuilder() {
       await find.clickByPartialLinkText('Visual Builder');
     }
@@ -161,6 +166,22 @@ export function VisualizePageProvider({ getService, getPageObjects }) {
       return element.getVisibleText();
     }
 
+    async getVegaSpec() {
+      // Adapted from console_page.js:getVisibleTextFromAceEditor(). Is there a common utilities file?
+      const editor = await testSubjects.find('vega-editor');
+      const lines = await editor.findAllByClassName('ace_line_group');
+      const linesText = await Bluebird.map(lines, l => l.getVisibleText());
+      return linesText.join('\n');
+    }
+
+    async getVegaViewContainer() {
+      return await find.byCssSelector('div.vega-view-container');
+    }
+
+    async getVegaControlContainer() {
+      return await find.byCssSelector('div.vega-controls-container');
+    }
+
     async setFromTime(timeString) {
       const input = await find.byCssSelector('input[ng-model="absolute.from"]', defaultFindTimeout * 2);
       await input.clearValue();
diff --git a/yarn.lock b/yarn.lock
index a8ac64b1ca2c9..e5522cf0aeeea 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1797,6 +1797,18 @@ caniuse-db@^1.0.30000529, caniuse-db@^1.0.30000539, caniuse-db@^1.0.30000597, ca
   version "1.0.30000789"
   resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000789.tgz#5cf3fec75480041ab162ca06413153141e234325"
 
+canvas-prebuilt@^1.6:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/canvas-prebuilt/-/canvas-prebuilt-1.6.0.tgz#f8dd9abe81fdc2103a39d8362df3219d6d83f788"
+  dependencies:
+    node-pre-gyp "^0.6.29"
+
+canvas@^1.6:
+  version "1.6.9"
+  resolved "https://registry.yarnpkg.com/canvas/-/canvas-1.6.9.tgz#e3f95cec7b16bf2d6f3fc725c02d940d3258f69b"
+  dependencies:
+    nan "^2.4.0"
+
 capture-stack-trace@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/capture-stack-trace/-/capture-stack-trace-1.0.0.tgz#4a6fa07399c26bba47f0b2496b4d0fb408c5550d"
@@ -2247,14 +2259,14 @@ combined-stream@^1.0.5, combined-stream@~1.0.1, combined-stream@~1.0.5:
   dependencies:
     delayed-stream "~1.0.0"
 
+commander@2, commander@2.12.x, commander@^2.8.1, commander@~2.12.1:
+  version "2.12.2"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.12.2.tgz#0f5946c427ed9ec0d91a46bb9def53e54650e555"
+
 commander@2.11.0:
   version "2.11.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563"
 
-commander@2.12.x, commander@^2.8.1, commander@~2.12.1:
-  version "2.12.2"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-2.12.2.tgz#0f5946c427ed9ec0d91a46bb9def53e54650e555"
-
 commander@2.8.1, commander@2.8.x:
   version "2.8.1"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.8.1.tgz#06be367febfda0c330aa1e2a072d3dc9762425d4"
@@ -2275,6 +2287,10 @@ commondir@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
 
+compare-versions@3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-3.1.0.tgz#43310256a5c555aaed4193c04d8f154cf9c6efd5"
+
 component-bind@1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/component-bind/-/component-bind-1.0.0.tgz#00c608ab7dcd93897c0009651b1d3a8e1e73bbd1"
@@ -2719,16 +2735,134 @@ custom-event@~1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/custom-event/-/custom-event-1.0.1.tgz#5d02a46850adf1b4a317946a3928fccb5bfd0425"
 
+d3-array@1, d3-array@^1.1.1, d3-array@^1.2.0:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.1.tgz#d1ca33de2f6ac31efadb8e050a021d7e2396d5dc"
+
 d3-cloud@1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/d3-cloud/-/d3-cloud-1.2.1.tgz#a9cfdf3fb855804a9800866229bf016f71bd379a"
   dependencies:
     d3-dispatch "0.2.x"
 
+d3-collection@1:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/d3-collection/-/d3-collection-1.0.4.tgz#342dfd12837c90974f33f1cc0a785aea570dcdc2"
+
+d3-color@1:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-1.0.3.tgz#bc7643fca8e53a8347e2fbdaffa236796b58509b"
+
+d3-contour@1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/d3-contour/-/d3-contour-1.1.2.tgz#21f5456fcf57645922d69a27a58e782c91f842b3"
+  dependencies:
+    d3-array "^1.1.1"
+
 d3-dispatch@0.2.x:
   version "0.2.6"
   resolved "https://registry.yarnpkg.com/d3-dispatch/-/d3-dispatch-0.2.6.tgz#e57df25906cdce5badeae79809ec0f73bbb184ab"
 
+d3-dispatch@1:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/d3-dispatch/-/d3-dispatch-1.0.3.tgz#46e1491eaa9b58c358fce5be4e8bed626e7871f8"
+
+d3-dsv@1:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/d3-dsv/-/d3-dsv-1.0.8.tgz#907e240d57b386618dc56468bacfe76bf19764ae"
+  dependencies:
+    commander "2"
+    iconv-lite "0.4"
+    rw "1"
+
+d3-force@1:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/d3-force/-/d3-force-1.1.0.tgz#cebf3c694f1078fcc3d4daf8e567b2fbd70d4ea3"
+  dependencies:
+    d3-collection "1"
+    d3-dispatch "1"
+    d3-quadtree "1"
+    d3-timer "1"
+
+d3-format@1:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-1.2.2.tgz#1a39c479c8a57fe5051b2e67a3bee27061a74e7a"
+
+d3-geo@1:
+  version "1.9.1"
+  resolved "https://registry.yarnpkg.com/d3-geo/-/d3-geo-1.9.1.tgz#157e3b0f917379d0f73bebfff3be537f49fa7356"
+  dependencies:
+    d3-array "1"
+
+d3-hierarchy@1:
+  version "1.1.5"
+  resolved "https://registry.yarnpkg.com/d3-hierarchy/-/d3-hierarchy-1.1.5.tgz#a1c845c42f84a206bcf1c01c01098ea4ddaa7a26"
+
+d3-interpolate@1:
+  version "1.1.6"
+  resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-1.1.6.tgz#2cf395ae2381804df08aa1bf766b7f97b5f68fb6"
+  dependencies:
+    d3-color "1"
+
+d3-path@1:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/d3-path/-/d3-path-1.0.5.tgz#241eb1849bd9e9e8021c0d0a799f8a0e8e441764"
+
+d3-quadtree@1:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/d3-quadtree/-/d3-quadtree-1.0.3.tgz#ac7987e3e23fe805a990f28e1b50d38fcb822438"
+
+d3-request@1:
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/d3-request/-/d3-request-1.0.6.tgz#a1044a9ef4ec28c824171c9379fae6d79474b19f"
+  dependencies:
+    d3-collection "1"
+    d3-dispatch "1"
+    d3-dsv "1"
+    xmlhttprequest "1"
+
+d3-scale-chromatic@^1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/d3-scale-chromatic/-/d3-scale-chromatic-1.1.1.tgz#811406e8e09dab78a49dac4a32047d5d3edd0c44"
+  dependencies:
+    d3-interpolate "1"
+
+d3-scale@1:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-1.0.7.tgz#fa90324b3ea8a776422bd0472afab0b252a0945d"
+  dependencies:
+    d3-array "^1.2.0"
+    d3-collection "1"
+    d3-color "1"
+    d3-format "1"
+    d3-interpolate "1"
+    d3-time "1"
+    d3-time-format "2"
+
+d3-shape@1:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-1.2.0.tgz#45d01538f064bafd05ea3d6d2cb748fd8c41f777"
+  dependencies:
+    d3-path "1"
+
+d3-time-format@2:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-2.1.1.tgz#85b7cdfbc9ffca187f14d3c456ffda268081bb31"
+  dependencies:
+    d3-time "1"
+
+d3-time@1:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-1.0.8.tgz#dbd2d6007bf416fe67a76d17947b784bffea1e84"
+
+d3-timer@1:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-1.0.7.tgz#df9650ca587f6c96607ff4e60cc38229e8dd8531"
+
+d3-voronoi@1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/d3-voronoi/-/d3-voronoi-1.1.2.tgz#1687667e8f13a2d158c80c1480c5a29cb0d8973c"
+
 d3@3.5.6:
   version "3.5.6"
   resolved "https://registry.yarnpkg.com/d3/-/d3-3.5.6.tgz#9451c651ca733fb9672c81fb7f2655164a73a42d"
@@ -5103,6 +5237,10 @@ history@^4.7.2:
     value-equal "^0.4.0"
     warning "^3.0.0"
 
+hjson@3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/hjson/-/hjson-3.1.0.tgz#dd468d0a74fe227b79afd85b0df677433a633501"
+
 hmac-drbg@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz#d2745701025a6c775a6c545793ed502fc0c649a1"
@@ -5288,7 +5426,7 @@ husky@0.8.1:
   version "0.8.1"
   resolved "https://registry.yarnpkg.com/husky/-/husky-0.8.1.tgz#ecc797b8c4c6893a33f48703bc97a9a5e50d860f"
 
-iconv-lite@0.4.19, iconv-lite@^0.4.13, iconv-lite@^0.4.17, iconv-lite@~0.4.13:
+iconv-lite@0.4, iconv-lite@0.4.19, iconv-lite@^0.4.13, iconv-lite@^0.4.17, iconv-lite@~0.4.13:
   version "0.4.19"
   resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.19.tgz#f7468f60135f5e5dad3399c0a81be9a1603a082b"
 
@@ -6500,6 +6638,10 @@ json-stable-stringify@^1.0.0, json-stable-stringify@^1.0.1:
   dependencies:
     jsonify "~0.0.0"
 
+json-stringify-pretty-compact@1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/json-stringify-pretty-compact/-/json-stringify-pretty-compact-1.0.4.tgz#d5161131be27fd9748391360597fcca250c6c5ce"
+
 json-stringify-safe@5.0.1, json-stringify-safe@5.0.x, json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.0, json-stringify-safe@~5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
@@ -6959,7 +7101,7 @@ lodash._topath@^3.0.0:
   dependencies:
     lodash.isarray "^3.0.0"
 
-lodash.assign@^4.2.0:
+lodash.assign@^4.0.3, lodash.assign@^4.0.6, lodash.assign@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"
 
@@ -7701,7 +7843,7 @@ mv@~2:
     ncp "~2.0.0"
     rimraf "~2.4.0"
 
-nan@^2.0.8, nan@^2.3.0, nan@^2.3.2:
+nan@^2.0.8, nan@^2.3.0, nan@^2.3.2, nan@^2.4.0:
   version "2.8.0"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a"
 
@@ -7855,7 +7997,7 @@ node-notifier@^5.0.2:
     shellwords "^0.1.0"
     which "^1.2.12"
 
-node-pre-gyp@^0.6.39:
+node-pre-gyp@^0.6.29, node-pre-gyp@^0.6.39:
   version "0.6.39"
   resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.39.tgz#c00e96860b23c0e1420ac7befc5044e1d78d8649"
   dependencies:
@@ -9955,6 +10097,10 @@ run-async@^2.0.0, run-async@^2.2.0:
   dependencies:
     is-promise "^2.1.0"
 
+rw@1:
+  version "1.3.3"
+  resolved "https://registry.yarnpkg.com/rw/-/rw-1.3.3.tgz#3f862dfa91ab766b14885ef4d01124bfda074fb4"
+
 rx-lite-aggregates@^4.0.8:
   version "4.0.8"
   resolved "https://registry.yarnpkg.com/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz#753b87a89a11c95467c4ac1626c4efc4e05c67be"
@@ -11030,6 +11176,12 @@ topo@2.x.x:
   dependencies:
     hoek "4.x.x"
 
+topojson-client@3:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/topojson-client/-/topojson-client-3.0.0.tgz#1f99293a77ef42a448d032a81aa982b73f360d2f"
+  dependencies:
+    commander "2"
+
 tough-cookie@>=0.12.0, tough-cookie@^2.0.0, tough-cookie@^2.3.1, tough-cookie@^2.3.2, tough-cookie@~2.3.0, tough-cookie@~2.3.3:
   version "2.3.3"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561"
@@ -11102,6 +11254,10 @@ trunc-text@1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/trunc-text/-/trunc-text-1.0.2.tgz#b582bb3ddea9c9adc25017d737c48ebdd2157406"
 
+tslib@^1.8.0:
+  version "1.8.1"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.8.1.tgz#6946af2d1d651a7b1863b531d6e5afa41aa44eac"
+
 tty-browserify@0.0.0:
   version "0.0.0"
   resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6"
@@ -11448,6 +11604,230 @@ vary@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
 
+vega-crossfilter@2:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/vega-crossfilter/-/vega-crossfilter-2.0.0.tgz#29a8d789add5a2d0f25a4cdedb16713bf4f39061"
+  dependencies:
+    d3-array "1"
+    vega-dataflow "3"
+    vega-util "1"
+
+vega-dataflow@3:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/vega-dataflow/-/vega-dataflow-3.0.4.tgz#403a79d797a61016c66a90fb58c6a6530759384b"
+  dependencies:
+    vega-loader "2"
+    vega-util "1"
+
+vega-encode@2:
+  version "2.0.6"
+  resolved "https://registry.yarnpkg.com/vega-encode/-/vega-encode-2.0.6.tgz#ba8c81e9b0b4fe04b879b643855222c6dd8d4849"
+  dependencies:
+    d3-array "1"
+    d3-format "1"
+    d3-interpolate "1"
+    vega-dataflow "3"
+    vega-scale "^2.1"
+    vega-util "1"
+
+vega-event-selector@2, vega-event-selector@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/vega-event-selector/-/vega-event-selector-2.0.0.tgz#6af8dc7345217017ceed74e9155b8d33bad05d42"
+
+vega-expression@2:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/vega-expression/-/vega-expression-2.3.1.tgz#d802a329190bdeb999ce6d8083af56b51f686e83"
+  dependencies:
+    vega-util "1"
+
+vega-force@2:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/vega-force/-/vega-force-2.0.0.tgz#03084bfcb6f762d01162fb71dee165067fe0e7af"
+  dependencies:
+    d3-force "1"
+    vega-dataflow "3"
+    vega-util "1"
+
+vega-geo@^2.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/vega-geo/-/vega-geo-2.1.1.tgz#eaf128927cd146e1c0843d15f25a0a08d5dbf524"
+  dependencies:
+    d3-array "1"
+    d3-contour "1"
+    d3-geo "1"
+    vega-dataflow "3"
+    vega-projection "1"
+    vega-util "1"
+
+vega-hierarchy@^2.1:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/vega-hierarchy/-/vega-hierarchy-2.1.0.tgz#92119c43a9dc8f534c1836446661161dc9e42196"
+  dependencies:
+    d3-collection "1"
+    d3-hierarchy "1"
+    vega-dataflow "3"
+    vega-util "1"
+
+vega-lite@2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/vega-lite/-/vega-lite-2.0.3.tgz#3b42a3ee002cfd0d6c082a2b029d30f01c987ab4"
+  dependencies:
+    json-stable-stringify "^1.0.1"
+    tslib "^1.8.0"
+    vega-event-selector "^2.0.0"
+    vega-util "^1.6.2"
+    yargs "^10.0.3"
+
+vega-loader@2:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/vega-loader/-/vega-loader-2.0.3.tgz#1c1c221128c27a85f1fe5b4e9f8d709d541723e6"
+  dependencies:
+    d3-dsv "1"
+    d3-request "1"
+    d3-time-format "2"
+    topojson-client "3"
+    vega-util "1"
+
+vega-parser@2, vega-parser@^2.2:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/vega-parser/-/vega-parser-2.3.2.tgz#ba5e6cd789369604d066139a1d9b16934228191d"
+  dependencies:
+    d3-array "1"
+    d3-color "1"
+    d3-format "1"
+    d3-time-format "2"
+    vega-dataflow "3"
+    vega-event-selector "2"
+    vega-expression "2"
+    vega-scale "2"
+    vega-scenegraph "2"
+    vega-statistics "^1.2"
+    vega-util "^1.6"
+
+vega-projection@1:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/vega-projection/-/vega-projection-1.0.0.tgz#8875aefe9bc0b7b215ff2fa3358626efffa5c664"
+  dependencies:
+    d3-geo "1"
+
+vega-runtime@2:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/vega-runtime/-/vega-runtime-2.0.1.tgz#ef971ca3496df1cdbc0725699540952276c5f145"
+  dependencies:
+    vega-dataflow "3"
+    vega-util "1"
+
+vega-scale@2, vega-scale@^2.1:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/vega-scale/-/vega-scale-2.1.0.tgz#2b992cbb652a64d64b66015bf3a329ecaa7a3d32"
+  dependencies:
+    d3-array "1"
+    d3-interpolate "1"
+    d3-scale "1"
+    d3-scale-chromatic "^1.1"
+    d3-time "1"
+    vega-util "1"
+
+vega-scenegraph@2:
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/vega-scenegraph/-/vega-scenegraph-2.0.4.tgz#891afdfce9964a434e640712ee8a135ea528becc"
+  dependencies:
+    d3-path "1"
+    d3-shape "1"
+    vega-loader "2"
+    vega-util "^1.1"
+  optionalDependencies:
+    canvas "^1.6"
+    canvas-prebuilt "^1.6"
+
+vega-schema-url-parser@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/vega-schema-url-parser/-/vega-schema-url-parser-1.0.0.tgz#fc17631e354280d663ed39e3fa8eddb62145402e"
+
+vega-statistics@^1.2:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/vega-statistics/-/vega-statistics-1.2.1.tgz#a35b3fc3d0039f8bb0a8ba1381d42a1df79ecb34"
+  dependencies:
+    d3-array "1"
+
+vega-transforms@^1.1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/vega-transforms/-/vega-transforms-1.1.2.tgz#3881512569d4d1c1e62c3272ee06023f1f622468"
+  dependencies:
+    d3-array "1"
+    vega-dataflow "3"
+    vega-statistics "^1.2"
+    vega-util "1"
+
+vega-util@1, vega-util@^1.1, vega-util@^1.6, vega-util@^1.6.2:
+  version "1.6.2"
+  resolved "https://registry.yarnpkg.com/vega-util/-/vega-util-1.6.2.tgz#9d9d7cc65dfc9cd70eeb8dba8bb5c9924be5dacc"
+
+vega-view-transforms@^1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/vega-view-transforms/-/vega-view-transforms-1.1.1.tgz#191110f5f586fdf4fce5cee652afde193a4d28be"
+  dependencies:
+    vega-dataflow "3"
+    vega-scenegraph "2"
+    vega-util "1"
+
+vega-view@2:
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/vega-view/-/vega-view-2.0.4.tgz#a67895f0e45623a3a1181a0725afedfd560f4797"
+  dependencies:
+    d3-array "1"
+    vega-dataflow "3"
+    vega-parser "2"
+    vega-runtime "2"
+    vega-scenegraph "2"
+    vega-util "1"
+
+vega-voronoi@2:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/vega-voronoi/-/vega-voronoi-2.0.0.tgz#6df399181dc070a2ef52234ebfe5d7cebd0f3802"
+  dependencies:
+    d3-voronoi "1"
+    vega-dataflow "3"
+    vega-util "1"
+
+vega-wordcloud@2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/vega-wordcloud/-/vega-wordcloud-2.0.2.tgz#853be1b1492ba749001e5483be30380662ff2c59"
+  dependencies:
+    vega-dataflow "3"
+    vega-scale "2"
+    vega-statistics "^1.2"
+    vega-util "1"
+  optionalDependencies:
+    canvas "^1.6"
+    canvas-prebuilt "^1.6"
+
+vega@3.0.8:
+  version "3.0.8"
+  resolved "https://registry.yarnpkg.com/vega/-/vega-3.0.8.tgz#14c38908ab101048309866f1655f7d031dca7e3f"
+  dependencies:
+    vega-crossfilter "2"
+    vega-dataflow "3"
+    vega-encode "2"
+    vega-expression "2"
+    vega-force "2"
+    vega-geo "^2.1"
+    vega-hierarchy "^2.1"
+    vega-loader "2"
+    vega-parser "^2.2"
+    vega-projection "1"
+    vega-runtime "2"
+    vega-scale "^2.1"
+    vega-scenegraph "2"
+    vega-statistics "^1.2"
+    vega-transforms "^1.1"
+    vega-util "^1.6"
+    vega-view "2"
+    vega-view-transforms "^1.1"
+    vega-voronoi "2"
+    vega-wordcloud "2"
+    yargs "4"
+
 vendors@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/vendors/-/vendors-1.0.1.tgz#37ad73c8ee417fb3d580e785312307d274847f22"
@@ -11750,6 +12130,10 @@ window-size@0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.0.tgz#5438cd2ea93b202efa3a19fe8887aee7c94f9c9d"
 
+window-size@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
+
 with@~4.0.0:
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/with/-/with-4.0.3.tgz#eefd154e9e79d2c8d3417b647a8f14d9fecce14e"
@@ -11923,6 +12307,10 @@ xmlhttprequest-ssl@1.5.3:
   version "1.5.3"
   resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.3.tgz#185a888c04eca46c3e4070d99f7b49de3528992d"
 
+xmlhttprequest@1:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz#67fe075c5c24fef39f9d65f5f7b7fe75171968fc"
+
 "xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.0, xtend@~4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"
@@ -11935,6 +12323,13 @@ yallist@^2.0.0, yallist@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
 
+yargs-parser@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-2.4.1.tgz#85568de3cf150ff49fa51825f03a8c880ddcc5c4"
+  dependencies:
+    camelcase "^3.0.0"
+    lodash.assign "^4.0.6"
+
 yargs-parser@^4.2.0:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-4.2.1.tgz#29cceac0dc4f03c6c87b4a9f217dd18c9f74871c"
@@ -11959,6 +12354,25 @@ yargs-parser@^8.1.0:
   dependencies:
     camelcase "^4.1.0"
 
+yargs@4:
+  version "4.8.1"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-4.8.1.tgz#c0c42924ca4aaa6b0e6da1739dfb216439f9ddc0"
+  dependencies:
+    cliui "^3.2.0"
+    decamelize "^1.1.1"
+    get-caller-file "^1.0.1"
+    lodash.assign "^4.0.3"
+    os-locale "^1.4.0"
+    read-pkg-up "^1.0.1"
+    require-directory "^2.1.1"
+    require-main-filename "^1.0.1"
+    set-blocking "^2.0.0"
+    string-width "^1.0.1"
+    which-module "^1.0.0"
+    window-size "^0.2.0"
+    y18n "^3.2.1"
+    yargs-parser "^2.4.1"
+
 yargs@^10.0.3:
   version "10.1.1"
   resolved "https://registry.yarnpkg.com/yargs/-/yargs-10.1.1.tgz#5fe1ea306985a099b33492001fa19a1e61efe285"