diff --git a/.gitignore b/.gitignore index 1ce9a64..61e3ac2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ npm-debug.log* package-lock.json yarn.lock .project +.idea *.zip diff --git a/README.md b/README.md index 3db253c..aa8c5ec 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ This project is a Kibana plugin that provides two visualizations: - Ability to compute column total using formula - Support for numeric pretty format using [Numeral.js](http://numeraljs.com/#format) (ex: `0,0.00`) - Support for date pretty format using [Moment.js](http://momentjs.com/docs/#/displaying/format/) (ex: `YYYY-MM-DD`) + - Support for duration pretty format using Kibana duration format (with same options than Kibana Duration format) - Support for column alignment (ex: `left`, `right`) - Support for template rendering using [Handlebars](https://handlebarsjs.com/guide/expressions.html) (ex: `{{value}}`) - Template can reference other columns (ex: `{{value}}`) @@ -57,8 +58,8 @@ This project is a Kibana plugin that provides two visualizations: - Add a row number column - Ability to add the visualization to a Canvas workpad (Kibana 7.9+) - Ability to use dashboard drilldowns (Kibana 7.9+) -- Kibana supported versions: all versions from 5.5 to 8.4 -- OpenSearch Dashboards supported versions : all versions from 1.0 to 2.2 +- Kibana supported versions: all versions from 5.5 to 8.12 +- OpenSearch Dashboards supported versions : all versions from 1.x to 2.x ## Demo @@ -267,6 +268,32 @@ Helper | Description [encodeURIComponent(str)](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent) | Encodes the provided string as a Uniform Resource Identifier (URI) component. Example: `{{value}}` +## Row Functions + +Function | Description +:----------- | :---------- +rowValue(colName: string, actionName: string, fallback, qFilters: any) | filter all rows that match `qFilters`, group all values of `colName`, then calc the to single value based `actionName`. if there are no rows , return `fallback`

`colName` can be a string or colX where X is the index of the column.

`actionNmae` can be `first`, `last`, `max`, `min`, `sum`, `avg`

`qFilters` : a json object in { "key": value } format
--- `key` format is the same `colName`.
--- if the value is true, then the row `colName` must match `colName` of the current row
--- if value is `base.colX` or `base[name]` the row must match the column in the current row
--- if value is `row.colX` or `row[name]` the row must match the other column in the row
--- otherwise the row must match value +colShare( colName: string, fallback:unknown = 0, qFilters = {} ) | find the percent of the value of `colName` in current row from the sum of all values of `colName` in all the rows that match the `qFllter`. If no rows matched return `fallback` +colChange( colName: string, fallback:unknown = 0, qFilters = {} ) | find the percent of the value of `colName` in current row from the value of `colName` in first row that match the `qFllter`. If no rows matched return `fallback` + +## String Functions +Function | Description +:----------- | :---------- +findRe( ss: string , pattern : string, group: number or string, fallback: string ) | search the regular expression `pattern` in `ss`. If found return `group` in the match, otherwise return `fallback`. `group` can be number ( for unnamed groups) or string ( for named groups) +function findReGroups( ss: string , reStr : string ) : string[] | search the regular expression `pattern` in `ss`. return array of unnamed groups +function findReNamed( ss: string , reStr : string ) : { [key: string]: string; } | search the regular expression `pattern` in `ss`. return object of named groups +strJoin(s1:string, s2: string) | return joining of `s1` and `s2` +strColor(ss: string, hue? : number, saturation?: number, lightness?:number, hRange?: number, sRange?: number, lRange?: number) : string | return a css color based on a `ss` +strHash(ss: string, algorithm : string = 'sha1') : string | return an hash of `ss` using the `algorithm` + + +## Percent Functions +Function | Description +:----------- | :---------- +percentFrom (num: number, from: number) : number | return how much percent is `num` from `from` +percentOf(num: number, percent: number) : number | return `percent%` of `num` +percentChange(valNew: number, valOld: number) : number | return the percent change `valNew` from `valOld` + ## Change Log Versions and Release Notes are listed in [Releases](https://github.com/fbaligand/kibana-enhanced-table/releases) page @@ -280,36 +307,70 @@ Thanks for their great work ! ## Development -To run enhanced-table plugin in development mode (that enables hot code reload), follow these instructions: -- execute these commands : -``` bash -git clone https://github.com/opensearch-project/OpenSearch-Dashboards +You can run the enhanced-table plugin in development mode that supports hot code reload. + +Follow those steps: + +### Clone OpenSearch-Dashboards & kibina-enhanced-table +```bash +git clone https://github.com/opensearch-project/OpenSearch-Dashboards -b 1.0.0 cd OpenSearch-Dashboards -git checkout X.Y.Z # replace 'X.Y.Z' by desired OpenSearch-Dashboards version -cd plugins -git clone https://github.com/fbaligand/kibana-enhanced-table.git enhancedTable -git checkout osd +git clone https://github.com/fbaligand/kibana-enhanced-table.git -b osd plugins/enhancedTable ``` -- install the version of Node.js noted in `OpenSearch-Dashboards/.node-version` file -- ensure that node binary directory is in PATH environment variable -- install the latest version of yarn: `npm install -g yarn` -- execute these commands: -``` bash -cd OpenSearch-Dashboards -yarn osd bootstrap +### Update OpenSearch-Dashboards config + +Update `OpenSearch-Dashboards/config/opensearch_dashboards.yml` with opensearch hosts and user credentials. + +```yaml +opensearch.hosts: [""] +opensearch.username: "" +opensearch.password: "" +``` + +- - - + +If you do not have opensearch server running yet, +you can use the `opensearchproject/opensearch` docker image + +```bash +docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" opensearchproject/opensearch:latest +``` +and update config with the following settings: + +```yaml +opensearch.hosts: ["https://localhost:9200"] +opensearch.username: "admin" # Default username on the docker image +opensearch.password: "admin" # Default password on the docker image +opensearch.ssl.verificationMode: none +``` +### Install Node.js and yarn +install the version of Node.js listed in the .node-version file. +```bash +volta install node@$(<.node-version) +npm install -g yarn +``` + +### Build with Hot Reload +```bash cd plugins/enhancedTable +yarn osd bootstrap yarn install yarn start ``` -- in your browser, call `http://localhost:5601` and enjoy! - - -To build a distributable archive, execute this command: -``` bash -yarn compile-and-build --opensearch-dashboards-version X.Y.Z # replace 'X.Y.Z' by target OSD version +Now, you can open your browser, + + - In your browser, open the generated URL displayed in the console. It is something like: `http://localhost:5601/abc` + - Now, each time you change the code, the plugin will be reloaded with the changes. + - Happy coding :-) + +### Build a distributable archive +```bash +yarn build ``` -The zip archive is generated into `build` directory. +The result artifact located at `build/enhancedTable-X.Y.Z_osd-A.B.C.zip` where: +- X.Y.Z is the `${npm_package_version}` +- A.B.C is the OpenSearch-Dashboards version ## Donation diff --git a/package.json b/package.json index b993a2b..4e5b75c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "enhanced-table", - "version": "1.13.3", + "version": "1.14.0", "description": "This visualization plugin is like a Data Table, but with enhanced features like computed columns, filter bar and pivot table", "license": "Apache-2.0", "homepage": "https://github.com/fbaligand/kibana-enhanced-table", @@ -14,9 +14,10 @@ "lint": "../../node_modules/.bin/eslint .", "start": "cd ../.. && node scripts/opensearch_dashboards --dev", "debug": "node --nolazy --inspect ../../scripts/opensearch_dashboards --dev", - "build": "../../node_modules/.bin/plugin-helpers build", - "compile-and-build": "node ../../scripts/plugin_helpers.js build", - "compile": "rm -rf ./target && node ../../scripts/build_opensearch_dashboards_platform_plugins.js --scan-dir ." + "build": "yarn plugin-helpers build", + "compile": "rm -rf ./target && node ../../scripts/build_opensearch_dashboards_platform_plugins.js --scan-dir .", + "plugin-helpers": "node ../../scripts/plugin_helpers", + "postbuild": "old=(build/*.zip) && new=\"$(echo $old | cut -f 1 -d '-')-${npm_package_version}_osd-$(echo $old | cut -f 2 -d '-')\" && echo renaming build artifact to $new && mv $old $new" }, "dependencies": { "angular": "^1.8.0", @@ -26,4 +27,4 @@ }, "devDependencies": { } -} \ No newline at end of file +} diff --git a/public/enhanced-table-vis-controller.js b/public/enhanced-table-vis-controller.js index a6fa9ca..1120565 100644 --- a/public/enhanced-table-vis-controller.js +++ b/public/enhanced-table-vis-controller.js @@ -10,6 +10,10 @@ import { formulaFunctions } from './utils/formula_functions'; import { Parser } from 'expr-eval'; import handlebars from 'handlebars/dist/handlebars'; +import * as parser_rows from "./parser/rows"; +import * as parser_percents from "./parser/percents"; +import * as parser_strings from "./parser/strings"; + // EnhancedTableVis AngularJS controller function EnhancedTableVisController ($scope, config) { @@ -174,8 +178,10 @@ function EnhancedTableVisController ($scope, config) { realFormula = realFormula.replace(/(col|formattedCol)\s*\(/g, '$1(row, '); realFormula = realFormula.replace(/(sumSplitCols)\s*\(/g, '$1(row'); - // add 'table' & 'row' param for functions that require whole table - realFormula = realFormula.replace(/(total|cell|formattedCell)\s*\(/g, '$1(table, row, '); + const functionsWithTableRow = ['total', 'cell', 'formattedCell', 'rowValue', 'colShare', 'colChange' ]; + for ( const functionName of functionsWithTableRow ) { + realFormula = realFormula.replace( new RegExp(`(${functionName})\\s*\\(`, 'g'), '$1(table, row, '); + } // replace 'total' variable by 'totalHits' realFormula = realFormula.replace(/([^\w]|^)total([^(\w]|$)/g, '$1totalHits$2'); @@ -326,6 +332,13 @@ function EnhancedTableVisController ($scope, config) { return count; }; + // add from parser directory + (parser_rows ).addParser(parser,EnhancedTableError); + (parser_percents).addParser(parser); + (parser_strings ).addParser(parser); + + + // parse formula and return final formula object try { return { @@ -1230,4 +1243,4 @@ function EnhancedTableVisController ($scope, config) { } -export { EnhancedTableVisController }; \ No newline at end of file +export { EnhancedTableVisController }; diff --git a/public/parser/percents.ts b/public/parser/percents.ts new file mode 100644 index 0000000..8b1437c --- /dev/null +++ b/public/parser/percents.ts @@ -0,0 +1,17 @@ +export let parser: any; + +function percentFrom (num: number,from: number) : number{ + return from > 0 ? num * 100 / from : 0; +} +function percentOf(num: number,percent: number) : number { + return num*percent / 100; +} +function percentChange(valNew: number,valOld: number) : number{ + return percentFrom(valNew - valOld, valOld) +} + +export function addParser(parser) { + parser.functions.percentFrom = percentFrom; + parser.functions.percentOf = percentOf; + parser.functions.percentChange = percentChange; +} diff --git a/public/parser/rows.ts b/public/parser/rows.ts new file mode 100644 index 0000000..9db07ab --- /dev/null +++ b/public/parser/rows.ts @@ -0,0 +1,409 @@ +export let EnhancedTableError : any; +export let parser: any; + + +enum ValueSource { + None = 0, + Base = 1, + Row = 2 +} +class Val { + constructor(val: any) { + this.val = val; + + let matches; + const colType = typeof val; + this.isString = colType === 'string'; + this.isNumber = colType === 'number'; + this.isBoolean = colType === 'boolean'; + + this.isVal = true; + this.isVar = false; + this.id = 0; + this.src = ValueSource.None; + + if (this.isString) { + matches = val.match(/^(base|row).col(\d+)$/) + if (matches) { + this.id = parseInt(matches[2]); + this.src = matches[1] === 'base' ? ValueSource.Base : ValueSource.Row; + this.isVal = false; + this.isVar = true; + } + + matches = val.match(/^(base|row)\[(.*?)]$/) + if (matches) { + this.id = matches[2] + this.src = matches[1] === 'base' ? ValueSource.Base : ValueSource.Row; + this.isVal = false; + this.isVar = true; + } + } + } + + val: any; + public readonly isString : boolean; + public readonly isBoolean : boolean; + public readonly isNumber : boolean; + public readonly isVal : boolean; + public readonly isVar : boolean; + public readonly id : number | string; + public readonly src : ValueSource = ValueSource.None; + +} +class Row { + constructor(row: any) { + this.row = row; + } + + colId(name: string|number) : string | number { + let id : string | number = name; + if (typeof name === 'string') { + const matches = name.match(/col(\d+)/); + if (matches) { + id = parseInt(matches[1]) + } + } + return id; + } + + colVal(name: string|number) : any { + const colId = this.colId(name); + const value = parser.functions.col(this.row, colId, null) + if (value !== undefined) { + return value; + } + } + public readonly row : any; +} + + +class Actions { + constructor() { + this.byName = {} + } + add(action) { + this.byName[action.name] = action; + } + find(name: string) : ActionBase | null { + if ( name in this.byName ) { + return this.byName[name]; + } + return null + } + byName : Record +} + +class ActionBase { + constructor(name) { + this.name = name; + } + calc(rows,base,colName,fallback) { + throw new EnhancedTableError("Unknown Calc") + } + public readonly name : string; +} +class ActionFirst extends ActionBase { + constructor() { + super('first') + } + calc(rows, base, colName, fallback) { + if ( rows.length > 0 ) { + const row = new Row(rows[0]) + const val = row.colVal(colName) + if (val !== undefined) { + return val; + } + } + return fallback; + } +} +class ActionLast extends ActionBase { + constructor() { + super('last') + } + calc(rows, base, colName, fallback) { + if ( rows.length > 0 ) { + const row = new Row(rows[rows.length-1]) + const val = row.colVal(colName) + if (val !== undefined) { + return val; + } + } + } +} +class ActionSum extends ActionBase { + constructor() { + super('sum') + } + calc(rows, base, colName, fallback) { + let sum = 0; + let cnt = 0; + rows.forEach(function (row1) { + const row = new Row(row1) + const val = row.colVal(colName) + if (val !== undefined) { + sum += val; + cnt++; + } + }); + return cnt > 0 ? sum : fallback; + } +} +class ActionAvg extends ActionBase { + constructor() { + super('avg') + } + calc(rows, base, colName, fallback) { + let sum = 0; + let cnt = 0; + rows.forEach(function (row1) { + const row = new Row(row1) + const val = row.colVal(colName) + if (val !== undefined) { + sum += val; + cnt++; + } + }); + return cnt > 0 ? sum/cnt : fallback; + } +} +class ActionMax extends ActionBase { + constructor() { + super('max') + } + calc(rows, base, colName, fallback) { + let max = null; + rows.forEach(function (row1) { + const row = new Row(row1) + const val = row.colVal(colName) + if ( max === null || val > max ) { + max = val; + } + }); + return max !== null ? max : fallback; + } +} +class ActionMin extends ActionBase { + constructor() { + super('min') + } + calc(rows, base, colName, fallback) { + let min = null; + rows.forEach(function (row1) { + const row = new Row(row1) + const val = row.colVal(colName) + if ( min === null || val < min ) { + min = val; + } + }); + return min !== null ? min : fallback; + } +} + +class Filter { + constructor(qFilters: string) { + this.filter = this.parse(qFilters) + } + + parse(qFilters) : any { + try { + return JSON.parse(qFilters); + } catch (ee) { + throw new EnhancedTableError(`Invalid Filters format, Expected valid JSON. ${ee}`); + } + } + + valid(row, base) { + return this.valid1(row, base, this.filter) + } + + valid1(row, base , filter) { + if (Array.isArray(filter)) { + for (const _filter of filter) { + if (!this.valid1(row, base, _filter)) { + return false; + } + } + return true; + } + + if (typeof filter === 'object') { + for (let [filterName, filterVal] of Object.entries(filter)) { + const colIdx1 = row.colId(filterName) + const colVal1 = row.colVal(filterName) + + const val2 = new Val(filterVal) + + if (val2.isVal) { + if (val2.isBoolean) { + if (val2.val === true) { + const colVal2 = base.colVal(colIdx1) + if (colVal1 !== colVal2) { + return false; + } + } + continue; + } else { + if (colVal1 !== val2.val) { + return false; + } + continue; + } + } + if (val2.isVar) { + let srcRow; + + if (val2.src === ValueSource.Row) { + srcRow = row; + } + if (val2.src === ValueSource.Base) { + srcRow = base; + } + if (srcRow) { + const colVal2 = srcRow.colVal(val2.id) + if (colVal1 !== colVal2) { + return false; + } + } + } + } + return true; + } + } + + cache(base) { + const cacheInfo = { + filter: new Filter(JSON.stringify(this.filter)), + cacheable: true + } + return this.cache1(base, cacheInfo, cacheInfo.filter.filter) + } + + cache1(base, cacheInfo, filter) { + if ( !cacheInfo.cacheable) { + return cacheInfo; + } + if (Array.isArray(filter)) { + for (const _filter of filter) { + this.cache1(base, cacheInfo, _filter) + } + return cacheInfo; + } + + if (typeof filter === 'object') { + for (let [filterName, filterVal] of Object.entries(filter)) { + const val2 = new Val(filterVal) + if (val2.isVal) { + if (val2.isBoolean) { + if (val2.val === true) { + const colIdx1 = base.colId(filterName) + filter[filterName] = base.colVal(colIdx1) + } + } + } + if (val2.isVar) { + cacheInfo.cacheable = false; + } + } + return cacheInfo; + } + } + + private filter: any; +} + +class RowValueKey { + constructor(filter, base, colName, actionName, fallback) { + let cacheInfo = filter.cache(base) + this.filter = cacheInfo.filter + this.colName = colName + this.actionName = actionName + this.fallback = fallback + this.cacheable = cacheInfo.cacheable; + this.key = this.findKey() + } + findKey() : string { + return JSON.stringify({ + 'colName':this.colName, + 'actionName': this.actionName, + 'fallback': this.fallback, + 'filter' : this.filter + }) + } + public readonly filter; + public readonly colName; + public readonly actionName; + public readonly fallback; + public readonly key; + public readonly cacheable; +} + +const actions = new Actions(); + +actions.add( new ActionFirst() ); +actions.add( new ActionLast() ); +actions.add( new ActionSum() ); +actions.add( new ActionAvg() ); +actions.add( new ActionMax() ); +actions.add( new ActionMin() ); + + +function rowValue(table: any, base1: unknown, colName: unknown, actionName: string, fallback: unknown, qFilters: any) : unknown { + if ( table.rowValueCache === undefined ) { + table.rowValueCache = new Map() + } + const base = new Row(base1); + + let filter = null; + let rows = table.rows; + + + filter = new Filter(qFilters !== undefined ? qFilters : []); + const rowValueKey = (new RowValueKey(filter, base,colName,actionName, fallback)) + if ( rowValueKey.cacheable && table.rowValueCache.has(rowValueKey.key) ) { + return table.rowValueCache.get(rowValueKey.key) + } + + rows = rows.filter((row) => { + return filter.valid(new Row(row), base) + }); + + if (actionName === '' || actionName === undefined) { + actionName = 'first' + } + + const action = actions.find(actionName); + + let rv = action ? action.calc(rows, base, colName, fallback) : fallback; + + if ( rowValueKey.cacheable ) { + table.rowValueCache.set(rowValueKey.key, rv) + } + return rv +} + +function colShare( table: any, base1: unknown, colName: string, fallback:unknown = 0, qFilters = {} ) { + const sum = ( rowValue(table, base1, colName, "sum", 0, qFilters )) as number; + const val = (new Row(base1)).colVal(colName); + return sum !== 0 ? 100*val/sum : fallback; +} + +function colChange( table: any, base1: unknown, colName: string, fallback:unknown = 0, qFilters = {} ) { + const oldVal = ( rowValue(table, base1, colName, "first", 0, qFilters )) as number; + const newVal = (new Row(base1)).colVal(colName); + return oldVal !== 0 ? (100*(newVal-oldVal)) / oldVal : fallback; +} +export function addParser(_parser, _EnhancedTableError ) { + EnhancedTableError = _EnhancedTableError; + parser = _parser + + parser.functions.rowValue = rowValue; + parser.functions.colShare = colShare; + parser.functions.colChange= colChange; +} + + + +// sample : percentFrom( col2, rowsl( "col2" , "sum" , 0, '{ "col0" : true }' )) // col2 share from all rows group by col0 +// sample : colShare( "col2", 0, '{ "col0" : true }' ) // col2 share from all rows group by col0 diff --git a/public/parser/strings.ts b/public/parser/strings.ts new file mode 100644 index 0000000..d4d8f05 --- /dev/null +++ b/public/parser/strings.ts @@ -0,0 +1,107 @@ +import { createHash } from 'crypto' +import { Readable } from 'stream' +export let EnhancedTableError : any = Error; +export let parser: any; + +function findRe( ss: string , reStr : string, group: number|string, fallback: string ) : string { + let re: RegExp; + try { + re = new RegExp(reStr); + } catch (ee) { + throw EnhancedTableError(ee.toString()); + } + const matches = ss.match(re); + if ( matches ) { + if ( typeof group === "number" ) { + if (group < matches.length) { + return matches[group]; + } + } else { + if (group in matches.groups) { + return matches.groups[group]; + } + } + } + return fallback; +} + +function findReGroups( ss: string , reStr : string ) : string[] { + let re: RegExp; + try { + re = new RegExp(reStr); + } catch (ee) { + throw EnhancedTableError(ee.toString()); + } + const matches = ss.match(re); + if ( matches ) { + return matches; + } + return []; +} + +function findReNamed( ss: string , reStr : string ) : { [key: string]: string; } { + let re: RegExp; + try { + re = new RegExp(reStr); + } catch (ee) { + throw EnhancedTableError(ee.toString()); + } + const matches = ss.match(re); + if ( matches ) { + return matches.groups; + } + return {}; +} + +function strJoin(s1:string, s2: string) : string { + return s1 + s2; +} +function strColor(ss: string, hue? : number, saturation?: number, lightness?:number, hRange?: number, sRange?: number, lRange?: number) : string { + if ( hue === undefined ) { hue = 180; } + if ( hRange === undefined ) { hRange = 180; } + if ( saturation === undefined ) { saturation = 80; } + if ( sRange === undefined ) { sRange = 20; } + if ( lightness === undefined ) { lightness = 90; } + if ( lRange === undefined ) { lRange = 20; } + + function findHashes(ss) { + const hashes = []; + let hash = strHash(ss); + hashes[0] = parseInt(hash.substring(0,2),16); + hashes[1] = parseInt(hash.substring(2,4),16); + hashes[2] = parseInt(hash.substring(4,6),16); + return hashes; + } + + function findVal(val : number, min: number, max: number, rangeFull: number, rangeChange: number ) : number { + val = val - Math.round(rangeFull / 2) + rangeChange % rangeFull; + if ( val > max ) { + return max; + } + if ( val < min ) { + return min; + } + return val + + } + const hashes = findHashes(ss); + hue = findVal( hue , 0, 360, hRange, hashes[0]) + saturation = findVal( saturation , 0, 100, sRange, hashes[1]) + lightness = findVal( lightness , 0, 100, lRange, hashes[2]) + + return `hsl(${(hue)}, ${saturation}%, ${lightness}%)`; +} + +function strHash(ss: string, algorithm : string = 'sha1') : string { + const hash = createHash(algorithm); + return hash.update(ss).digest('hex'); +} + +export function addParser(parser) { + parser.functions.findRe = findRe; + parser.functions.findReGroups = findReGroups; + parser.functions.findReNamed = findReNamed; + parser.functions.strColor = strColor; + parser.functions.strJoin = strJoin; + parser.functions.strHash = strHash; +}