[Lens] Add specific IP and Range/Interval sorting to datatable

package.json

@@ -232,6 +232,7 @@
     "intl-messageformat": "^2.2.0",
     "intl-relativeformat": "^2.1.0",
     "io-ts": "^2.0.5",
+    "ipaddr.js": "2.0.0",
     "isbinaryfile": "4.0.2",
     "joi": "^13.5.2",
     "jquery": "^3.5.0",

x-pack/plugins/lens/public/datatable_visualization/expression.tsx

@@ -22,7 +22,7 @@ import {
   EuiBasicTableColumn,
   EuiTableActionsColumnType,
 } from '@elastic/eui';
-import { orderBy } from 'lodash';
+
 import { IAggType } from 'src/plugins/data/public';
 import { Datatable, DatatableColumnMeta, RenderMode } from 'src/plugins/expressions';
 import {
@@ -41,6 +41,7 @@ import { VisualizationContainer } from '../visualization_container';
 import { EmptyPlaceholder } from '../shared_components';
 import { desanitizeFilterContext } from '../utils';
 import { LensIconChartDatatable } from '../assets/chart_datatable';
+import { getSortingCriteria } from './sorting';
 
 export const LENS_EDIT_SORT_ACTION = 'sort';
 
@@ -92,6 +93,10 @@ export interface DatatableRender {
   value: DatatableProps;
 }
 
+function isRange(meta: { params?: { id?: string } } | undefined) {
+  return meta?.params?.id === 'range';
+}
+
 export const getDatatable = ({
   formatFactory,
 }: {
@@ -139,17 +144,18 @@ export const getDatatable = ({
 
     if (sortBy && sortDirection !== 'none') {
       // Sort on raw values for these types, while use the formatted value for the rest
-      const sortingCriteria = ['number', 'date'].includes(
-        columnsReverseLookup[sortBy]?.meta?.type || ''
-      )
-        ? sortBy
-        : (row: Record<string, unknown>) => formatters[sortBy]?.convert(row[sortBy]);
-      // replace the table here
-      context.inspectorAdapters.tables[layerId].rows = orderBy(
-        firstTable.rows || [],
-        [sortingCriteria],
-        sortDirection as Direction
+      const sortingCriteria = getSortingCriteria(
+        isRange(columnsReverseLookup[sortBy]?.meta)
+          ? 'range'
+          : columnsReverseLookup[sortBy]?.meta?.type,
+        sortBy,
+        formatters[sortBy],
+        sortDirection
       );
+      // replace the table here
+      context.inspectorAdapters.tables[layerId].rows = (firstTable.rows || [])
+        .slice()
+        .sort(sortingCriteria);
       // replace also the local copy
       firstTable.rows = context.inspectorAdapters.tables[layerId].rows;
     }

x-pack/plugins/lens/public/datatable_visualization/sorting.test.tsx

@@ -0,0 +1,187 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getSortingCriteria } from './sorting';
+import { FieldFormat } from 'src/plugins/data/public';
+import { DatatableColumnType } from 'src/plugins/expressions';
+
+function getMockFormatter() {
+  return { convert: (v: unknown) => `${v as string}` } as FieldFormat;
+}
+
+function testSorting({
+  input,
+  output,
+  direction,
+  type,
+  keepLast,
+}: {
+  input: unknown[];
+  output: unknown[];
+  direction: 'asc' | 'desc';
+  type: DatatableColumnType | 'range';
+  keepLast?: boolean; // special flag to handle values that should always be last no matter the direction
+}) {
+  const datatable = input.map((v) => ({
+    a: v,
+  }));
+  const sorted = output.map((v) => ({ a: v }));
+  if (direction === 'desc') {
+    sorted.reverse();
+    if (keepLast) {
+      // Cycle shift of the first element
+      const firstEl = sorted.shift()!;
+      sorted.push(firstEl);
+    }
+  }
+  const criteria = getSortingCriteria(type, 'a', getMockFormatter(), direction);
+  expect(datatable.sort(criteria)).toEqual(sorted);
+}
+
+describe('Data sorting criteria', () => {
+  describe('Numeric values', () => {
+    for (const direction of ['asc', 'desc'] as const) {
+      it(`should provide the number criteria of numeric values (${direction})`, () => {
+        testSorting({
+          input: [7, 6, 5, -Infinity, Infinity],
+          output: [-Infinity, 5, 6, 7, Infinity],
+          direction,
+          type: 'number',
+        });
+      });
+
+      it(`should provide the number criteria for date values (${direction})`, () => {
+        const now = Date.now();
+        testSorting({
+          input: [now, 0, now - 150000],
+          output: [0, now - 150000, now],
+          direction,
+          type: 'date',
+        });
+      });
+    }
+  });
+
+  describe('String or anything else as string', () => {
+    for (const direction of ['asc', 'desc'] as const) {
+      it(`should provide the string criteria for terms values (${direction})`, () => {
+        testSorting({
+          input: ['a', 'b', 'c', 'd', '12'],
+          output: ['12', 'a', 'b', 'c', 'd'],
+          direction,
+          type: 'string',
+        });
+      });
+
+      it(`should provide the string criteria for other types of values (${direction})`, () => {
+        testSorting({
+          input: [true, false, false],
+          output: [false, false, true],
+          direction,
+          type: 'boolean',
+        });
+      });
+    }
+  });
+
+  describe('IP sorting', () => {
+    for (const direction of ['asc', 'desc'] as const) {
+      it(`should provide the IP criteria for IP values (IPv4 only values) - ${direction}`, () => {
+        testSorting({
+          input: ['127.0.0.1', '192.168.1.50', '200.100.100.10', '10.0.1.76', '8.8.8.8'],
+          output: ['8.8.8.8', '10.0.1.76', '127.0.0.1', '192.168.1.50', '200.100.100.10'],
+          direction,
+          type: 'ip',
+        });
+      });
+
+      it(`should provide the IP criteria for IP values (IPv6 only values) - ${direction}`, () => {
+        testSorting({
+          input: [
+            'fc00::123',
+            '::1',
+            '2001:0db8:85a3:0000:0000:8a2e:0370:7334',
+            '2001:db8:1234:0000:0000:0000:0000:0000',
+            '2001:db8:1234::', // equivalent to the above
+          ],
+          output: [
+            '::1',
+            '2001:db8:1234::',
+            '2001:db8:1234:0000:0000:0000:0000:0000',
+            '2001:0db8:85a3:0000:0000:8a2e:0370:7334',
+            'fc00::123',
+          ],
+          direction,
+          type: 'ip',
+        });
+      });
+
+      it(`should provide the IP criteria for IP values (mixed values) - ${direction}`, () => {
+        // A mix of IPv4, IPv6, IPv4 mapped to IPv6
+        testSorting({
+          input: [
+            'fc00::123',
+            '192.168.1.50',
+            '::FFFF:192.168.1.50', // equivalent to the above with the IPv6 mapping
+            '10.0.1.76',
+            '8.8.8.8',
+            '::1',
+          ],
+          output: [
+            '::1',
+            '8.8.8.8',
+            '10.0.1.76',
+            '192.168.1.50',
+            '::FFFF:192.168.1.50',
+            'fc00::123',
+          ],
+          direction,
+          type: 'ip',
+        });
+      });
+
+      it(`should provide the IP criteria for IP values (mixed values with invalid "Other" field) - ${direction}`, () => {
+        testSorting({
+          input: ['fc00::123', '192.168.1.50', 'Other', '10.0.1.76', '8.8.8.8', '::1'],
+          output: ['::1', '8.8.8.8', '10.0.1.76', '192.168.1.50', 'fc00::123', 'Other'],
+          direction,
+          type: 'ip',
+          keepLast: true,
+        });
+      });
+    }
+  });
+
+  describe('Range sorting', () => {
+    for (const direction of ['asc', 'desc'] as const) {
+      it(`should sort closed ranges - ${direction}`, () => {
+        testSorting({
+          input: [
+            { gte: 1, lt: 5 },
+            { gte: 0, lt: 5 },
+            { gte: 0, lt: 1 },
+          ],
+          output: [
+            { gte: 0, lt: 1 },
+            { gte: 0, lt: 5 },
+            { gte: 1, lt: 5 },
+          ],
+          direction,
+          type: 'range',
+        });
+      });
+
+      it(`should sort open ranges - ${direction}`, () => {
+        testSorting({
+          input: [{ gte: 1, lt: 5 }, { gte: 0, lt: 5 }, { gte: 0 }],
+          output: [{ gte: 0, lt: 5 }, { gte: 0 }, { gte: 1, lt: 5 }],
+          direction,
+          type: 'range',
+        });
+      });
+    }
+  });
+});

x-pack/plugins/lens/public/datatable_visualization/sorting.tsx

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import ipaddr from 'ipaddr.js';
+import type { IPv4, IPv6 } from 'ipaddr.js';
+import { FieldFormat } from 'src/plugins/data/public';
+
+function isIPv6Address(ip: IPv4 | IPv6): ip is IPv6 {
+  return ip.kind() === 'ipv6';
+}
+
+function getSafeIpAddress(ip: string, directionFactor: number) {
+  if (!ipaddr.isValid(ip)) {
+    // for non valid IPs have the same behaviour as for now (we assume it's only the "Other" string)
+    // create a mock object which has all a special value to keep them always at the bottom of the list
+    return { parts: Array(8).fill(directionFactor * Infinity) };
+  }
+  const parsedIp = ipaddr.parse(ip);
+  return isIPv6Address(parsedIp) ? parsedIp : parsedIp.toIPv4MappedAddress();
+}
+
+function getIPCriteria(sortBy: string, directionFactor: number) {
+  // Create a set of 8 function to sort based on the 8 IPv6 slots of an address
+  // For IPv4 bring them to the IPv6 "mapped" format and then sort
+  return (rowA: Record<string, unknown>, rowB: Record<string, unknown>) => {
+    const ipAString = rowA[sortBy] as string;
+    const ipBString = rowB[sortBy] as string;
+    const ipA = getSafeIpAddress(ipAString, directionFactor);
+    const ipB = getSafeIpAddress(ipBString, directionFactor);
+
+    // Now compare each part of the IPv6 address and exit when a value != 0 is found
+    let i = 0;
+    let diff = ipA.parts[i] - ipB.parts[i];
+    while (!diff && i < 7) {
+      i++;
+      diff = ipA.parts[i] - ipB.parts[i];
+    }
+
+    // in case of same address but written in different styles, sort by string length
+    if (diff === 0) {
+      return directionFactor * (ipAString.length - ipBString.length);
+    }
+    return directionFactor * diff;
+  };
+}
+
+function getRangeCriteria(sortBy: string, directionFactor: number) {
+  // fill missing fields with these open bounds to perform number sorting
+  const openRange = { gte: -Infinity, lt: Infinity };
+  return (rowA: Record<string, unknown>, rowB: Record<string, unknown>) => {
+    const rangeA = { ...openRange, ...(rowA[sortBy] as Omit<Range, 'type'>) };
+    const rangeB = { ...openRange, ...(rowB[sortBy] as Omit<Range, 'type'>) };
+
+    const fromComparison = rangeA.gte - rangeB.gte;
+    const toComparison = rangeA.lt - rangeB.lt;
+
+    return directionFactor * (fromComparison || toComparison);
+  };
+}
+
+export function getSortingCriteria(
+  type: string | undefined,
+  sortBy: string,
+  formatter: FieldFormat,
+  direction: string
+) {
+  // handle the direction with a multiply factor.
+  const directionFactor = direction === 'asc' ? 1 : -1;
+
+  if (['number', 'date'].includes(type || '')) {
+    return (rowA: Record<string, unknown>, rowB: Record<string, unknown>) =>
+      directionFactor * ((rowA[sortBy] as number) - (rowB[sortBy] as number));
+  }
+  // this is a custom type, and can safely assume the gte and lt fields are all numbers or undefined
+  if (type === 'range') {
+    return getRangeCriteria(sortBy, directionFactor);
+  }
+  // IP have a special sorting
+  if (type === 'ip') {
+    return getIPCriteria(sortBy, directionFactor);
+  }
+  // use a string sorter for the rest
+  return (rowA: Record<string, unknown>, rowB: Record<string, unknown>) => {
+    const aString = formatter.convert(rowA[sortBy]);
+    const bString = formatter.convert(rowB[sortBy]);
+    return directionFactor * aString.localeCompare(bString);
+  };
+}

yarn.lock

@@ -16449,6 +16449,11 @@ [email protected], ipaddr.js@^1.9.0:
   resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.0.tgz#37df74e430a0e47550fe54a2defe30d8acd95f65"
   integrity sha512-M4Sjn6N/+O6/IXSJseKqHoFc+5FdGJ22sXqnjTpdZweHK64MzEPAyQZyEU3R/KRv2GLoa7nNtg/C2Ev6m7z+eA==
 
+[email protected]:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-2.0.0.tgz#77ccccc8063ae71ab65c55f21b090698e763fc6e"
+  integrity sha512-S54H9mIj0rbxRIyrDMEuuER86LdlgUg9FSeZ8duQb6CUG2iRrA36MYVQBSprTF/ZeAwvyQ5mDGuNvIPM0BIl3w==
+
 [email protected]:
   version "5.0.6"
   resolved "https://registry.yarnpkg.com/iron/-/iron-5.0.6.tgz#7121d4a6e3ac2f65e4d02971646fea1995434744"