Adds support for adhoc dataviews

x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx

@@ -252,21 +252,46 @@ export const LensTopNavMenu = ({
     [dispatch]
   );
   const dispatchChangeIndexPattern = React.useCallback(
-    async (indexPatternId) => {
-      const newIndexPatterns = await indexPatternService.ensureIndexPattern({
-        id: indexPatternId,
-        cache: dataViews.indexPatterns,
-      });
+    async (dataViewOrId, isAdHoc?) => {
+      const indexPatternId = typeof dataViewOrId === 'string' ? dataViewOrId : dataViewOrId.id;
+      const [newIndexPatternRefs, newIndexPatterns] = await Promise.all([
+        // Reload refs in case it's a new indexPattern created on the spot
+        dataViews.indexPatternRefs[indexPatternId]
+          ? dataViews.indexPatternRefs
+          : indexPatternService.loadIndexPatternRefs({
+              isFullEditor: true,
+            }),
+        indexPatternService.ensureIndexPattern({
+          id: indexPatternId,
+          cache: dataViews.indexPatterns,
+        }),
+      ]);
+      let indexPatternRefsEnhanced = newIndexPatternRefs;
+      if (isAdHoc) {
+        indexPatternRefsEnhanced = [
+          ...indexPatternRefsEnhanced,
+          {
+            title: dataViewOrId.title,
+            name: dataViewOrId.name,
+            id: indexPatternId,
+            adHoc: true,
+          },
+        ];
+      }
       dispatch(
         changeIndexPattern({
-          dataViews: { indexPatterns: newIndexPatterns },
+          dataViews: {
+            indexPatterns: newIndexPatterns,
+            indexPatternRefs: indexPatternRefsEnhanced,
+          },
           datasourceIds: Object.keys(datasourceStates),
           visualizationIds: visualization.activeId ? [visualization.activeId] : [],
           indexPatternId,
         })
       );
     },
     [
+      dataViews.indexPatternRefs,
       dataViews.indexPatterns,
       datasourceStates,
       dispatch,
@@ -694,14 +719,20 @@ export const LensTopNavMenu = ({
             closeDataViewEditor.current = dataViewEditor.openEditor({
               onSave: async (dataView) => {
                 if (dataView.id) {
-                  dispatchChangeIndexPattern(dataView.id);
+                  dispatchChangeIndexPattern(dataView, !dataView.isPersisted());
+                  setCurrentIndexPattern(dataView);
+                  if (!dataView.isPersisted()) {
+                    // add the ad-hoc dataview on the indexPatterns list
+                    setIndexPatterns([...indexPatterns, dataView]);
+                  }
                   refreshFieldList();
                 }
               },
+              allowAdHocDataView: true,
             });
           }
         : undefined,
-    [canEditDataView, dataViewEditor, dispatchChangeIndexPattern, refreshFieldList]
+    [canEditDataView, dataViewEditor, dispatchChangeIndexPattern, indexPatterns, refreshFieldList]
   );
 
   const dataViewPickerProps = {

x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts

@@ -42,7 +42,8 @@ import { loadIndexPatternRefs, loadIndexPatterns } from '../../indexpattern_serv
 function getIndexPatterns(
   references?: SavedObjectReference[],
   initialContext?: VisualizeFieldContext | VisualizeEditorContext,
-  initialId?: string
+  initialId?: string,
+  adHocDataviews?: string[]
 ) {
   const indexPatternIds = [];
   if (initialContext) {
@@ -66,6 +67,9 @@ function getIndexPatterns(
       }
     }
   }
+  if (adHocDataviews) {
+    indexPatternIds.push(...adHocDataviews);
+  }
   return [...new Set(indexPatternIds)];
 }
 
@@ -111,8 +115,26 @@ export async function initializeDataViews(
     Object.keys(datasourceMap).every((datasourceId) => !datasourceStates[datasourceId]?.state)
       ? fallbackId
       : undefined;
-
-  const usedIndexPatterns = getIndexPatterns(references, initialContext, initialId);
+  const adHocDataviewsIds: string[] = [];
+  let adHocDataviews;
+  Object.keys(datasourceMap).forEach((datasourceId) => {
+    const datasource = datasourceMap[datasourceId];
+    const datasourceState = datasourceStates[datasourceId]?.state;
+    const adHocSpecs = datasource?.getAdHocIndexSpecs?.(datasourceState);
+    if (adHocSpecs) {
+      const dataViewsIds: string[] = Object.keys(adHocSpecs);
+      if (dataViewsIds.length) {
+        adHocDataviewsIds.push(...dataViewsIds);
+        adHocDataviews = Object.values(adHocSpecs);
+      }
+    }
+  });
+  const usedIndexPatterns = getIndexPatterns(
+    references,
+    initialContext,
+    initialId,
+    adHocDataviewsIds
+  );
 
   // load them
   const availableIndexPatterns = new Set(indexPatternRefs.map(({ id }: IndexPatternRef) => id));
@@ -124,6 +146,7 @@ export async function initializeDataViews(
     patterns: usedIndexPatterns,
     notUsedPatterns,
     cache: {},
+    adHocDataviews,
   });
 
   return { indexPatternRefs, indexPatterns };

x-pack/plugins/lens/public/embeddable/embeddable.tsx

@@ -764,8 +764,23 @@ export class Embeddable
 
     this.activeDataInfo.activeDatasource = this.deps.datasourceMap[activeDatasourceId];
     const docDatasourceState = this.savedVis?.state.datasourceStates[activeDatasourceId];
+    const adHocIndexPatterns =
+      this.activeDataInfo.activeDatasource?.getAdHocIndexSpecs?.(docDatasourceState);
+
+    const adHocDataviews: DataView[] = [];
+
+    if (adHocIndexPatterns) {
+      const adHocSpecs = Object.values(adHocIndexPatterns);
+      if (adHocSpecs?.length) {
+        for (const addHocDataView of adHocSpecs) {
+          const d = await this.deps.dataViews.create(addHocDataView);
+          adHocDataviews.push(d);
+        }
+      }
+    }
+    const allIndexPatterns = [...this.indexPatterns, ...adHocDataviews];
 
-    const indexPatternsCache = this.indexPatterns.reduce(
+    const indexPatternsCache = allIndexPatterns.reduce(
       (acc, indexPattern) => ({
         [indexPattern.id!]: convertDataViewIntoLensIndexPattern(indexPattern),
         ...acc,

x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx

@@ -202,6 +202,10 @@ export function getIndexPatternDatasource({
       return Object.keys(state.layers);
     },
 
+    getAdHocIndexSpecs(state: IndexPatternPersistedState) {
+      return state?.adHocIndexPatterns;
+    },
+
     removeColumn({ prevState, layerId, columnId, indexPatterns }) {
       const indexPattern = indexPatterns[prevState.layers[layerId]?.indexPatternId];
       return mergeLayer({
@@ -540,15 +544,18 @@ export function getIndexPatternDatasource({
           }
           return null;
         },
-        getSourceId: () => layer.indexPatternId,
-        getFilters: (activeData: FramePublicAPI['activeData'], timeRange?: TimeRange) =>
-          getFiltersInLayer(
+        getSourceId: () => {
+          return layer.adHocSpec?.id || layer.indexPatternId;
+        },
+        getFilters: (activeData: FramePublicAPI['activeData'], timeRange?: TimeRange) => {
+          return getFiltersInLayer(
             layer,
             visibleColumnIds,
             activeData?.[layerId],
             indexPatterns[layer.indexPatternId],
             timeRange
-          ),
+          );
+        },
         getVisualDefaults: () => getVisualDefaultsForLayer(layer),
         getMaxPossibleNumValues: (columnId) => {
           if (layer && layer.columns[columnId]) {

x-pack/plugins/lens/public/indexpattern_datasource/loader.ts

@@ -49,16 +49,25 @@ function getLayerReferenceName(layerId: string) {
 
 export function extractReferences({ layers }: IndexPatternPrivateState) {
   const savedObjectReferences: SavedObjectReference[] = [];
-  const persistableLayers: Record<string, Omit<IndexPatternLayer, 'indexPatternId'>> = {};
+  const persistableState: IndexPatternPersistedState = {
+    layers: {},
+    adHocIndexPatterns: {},
+  };
   Object.entries(layers).forEach(([layerId, { indexPatternId, ...persistableLayer }]) => {
-    savedObjectReferences.push({
-      type: 'index-pattern',
-      id: indexPatternId,
-      name: getLayerReferenceName(layerId),
-    });
-    persistableLayers[layerId] = persistableLayer;
+    persistableState.layers[layerId] = persistableLayer;
+    if (persistableLayer.adHocSpec) {
+      if (!persistableState.adHocIndexPatterns![indexPatternId]) {
+        persistableState.adHocIndexPatterns![indexPatternId] = persistableLayer.adHocSpec!;
+      }
+    } else {
+      savedObjectReferences.push({
+        type: 'index-pattern',
+        id: indexPatternId,
+        name: getLayerReferenceName(layerId),
+      });
+    }
   });
-  return { savedObjectReferences, state: { layers: persistableLayers } };
+  return { savedObjectReferences, state: persistableState };
 }
 
 export function injectReferences(
@@ -69,11 +78,14 @@ export function injectReferences(
   Object.entries(state.layers).forEach(([layerId, persistedLayer]) => {
     layers[layerId] = {
       ...persistedLayer,
-      indexPatternId: references.find(({ name }) => name === getLayerReferenceName(layerId))!.id,
+      indexPatternId:
+        persistedLayer.adHocSpec?.id ||
+        references.find(({ name }) => name === getLayerReferenceName(layerId))!.id,
     };
   });
   return {
     layers,
+    adHocIndexPatterns: state.adHocIndexPatterns,
   };
 }
 
@@ -117,7 +129,11 @@ function getUsedIndexPatterns({
   const usedPatterns = (
     initialContext
       ? indexPatternIds
-      : uniq(state ? Object.values(state.layers).map((l) => l.indexPatternId) : [fallbackId])
+      : uniq(
+          state
+            ? Object.values(state.layers).map((l) => l.adHocSpec?.id ?? l.indexPatternId)
+            : [fallbackId]
+        )
   )
     // take out the undefined from the list
     .filter(Boolean);
@@ -154,6 +170,11 @@ export function loadInitialState({
     indexPatternRefs,
   });
 
+  if (persistedState?.adHocIndexPatterns) {
+    Object.entries(persistedState?.adHocIndexPatterns).forEach(([id, { name, title }]) => {
+      indexPatternRefs.push({ id, name, title: title || '', adHoc: true });
+    });
+  }
   const availableIndexPatterns = new Set(indexPatternRefs.map(({ id }: IndexPatternRef) => id));
 
   const notUsedPatterns: string[] = difference([...availableIndexPatterns], usedPatterns);

x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts

@@ -200,6 +200,12 @@ export function insertNewColumn({
   if (layer.columns[columnId]) {
     throw new Error(`Can't insert a column with an ID that is already in use`);
   }
+  if (indexPattern.spec) {
+    layer = {
+      ...layer,
+      adHocSpec: indexPattern.spec,
+    };
+  }
 
   const baseOptions = {
     indexPattern,
@@ -1356,6 +1362,7 @@ export function updateLayerIndexPattern(
   return {
     ...layer,
     indexPatternId: newIndexPattern.id,
+    adHocSpec: newIndexPattern?.spec,
     columns: newColumns,
     columnOrder: newColumnOrder,
   };

x-pack/plugins/lens/public/indexpattern_datasource/types.ts

@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import type { DataViewSpec } from '@kbn/data-views-plugin/common';
 import type { DragDropIdentifier } from '../drag_drop/providers';
 import type { IncompleteColumn, GenericIndexPatternColumn } from './operations';
 import type { DragDropOperation } from '../types';
@@ -55,10 +55,12 @@ export interface IndexPatternLayer {
   indexPatternId: string;
   // Partial columns represent the temporary invalid states
   incompleteColumns?: Record<string, IncompleteColumn>;
+  adHocSpec?: DataViewSpec;
 }
 
 export interface IndexPatternPersistedState {
   layers: Record<string, Omit<IndexPatternLayer, 'indexPatternId'>>;
+  adHocIndexPatterns?: Record<string, DataViewSpec>;
 }
 
 export type PersistedIndexPatternLayer = Omit<IndexPatternLayer, 'indexPatternId'>;

x-pack/plugins/lens/public/indexpattern_service/loader.ts

@@ -6,7 +6,7 @@
  */
 
 import { isNestedField } from '@kbn/data-views-plugin/common';
-import type { DataViewsContract, DataView } from '@kbn/data-views-plugin/public';
+import type { DataViewsContract, DataView, DataViewSpec } from '@kbn/data-views-plugin/public';
 import { keyBy } from 'lodash';
 import { HttpSetup } from '@kbn/core/public';
 import { IndexPattern, IndexPatternField, IndexPatternMap, IndexPatternRef } from '../types';
@@ -15,7 +15,7 @@ import { BASE_API_URL, DateRange, ExistingFields } from '../../common';
 import { DataViewsState } from '../state_management';
 
 type ErrorHandler = (err: Error) => void;
-type MinimalDataViewsContract = Pick<DataViewsContract, 'get' | 'getIdsWithTitle'>;
+type MinimalDataViewsContract = Pick<DataViewsContract, 'get' | 'getIdsWithTitle' | 'create'>;
 
 /**
  * All these functions will be used by the Embeddable instance too,
@@ -92,6 +92,7 @@ export function convertDataViewIntoLensIndexPattern(
     fields: newFields,
     getFieldByName: getFieldByNameFactory(newFields),
     hasRestrictions: !!typeMeta?.aggs,
+    spec: dataView.isPersisted() ? undefined : dataView.toSpec(false),
   };
 }
 
@@ -122,12 +123,14 @@ export async function loadIndexPatterns({
   patterns,
   notUsedPatterns,
   cache,
+  adHocDataviews,
   onIndexPatternRefresh,
 }: {
   dataViews: MinimalDataViewsContract;
   patterns: string[];
   notUsedPatterns?: string[];
   cache: Record<string, IndexPattern>;
+  adHocDataviews?: DataViewSpec[];
   onIndexPatternRefresh?: () => void;
 }) {
   const missingIds = patterns.filter((id) => !cache[id]);
@@ -157,6 +160,12 @@ export async function loadIndexPatterns({
       }
     }
   }
+  if (adHocDataviews?.length) {
+    for (const addHocDataView of adHocDataviews) {
+      const d = await dataViews.create(addHocDataView);
+      indexPatterns.push(d);
+    }
+  }
 
   const indexPatternsObject = indexPatterns.reduce(
     (acc, indexPattern) => ({

x-pack/plugins/lens/public/state_management/selectors.ts

@@ -113,8 +113,16 @@ export const selectSavedObjectFormat = createSelector(
       references.push(...savedObjectReferences);
     });
 
+    const adHocFilters = filters
+      .filter((f) => !references.some((r) => r.type === 'index-pattern' && r.id === f.meta.index))
+      .map((f) => ({ ...f, meta: { ...f.meta, value: undefined } }));
+
+    const referencedFilters = filters.filter((f) =>
+      references.some((r) => r.type === 'index-pattern' && r.id === f.meta.index)
+    );
+
     const { state: persistableFilters, references: filterReferences } =
-      extractFilterReferences(filters);
+      extractFilterReferences(referencedFilters);
 
     references.push(...filterReferences);
 
@@ -128,7 +136,7 @@ export const selectSavedObjectFormat = createSelector(
       state: {
         visualization: visualization.state,
         query,
-        filters: persistableFilters,
+        filters: [...persistableFilters, ...adHocFilters],
         datasourceStates: persistibleDatasourceStates,
       },
     };