diff --git a/ee/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr b/ee/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr index d8d0967eb9896..41094c453fec8 100644 --- a/ee/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr +++ b/ee/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr @@ -50,7 +50,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT %(limit)s OFFSET %(offset)s @@ -107,7 +107,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT %(limit)s OFFSET %(offset)s @@ -163,7 +163,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT %(limit)s OFFSET %(offset)s @@ -219,7 +219,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT %(limit)s OFFSET %(offset)s @@ -249,7 +249,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -292,7 +292,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -322,7 +322,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -365,7 +365,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -395,7 +395,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -438,7 +438,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -468,7 +468,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -511,7 +511,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -541,7 +541,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -592,7 +592,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -622,7 +622,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -673,7 +673,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -703,7 +703,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -754,7 +754,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -784,7 +784,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -835,7 +835,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -865,7 +865,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -916,7 +916,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -946,7 +946,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -997,7 +997,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1027,7 +1027,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1078,7 +1078,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1108,7 +1108,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1159,7 +1159,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1204,7 +1204,7 @@ HAVING argMax(is_deleted, version) = 0 AND current_person_id = '00000000-0000-0000-0000-000000000000') as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1249,7 +1249,7 @@ HAVING argMax(is_deleted, version) = 0 AND current_person_id = '00000000-0000-0000-0000-000000000000') as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1294,7 +1294,7 @@ HAVING argMax(is_deleted, version) = 0 AND current_person_id = '00000000-0000-0000-0000-000000000000') as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1339,7 +1339,7 @@ HAVING argMax(is_deleted, version) = 0 AND current_person_id = '00000000-0000-0000-0000-000000000000') as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1384,7 +1384,7 @@ HAVING argMax(is_deleted, version) = 0 AND current_person_id = '00000000-0000-0000-0000-000000000000') as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1429,7 +1429,7 @@ HAVING argMax(is_deleted, version) = 0 AND current_person_id = '00000000-0000-0000-0000-000000000000') as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1474,7 +1474,7 @@ HAVING argMax(is_deleted, version) = 0 AND current_person_id = '00000000-0000-0000-0000-000000000000') as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1519,7 +1519,7 @@ HAVING argMax(is_deleted, version) = 0 AND current_person_id = '00000000-0000-0000-0000-000000000000') as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1562,7 +1562,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1605,7 +1605,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1648,7 +1648,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1691,7 +1691,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark.png index 4d8660c88304e..fb1fbd220178d 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light.png index 643ed8a4088bc..4bc0d3a54c2ef 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown--light.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light.png index 928391c9da297..3ddc4b47993f4 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--light.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-line-edit--dark.png b/frontend/__snapshots__/scenes-app-insights--trends-line-edit--dark.png index b8f2ab3e5f571..4c93657d5e0b6 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-line-edit--dark.png and b/frontend/__snapshots__/scenes-app-insights--trends-line-edit--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--user-paths-edit--light.png b/frontend/__snapshots__/scenes-app-insights--user-paths-edit--light.png index 59476e592f344..dd4d471e99c53 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--user-paths-edit--light.png and b/frontend/__snapshots__/scenes-app-insights--user-paths-edit--light.png differ diff --git a/frontend/src/lib/components/ActivityLog/activityLogLogic.tsx b/frontend/src/lib/components/ActivityLog/activityLogLogic.tsx index 4ba47decd2902..250f42da557cd 100644 --- a/frontend/src/lib/components/ActivityLog/activityLogLogic.tsx +++ b/frontend/src/lib/components/ActivityLog/activityLogLogic.tsx @@ -11,6 +11,7 @@ import { } from 'lib/components/ActivityLog/humanizeActivity' import { ACTIVITY_PAGE_SIZE } from 'lib/constants' import { PaginationManual } from 'lib/lemon-ui/PaginationControl' +import { cohortActivityDescriber } from 'scenes/cohorts/activityDescriptions' import { dataManagementActivityDescriber } from 'scenes/data-management/dataManagementDescribers' import { flagActivityDescriber } from 'scenes/feature-flags/activityDescriptions' import { notebookActivityDescriber } from 'scenes/notebooks/Notebook/notebookActivityDescriber' @@ -36,6 +37,8 @@ export const describerFor = (logItem?: ActivityLogItem): Describer | undefined = case ActivityScope.PLUGIN: case ActivityScope.PLUGIN_CONFIG: return pluginActivityDescriber + case ActivityScope.COHORT: + return cohortActivityDescriber case ActivityScope.INSIGHT: return insightActivityDescriber case ActivityScope.PERSON: diff --git a/frontend/src/scenes/cohorts/activityDescriptions.tsx b/frontend/src/scenes/cohorts/activityDescriptions.tsx new file mode 100644 index 0000000000000..ceb4726ce7b77 --- /dev/null +++ b/frontend/src/scenes/cohorts/activityDescriptions.tsx @@ -0,0 +1,55 @@ +import '../../lib/components/Cards/InsightCard/InsightCard.scss' + +import { + ActivityLogItem, + defaultDescriber, + HumanizedChange, + userNameForLogItem, +} from 'lib/components/ActivityLog/humanizeActivity' +import { Link } from 'lib/lemon-ui/Link' +import { urls } from 'scenes/urls' + +const nameOrLinkToCohort = (id?: string | null, name?: string | null): string | JSX.Element => { + const displayName = name || '(empty string)' + return id ? {displayName} : displayName +} + +export function cohortActivityDescriber(logItem: ActivityLogItem, asNotification?: boolean): HumanizedChange { + if (logItem.scope != 'Cohort') { + console.error('cohort describer received a non-cohort activity') + return { description: null } + } + + if (logItem.activity == 'created') { + return { + description: ( + <> + {userNameForLogItem(logItem)} created the cohort:{' '} + {nameOrLinkToCohort(logItem?.item_id, logItem?.detail.name)} + + ), + } + } + + if (logItem.activity == 'deleted') { + return { + description: ( + <> + {userNameForLogItem(logItem)} deleted the cohort: {logItem.detail.name} + + ), + } + } + + if (logItem.activity == 'updated') { + return { + description: ( + <> + {userNameForLogItem(logItem)} updated the cohort:{' '} + {nameOrLinkToCohort(logItem?.item_id, logItem?.detail.name)} + + ), + } + } + return defaultDescriber(logItem, asNotification, nameOrLinkToCohort(logItem?.detail.short_id)) +} diff --git a/frontend/src/scenes/dashboard/dashboardLogic.tsx b/frontend/src/scenes/dashboard/dashboardLogic.tsx index 1d18ab9c31ac9..f3856bc674fb6 100644 --- a/frontend/src/scenes/dashboard/dashboardLogic.tsx +++ b/frontend/src/scenes/dashboard/dashboardLogic.tsx @@ -1022,13 +1022,7 @@ export const dashboardLogic = kea([ const queryId = `${dashboardQueryId}::${uuid()}` const queryStartTime = performance.now() const apiUrl = `api/projects/${values.currentTeamId}/insights/${insight.id}/?${toParams({ - refresh: values.featureFlags[FEATURE_FLAGS.HOGQL_DASHBOARD_ASYNC] - ? hardRefreshWithoutCache - ? 'force_async' - : 'async' - : hardRefreshWithoutCache - ? 'force_blocking' - : 'blocking', + refresh: hardRefreshWithoutCache ? 'force_async' : 'async', from_dashboard: dashboardId, // needed to load insight in correct context client_query_id: queryId, session_id: currentSessionId(), diff --git a/frontend/src/scenes/pipeline/destinationsLogic.tsx b/frontend/src/scenes/pipeline/destinationsLogic.tsx index b920265e87760..95a201c783a7a 100644 --- a/frontend/src/scenes/pipeline/destinationsLogic.tsx +++ b/frontend/src/scenes/pipeline/destinationsLogic.tsx @@ -19,7 +19,14 @@ import { import type { pipelineDestinationsLogicType } from './destinationsLogicType' import { pipelineAccessLogic } from './pipelineAccessLogic' -import { BatchExportDestination, convertToPipelineNode, Destination, PipelineBackend } from './types' +import { + BatchExportDestination, + convertToPipelineNode, + Destination, + FunctionDestination, + PipelineBackend, + WebhookDestination, +} from './types' import { captureBatchExportEvent, capturePluginEvent, loadPluginsFromUrl } from './utils' export const pipelineDestinationsLogic = kea([ @@ -38,10 +45,13 @@ export const pipelineDestinationsLogic = kea([ toggleNode: (destination: Destination, enabled: boolean) => ({ destination, enabled }), deleteNode: (destination: Destination) => ({ destination }), deleteNodeBatchExport: (destination: BatchExportDestination) => ({ destination }), + deleteNodeHogFunction: (destination: FunctionDestination) => ({ destination }), + deleteNodeWebhook: (destination: WebhookDestination) => ({ destination }), + updatePluginConfig: (pluginConfig: PluginConfigTypeNew) => ({ pluginConfig }), updateBatchExportConfig: (batchExportConfig: BatchExportConfiguration) => ({ batchExportConfig }), }), - loaders(({ values }) => ({ + loaders(({ values, actions }) => ({ plugins: [ {} as Record, { @@ -86,6 +96,26 @@ export const pipelineDestinationsLogic = kea([ [pluginConfig.id]: pluginConfig, } }, + + deleteNodeWebhook: async ({ destination }) => { + await deleteWithUndo({ + endpoint: `projects/${teamLogic.values.currentTeamId}/plugin_configs`, + object: { + id: destination.id, + name: destination.name, + }, + callback: (undo) => { + if (undo) { + actions.loadPluginConfigs() + } + }, + }) + + const pluginConfigs = { ...values.pluginConfigs } + delete pluginConfigs[destination.id] + + return pluginConfigs + }, }, ], batchExportConfigs: [ @@ -109,9 +139,11 @@ export const pipelineDestinationsLogic = kea([ }, deleteNodeBatchExport: async ({ destination }) => { await api.batchExports.delete(destination.id) - return Object.fromEntries( - Object.entries(values.batchExportConfigs).filter(([id]) => id !== destination.id) - ) + + const batchExportConfigs = { ...values.batchExportConfigs } + delete batchExportConfigs[destination.id] + + return batchExportConfigs }, updateBatchExportConfig: ({ batchExportConfig }) => { return { ...values.batchExportConfigs, [batchExportConfig.id]: batchExportConfig } @@ -138,6 +170,27 @@ export const pipelineDestinationsLogic = kea([ // TODO: Support pagination? return (await api.hogFunctions.list()).results }, + + deleteNodeHogFunction: async ({ destination }) => { + if (destination.backend !== PipelineBackend.HogFunction) { + return values.hogFunctions + } + + await deleteWithUndo({ + endpoint: `projects/${teamLogic.values.currentTeamId}/hog_functions`, + object: { + id: destination.hog_function.id, + name: destination.name, + }, + callback: (undo) => { + if (undo) { + actions.loadHogFunctions() + } + }, + }) + + return values.hogFunctions.filter((hogFunction) => hogFunction.id !== destination.hog_function.id) + }, }, ], })), @@ -195,7 +248,7 @@ export const pipelineDestinationsLogic = kea([ }, ], }), - listeners(({ values, actions, asyncActions }) => ({ + listeners(({ values, actions }) => ({ toggleNode: ({ destination, enabled }) => { if (enabled && !values.canEnableNewDestinations) { lemonToast.error('Data pipelines add-on is required for enabling new destinations.') @@ -207,18 +260,17 @@ export const pipelineDestinationsLogic = kea([ actions.toggleNodeBatchExport({ destination: destination, enabled: enabled }) } }, - deleteNode: async ({ destination }) => { - if (destination.backend === PipelineBackend.BatchExport) { - await asyncActions.deleteNodeBatchExport(destination) - } else { - await deleteWithUndo({ - endpoint: `projects/${teamLogic.values.currentTeamId}/plugin_configs`, - object: { - id: destination.id, - name: destination.name, - }, - callback: actions.loadPluginConfigs, - }) + deleteNode: ({ destination }) => { + switch (destination.backend) { + case PipelineBackend.Plugin: + actions.deleteNodeWebhook(destination) + break + case PipelineBackend.BatchExport: + actions.deleteNodeBatchExport(destination) + break + case PipelineBackend.HogFunction: + actions.deleteNodeHogFunction(destination) + break } }, })), diff --git a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.scss b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.scss new file mode 100644 index 0000000000000..4e7140c67d64b --- /dev/null +++ b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.scss @@ -0,0 +1,9 @@ +.HogFunctionInputCode { + .monaco-editor { + border-radius: var(--radius); + + .overflow-guard { + border-radius: var(--radius); + } + } +} diff --git a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.tsx b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.tsx index 2f3cc63865729..f17226633c5f2 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.tsx +++ b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionInputs.tsx @@ -1,3 +1,5 @@ +import './HogFunctionInputs.scss' + import { Monaco } from '@monaco-editor/react' import { IconPencil, IconPlus, IconX } from '@posthog/icons' import { LemonButton, LemonCheckbox, LemonInput, LemonSelect } from '@posthog/lemon-ui' @@ -5,6 +7,7 @@ import { useValues } from 'kea' import { CodeEditor } from 'lib/components/CodeEditors' import { languages } from 'monaco-editor' import { useEffect, useMemo, useState } from 'react' +import { AutoSizer } from 'react-virtualized/dist/es/AutoSizer' import { groupsModel } from '~/models/groupsModel' import { HogFunctionInputSchemaType } from '~/types' @@ -74,11 +77,21 @@ function JsonConfigField(props: { onChange?: (value: string) => void className: string autoFocus: boolean - value?: string | object + value?: string }): JSX.Element { const suggestions = useAutocompleteOptions() const [monaco, setMonaco] = useState() + const [height, setHeight] = useState(200) + const [manualHeight, setManualHeight] = useState() + + useEffect(() => { + const value = typeof props.value !== 'string' ? JSON.stringify(props.value, null, 2) : props.value + const lineCount = (value?.split('\n').length ?? 1) + 1 + const lineHeight = 18 + setHeight(lineHeight * lineCount) + }, [props.value]) + useEffect(() => { if (!monaco) { return @@ -125,34 +138,64 @@ function JsonConfigField(props: { }, [suggestions, monaco]) return ( - props.onChange?.(v ?? '')} - options={{ - lineNumbers: 'off', - minimap: { - enabled: false, - }, - quickSuggestions: { - other: true, - strings: true, - }, - suggest: { - showWords: false, - showFields: false, - showKeywords: false, - }, - scrollbar: { - vertical: 'hidden', - verticalScrollbarSize: 0, - }, +
{ - setMonaco(monaco) - }} - /> + > + + {({ height }) => ( + props.onChange?.(v ?? '')} + height={height - 2} // Account for border + options={{ + lineNumbers: 'off', + minimap: { + enabled: false, + }, + quickSuggestions: { + other: true, + strings: true, + }, + suggest: { + showWords: false, + showFields: false, + showKeywords: false, + }, + scrollbar: { + vertical: 'hidden', + verticalScrollbarSize: 0, + }, + }} + onMount={(_editor, monaco) => { + setMonaco(monaco) + }} + /> + )} + + + {/* Using a standard resize css means we need overflow-hidden which hides parts of the editor unnecessarily */} +
{ + const startY = e.clientY + const startHeight = height + const onMouseMove = (event: MouseEvent): void => { + setManualHeight(startHeight + event.clientY - startY) + } + const onMouseUp = (): void => { + window.removeEventListener('mousemove', onMouseMove) + window.removeEventListener('mouseup', onMouseUp) + } + window.addEventListener('mousemove', onMouseMove) + window.addEventListener('mouseup', onMouseUp) + }} + /> +
) } diff --git a/frontend/src/scenes/session-recordings/apm/playerInspector/ItemPerformanceEvent.tsx b/frontend/src/scenes/session-recordings/apm/playerInspector/ItemPerformanceEvent.tsx index 738d80016dcf2..e195319f037f9 100644 --- a/frontend/src/scenes/session-recordings/apm/playerInspector/ItemPerformanceEvent.tsx +++ b/frontend/src/scenes/session-recordings/apm/playerInspector/ItemPerformanceEvent.tsx @@ -189,6 +189,7 @@ export function ItemPerformanceEvent({ ) : (
+ {/* We only show the status if it exists and is an error status */} {otherProps.response_status && otherProps.response_status >= 400 ? ( diff --git a/frontend/src/scenes/surveys/QuestionBranchingInput.tsx b/frontend/src/scenes/surveys/QuestionBranchingInput.tsx index 96c6ea55912d6..be078d80f050f 100644 --- a/frontend/src/scenes/surveys/QuestionBranchingInput.tsx +++ b/frontend/src/scenes/surveys/QuestionBranchingInput.tsx @@ -3,8 +3,9 @@ import './EditSurvey.scss' import { LemonSelect } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { LemonField } from 'lib/lemon-ui/LemonField' +import { truncate } from 'lib/utils' -import { MultipleSurveyQuestion, RatingSurveyQuestion, SurveyQuestionBranchingType } from '~/types' +import { MultipleSurveyQuestion, RatingSurveyQuestion, SurveyQuestionBranchingType, SurveyQuestionType } from '~/types' import { surveyLogic } from './surveyLogic' @@ -16,7 +17,7 @@ export function QuestionBranchingInput({ question: RatingSurveyQuestion | MultipleSurveyQuestion }): JSX.Element { const { survey, getBranchingDropdownValue } = useValues(surveyLogic) - const { setQuestionBranching } = useActions(surveyLogic) + const { setQuestionBranchingType } = useActions(surveyLogic) const availableNextQuestions = survey.questions .map((question, questionIndex) => ({ @@ -25,6 +26,8 @@ export function QuestionBranchingInput({ })) .filter((_, idx) => questionIndex !== idx) const branchingDropdownValue = getBranchingDropdownValue(questionIndex, question) + const hasResponseBasedBranching = + question.type === SurveyQuestionType.Rating || question.type === SurveyQuestionType.SingleChoice return ( <> @@ -33,7 +36,14 @@ export function QuestionBranchingInput({ className="max-w-80 whitespace-nowrap" value={branchingDropdownValue} data-attr={`branching-question-${questionIndex}`} - onSelect={(value) => setQuestionBranching(questionIndex, value)} + onSelect={(type) => { + let specificQuestionIndex + if (type.startsWith(SurveyQuestionBranchingType.SpecificQuestion)) { + specificQuestionIndex = parseInt(type.split(':')[1]) + type = SurveyQuestionBranchingType.SpecificQuestion + } + setQuestionBranchingType(questionIndex, type, specificQuestionIndex) + }} options={[ ...(questionIndex < survey.questions.length - 1 ? [ @@ -47,22 +57,122 @@ export function QuestionBranchingInput({ label: 'Confirmation message', value: SurveyQuestionBranchingType.ConfirmationMessage, }, - { - label: 'Specific question based on answer', - value: SurveyQuestionBranchingType.ResponseBased, - }, + ...(hasResponseBasedBranching + ? [ + { + label: 'Specific question based on answer', + value: SurveyQuestionBranchingType.ResponseBased, + }, + ] + : []), ...availableNextQuestions.map((question) => ({ - label: `${question.questionIndex + 1}. ${question.question}`, + label: truncate(`${question.questionIndex + 1}. ${question.question}`, 40), value: `${SurveyQuestionBranchingType.SpecificQuestion}:${question.questionIndex}`, })), ]} /> {branchingDropdownValue === SurveyQuestionBranchingType.ResponseBased && ( -
- TODO: dropdowns for the response-based branching -
+ )} ) } + +function QuestionResponseBasedBranchingInput({ + questionIndex, + question, +}: { + questionIndex: number + question: RatingSurveyQuestion | MultipleSurveyQuestion +}): JSX.Element { + const { survey, getResponseBasedBranchingDropdownValue } = useValues(surveyLogic) + const { setResponseBasedBranchingForQuestion } = useActions(surveyLogic) + + const availableNextQuestions = survey.questions + .map((question, questionIndex) => ({ + ...question, + questionIndex, + })) + .filter((_, idx) => questionIndex !== idx) + + let config: { value: string | number; label: string }[] = [] + + if (question.type === SurveyQuestionType.Rating && question.scale === 3) { + config = [ + { value: 'negative', label: '1 (Negative)' }, + { value: 'neutral', label: '2 (Neutral)' }, + { value: 'positive', label: '3 (Positive)' }, + ] + } else if (question.type === SurveyQuestionType.Rating && question.scale === 5) { + config = [ + { value: 'negative', label: '1 to 2 (Negative)' }, + { value: 'neutral', label: '3 (Neutral)' }, + { value: 'positive', label: '4 to 5 (Positive)' }, + ] + } else if (question.type === SurveyQuestionType.Rating && question.scale === 10) { + config = [ + // NPS categories + { value: 'detractors', label: '0 to 6 (Detractors)' }, + { value: 'passives', label: '7 to 8 (Passives)' }, + { value: 'promoters', label: '9 to 10 (Promoters)' }, + ] + } else if (question.type === SurveyQuestionType.SingleChoice) { + config = question.choices.map((choice, choiceIndex) => ({ + value: choiceIndex, + label: `Option ${choiceIndex + 1} ("${truncate(choice, 15)}")`, + })) + } + + return ( +
+ {config.map(({ value, label }, i) => ( +
+
+
+ If the answer is {label}, go to: +
+
+
+ { + let specificQuestionIndex + if (nextStep.startsWith(SurveyQuestionBranchingType.SpecificQuestion)) { + specificQuestionIndex = parseInt(nextStep.split(':')[1]) + nextStep = SurveyQuestionBranchingType.SpecificQuestion + } + setResponseBasedBranchingForQuestion( + questionIndex, + value, + nextStep, + specificQuestionIndex + ) + }} + options={[ + ...(questionIndex < survey.questions.length - 1 + ? [ + { + label: 'Next question', + value: SurveyQuestionBranchingType.NextQuestion, + }, + ] + : []), + { + label: 'Confirmation message', + value: SurveyQuestionBranchingType.ConfirmationMessage, + }, + ...availableNextQuestions.map((question) => ({ + label: truncate(`${question.questionIndex + 1}. ${question.question}`, 20), + value: `${SurveyQuestionBranchingType.SpecificQuestion}:${question.questionIndex}`, + })), + ]} + /> +
+
+ ))} +
+ ) +} diff --git a/frontend/src/scenes/surveys/SurveyEditQuestionRow.tsx b/frontend/src/scenes/surveys/SurveyEditQuestionRow.tsx index ec3b03d54b41d..e1b24b1bea182 100644 --- a/frontend/src/scenes/surveys/SurveyEditQuestionRow.tsx +++ b/frontend/src/scenes/surveys/SurveyEditQuestionRow.tsx @@ -87,11 +87,9 @@ export function SurveyEditQuestionHeader({ export function SurveyEditQuestionGroup({ index, question }: { index: number; question: any }): JSX.Element { const { survey, descriptionContentType } = useValues(surveyLogic) - const { setDefaultForQuestionType, setSurveyValue } = useActions(surveyLogic) + const { setDefaultForQuestionType, setSurveyValue, resetBranchingForQuestion } = useActions(surveyLogic) const { featureFlags } = useValues(enabledFeaturesLogic) - const hasBranching = - featureFlags[FEATURE_FLAGS.SURVEYS_BRANCHING_LOGIC] && - (question.type === SurveyQuestionType.Rating || question.type === SurveyQuestionType.SingleChoice) + const hasBranching = featureFlags[FEATURE_FLAGS.SURVEYS_BRANCHING_LOGIC] const initialDescriptionContentType = descriptionContentType(index) ?? 'text' @@ -134,6 +132,7 @@ export function SurveyEditQuestionGroup({ index, question }: { index: number; qu editingDescription, editingThankYouMessage ) + resetBranchingForQuestion(index) }} options={[ { @@ -201,6 +200,7 @@ export function SurveyEditQuestionGroup({ index, question }: { index: number; qu const newQuestions = [...survey.questions] newQuestions[index] = newQuestion setSurveyValue('questions', newQuestions) + resetBranchingForQuestion(index) }} /> @@ -212,8 +212,17 @@ export function SurveyEditQuestionGroup({ index, question }: { index: number; qu label: '1 - 5', value: 5, }, - ...(question.display === 'number' ? [{ label: '0 - 10', value: 10 }] : []), + ...(question.display === 'number' + ? [{ label: '0 - 10 (Net Promoter Score)', value: 10 }] + : []), ]} + onChange={(val) => { + const newQuestion = { ...survey.questions[index], scale: val } + const newQuestions = [...survey.questions] + newQuestions[index] = newQuestion + setSurveyValue('questions', newQuestions) + resetBranchingForQuestion(index) + }} />
diff --git a/frontend/src/scenes/surveys/constants.tsx b/frontend/src/scenes/surveys/constants.tsx index 738e51b2a209a..6411868691bbd 100644 --- a/frontend/src/scenes/surveys/constants.tsx +++ b/frontend/src/scenes/surveys/constants.tsx @@ -1,3 +1,5 @@ +import { allOperatorsMapping } from 'lib/utils' + import { Survey, SurveyQuestionDescriptionContentType, @@ -17,10 +19,14 @@ export const SurveyQuestionLabel = { [SurveyQuestionType.MultipleChoice]: 'Multiple choice select', } +// Create SurveyUrlMatchTypeLabels using allOperatorsMapping export const SurveyUrlMatchTypeLabels = { - [SurveyUrlMatchType.Contains]: '∋ contains', - [SurveyUrlMatchType.Regex]: '∼ matches regex', - [SurveyUrlMatchType.Exact]: '= equals', + [SurveyUrlMatchType.Exact]: allOperatorsMapping[SurveyUrlMatchType.Exact], + [SurveyUrlMatchType.IsNot]: allOperatorsMapping[SurveyUrlMatchType.IsNot], + [SurveyUrlMatchType.Contains]: allOperatorsMapping[SurveyUrlMatchType.Contains], + [SurveyUrlMatchType.NotIContains]: allOperatorsMapping[SurveyUrlMatchType.NotIContains], + [SurveyUrlMatchType.Regex]: allOperatorsMapping[SurveyUrlMatchType.Regex], + [SurveyUrlMatchType.NotRegex]: allOperatorsMapping[SurveyUrlMatchType.NotRegex], } export const defaultSurveyAppearance = { diff --git a/frontend/src/scenes/surveys/surveyLogic.test.ts b/frontend/src/scenes/surveys/surveyLogic.test.ts index 56e6615beb36f..4b12c4b1459f1 100644 --- a/frontend/src/scenes/surveys/surveyLogic.test.ts +++ b/frontend/src/scenes/surveys/surveyLogic.test.ts @@ -1,9 +1,9 @@ -import { expectLogic } from 'kea-test-utils' +import { expectLogic, partial } from 'kea-test-utils' import { surveyLogic } from 'scenes/surveys/surveyLogic' import { useMocks } from '~/mocks/jest' import { initKeaTests } from '~/test/init' -import { Survey, SurveyQuestionType, SurveyType } from '~/types' +import { Survey, SurveyQuestionBranchingType, SurveyQuestionType, SurveyType } from '~/types' const MULTIPLE_CHOICE_SURVEY: Survey = { id: '018b22a3-09b1-0000-2f5b-1bd8352ceec9', @@ -398,3 +398,439 @@ describe('single choice survey with open choice logic', () => { }) }) }) + +describe('set response-based survey branching', () => { + let logic: ReturnType + + beforeEach(() => { + initKeaTests() + logic = surveyLogic({ id: 'new' }) + logic.mount() + }) + + const SURVEY: Survey = { + id: '118b22a3-09b1-0000-2f5b-1bd8352ceec9', + name: 'My survey', + description: '', + type: SurveyType.Popover, + linked_flag: null, + linked_flag_id: null, + targeting_flag: null, + questions: [], + conditions: null, + appearance: { + position: 'right', + whiteLabel: false, + borderColor: '#c9c6c6', + placeholder: '', + backgroundColor: '#eeeded', + submitButtonText: 'Submit', + ratingButtonColor: 'white', + submitButtonColor: 'black', + thankYouMessageHeader: 'Thank you for your feedback!', + displayThankYouMessage: true, + ratingButtonActiveColor: 'black', + }, + created_at: '2023-10-12T06:46:32.113745Z', + created_by: { + id: 1, + uuid: '018aa8a6-10e8-0000-dba2-0e956f7bae38', + distinct_id: 'TGqg9Cn4jLkj9X87oXni9ZPBD6VbOxMtGV1GfJeB5LO', + first_name: 'test', + email: 'test@posthog.com', + is_email_verified: false, + }, + start_date: '2023-10-12T06:46:34.482000Z', + end_date: null, + archived: false, + targeting_flag_filters: undefined, + responses_limit: null, + } + + describe('main', () => { + // Single-choice question + it('set response-based branching for a single-choice question', async () => { + SURVEY.questions = [ + { + type: SurveyQuestionType.SingleChoice, + choices: ['Yes', 'No'], + question: 'Are you happy with our service?', + description: '', + }, + { + type: SurveyQuestionType.Open, + question: 'Glad to hear that. Tell us more!', + description: '', + }, + { + type: SurveyQuestionType.Open, + question: 'Sorry to hear that. Tell us more!', + description: '', + }, + ] + + await expectLogic(logic, () => { + logic.actions.loadSurveySuccess(SURVEY) + }).toDispatchActions(['loadSurveySuccess']) + + const questionIndex = 0 + + await expectLogic(logic, () => { + logic.actions.setQuestionBranchingType( + questionIndex, + SurveyQuestionBranchingType.ResponseBased, + undefined + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'Yes', + SurveyQuestionBranchingType.SpecificQuestion, + 1 + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'No', + SurveyQuestionBranchingType.SpecificQuestion, + 2 + ) + }) + .toDispatchActions([ + 'setQuestionBranchingType', + 'setResponseBasedBranchingForQuestion', + 'setResponseBasedBranchingForQuestion', + ]) + .toMatchValues({ + survey: partial({ + questions: [ + { + ...SURVEY.questions[0], + branching: { + type: SurveyQuestionBranchingType.ResponseBased, + responseValues: { Yes: 1, No: 2 }, + }, + }, + { ...SURVEY.questions[1] }, + { ...SURVEY.questions[2] }, + ], + }), + }) + }) + + // Rating question, scale 1-3 + it('set response-based branching for a rating question with scale 3', async () => { + SURVEY.questions = [ + { + type: SurveyQuestionType.Rating, + question: 'How happy are you?', + description: '', + display: 'number', + scale: 3, + lowerBoundLabel: 'Unhappy', + upperBoundLabel: 'Happy', + buttonText: 'Submit', + }, + { + type: SurveyQuestionType.Open, + question: 'Sorry to hear that. Tell us more!', + description: '', + }, + { + type: SurveyQuestionType.Open, + question: 'Seems you are not completely happy. Tell us more!', + description: '', + }, + { + type: SurveyQuestionType.Open, + question: 'Glad to hear that. Tell us more!', + description: '', + }, + ] + + await expectLogic(logic, () => { + logic.actions.loadSurveySuccess(SURVEY) + }).toDispatchActions(['loadSurveySuccess']) + + const questionIndex = 0 + + await expectLogic(logic, () => { + logic.actions.setQuestionBranchingType( + questionIndex, + SurveyQuestionBranchingType.ResponseBased, + undefined + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'negative', + SurveyQuestionBranchingType.SpecificQuestion, + 1 + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'neutral', + SurveyQuestionBranchingType.SpecificQuestion, + 2 + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'positive', + SurveyQuestionBranchingType.SpecificQuestion, + 3 + ) + }) + .toDispatchActions([ + 'setQuestionBranchingType', + 'setResponseBasedBranchingForQuestion', + 'setResponseBasedBranchingForQuestion', + 'setResponseBasedBranchingForQuestion', + ]) + .toMatchValues({ + survey: partial({ + questions: [ + { + ...SURVEY.questions[0], + branching: { + type: SurveyQuestionBranchingType.ResponseBased, + responseValues: { negative: 1, neutral: 2, positive: 3 }, + }, + }, + { ...SURVEY.questions[1] }, + { ...SURVEY.questions[2] }, + { ...SURVEY.questions[3] }, + ], + }), + }) + }) + + // Rating question, scale 1-5 + it('set response-based branching for a rating question with scale 5', async () => { + SURVEY.questions = [ + { + type: SurveyQuestionType.Rating, + question: 'How happy are you?', + description: '', + display: 'number', + scale: 5, + lowerBoundLabel: 'Unhappy', + upperBoundLabel: 'Happy', + buttonText: 'Submit', + }, + { + type: SurveyQuestionType.Open, + question: 'Sorry to hear that. Tell us more!', + description: '', + }, + { + type: SurveyQuestionType.Open, + question: 'Seems you are not completely happy. Tell us more!', + description: '', + }, + { + type: SurveyQuestionType.Open, + question: 'Glad to hear that. Tell us more!', + description: '', + }, + ] + + await expectLogic(logic, () => { + logic.actions.loadSurveySuccess(SURVEY) + }).toDispatchActions(['loadSurveySuccess']) + + const questionIndex = 0 + + await expectLogic(logic, () => { + logic.actions.setQuestionBranchingType( + questionIndex, + SurveyQuestionBranchingType.ResponseBased, + undefined + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'negative', + SurveyQuestionBranchingType.SpecificQuestion, + 1 + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'neutral', + SurveyQuestionBranchingType.SpecificQuestion, + 2 + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'positive', + SurveyQuestionBranchingType.SpecificQuestion, + 3 + ) + }) + .toDispatchActions([ + 'setQuestionBranchingType', + 'setResponseBasedBranchingForQuestion', + 'setResponseBasedBranchingForQuestion', + 'setResponseBasedBranchingForQuestion', + ]) + .toMatchValues({ + survey: partial({ + questions: [ + { + ...SURVEY.questions[0], + branching: { + type: SurveyQuestionBranchingType.ResponseBased, + responseValues: { negative: 1, neutral: 2, positive: 3 }, + }, + }, + { ...SURVEY.questions[1] }, + { ...SURVEY.questions[2] }, + { ...SURVEY.questions[3] }, + ], + }), + }) + }) + + // Rating question, scale 0-10 (NPS) + it('set response-based branching for a rating question with scale 10', async () => { + SURVEY.questions = [ + { + type: SurveyQuestionType.Rating, + question: 'How happy are you?', + description: '', + display: 'number', + scale: 10, + lowerBoundLabel: 'Unhappy', + upperBoundLabel: 'Happy', + buttonText: 'Submit', + }, + { + type: SurveyQuestionType.Open, + question: 'Sorry to hear that. Tell us more!', + description: '', + }, + { + type: SurveyQuestionType.Open, + question: 'Seems you are not completely happy. Tell us more!', + description: '', + }, + { + type: SurveyQuestionType.Open, + question: 'Glad to hear that. Tell us more!', + description: '', + }, + ] + + await expectLogic(logic, () => { + logic.actions.loadSurveySuccess(SURVEY) + }).toDispatchActions(['loadSurveySuccess']) + + const questionIndex = 0 + + await expectLogic(logic, () => { + logic.actions.setQuestionBranchingType( + questionIndex, + SurveyQuestionBranchingType.ResponseBased, + undefined + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'detractors', + SurveyQuestionBranchingType.SpecificQuestion, + 1 + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'passives', + SurveyQuestionBranchingType.SpecificQuestion, + 2 + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 'promoters', + SurveyQuestionBranchingType.SpecificQuestion, + 3 + ) + }) + .toDispatchActions([ + 'setQuestionBranchingType', + 'setResponseBasedBranchingForQuestion', + 'setResponseBasedBranchingForQuestion', + 'setResponseBasedBranchingForQuestion', + ]) + .toMatchValues({ + survey: partial({ + questions: [ + { + ...SURVEY.questions[0], + branching: { + type: SurveyQuestionBranchingType.ResponseBased, + responseValues: { detractors: 1, passives: 2, promoters: 3 }, + }, + }, + { ...SURVEY.questions[1] }, + { ...SURVEY.questions[2] }, + { ...SURVEY.questions[3] }, + ], + }), + }) + }) + + // Branch out to Next question / Confirmation message + it('branch out to next question or confirmation message', async () => { + SURVEY.questions = [ + { + type: SurveyQuestionType.SingleChoice, + choices: ['Yes', 'No'], + question: 'Are you happy with our service?', + description: '', + }, + { + type: SurveyQuestionType.Open, + question: 'Sorry to hear that. Tell us more!', + description: '', + }, + ] + + await expectLogic(logic, () => { + logic.actions.loadSurveySuccess(SURVEY) + }).toDispatchActions(['loadSurveySuccess']) + + const questionIndex = 0 + + await expectLogic(logic, () => { + logic.actions.setQuestionBranchingType( + questionIndex, + SurveyQuestionBranchingType.ResponseBased, + undefined + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 0, + SurveyQuestionBranchingType.ConfirmationMessage, + undefined + ) + logic.actions.setResponseBasedBranchingForQuestion( + questionIndex, + 1, + SurveyQuestionBranchingType.NextQuestion, + undefined + ) + }) + .toDispatchActions([ + 'setQuestionBranchingType', + 'setResponseBasedBranchingForQuestion', + 'setResponseBasedBranchingForQuestion', + ]) + .toMatchValues({ + survey: partial({ + questions: [ + { + ...SURVEY.questions[0], + branching: { + type: SurveyQuestionBranchingType.ResponseBased, + responseValues: { 0: SurveyQuestionBranchingType.ConfirmationMessage }, // Branching out to "Next question" is implicit + }, + }, + { ...SURVEY.questions[1] }, + ], + }), + }) + }) + }) +}) diff --git a/frontend/src/scenes/surveys/surveyLogic.tsx b/frontend/src/scenes/surveys/surveyLogic.tsx index 9b3964b64fdf3..b66a5326228a3 100644 --- a/frontend/src/scenes/surveys/surveyLogic.tsx +++ b/frontend/src/scenes/surveys/surveyLogic.tsx @@ -157,7 +157,18 @@ export const surveyLogic = kea([ isEditingDescription, isEditingThankYouMessage, }), - setQuestionBranching: (questionIndex, value) => ({ questionIndex, value }), + setQuestionBranchingType: (questionIndex, type, specificQuestionIndex) => ({ + questionIndex, + type, + specificQuestionIndex, + }), + setResponseBasedBranchingForQuestion: (questionIndex, responseValue, nextStep, specificQuestionIndex) => ({ + questionIndex, + responseValue, + nextStep, + specificQuestionIndex, + }), + resetBranchingForQuestion: (questionIndex) => ({ questionIndex }), archiveSurvey: true, setWritingHTMLDescription: (writingHTML: boolean) => ({ writingHTML }), setSurveyTemplateValues: (template: any) => ({ template }), @@ -661,38 +672,87 @@ export const surveyLogic = kea([ const newTemplateSurvey = { ...NEW_SURVEY, ...template } return newTemplateSurvey }, - setQuestionBranching: (state, { questionIndex, value }) => { + setQuestionBranchingType: (state, { questionIndex, type, specificQuestionIndex }) => { const newQuestions = [...state.questions] const question = newQuestions[questionIndex] - if ( - question.type !== SurveyQuestionType.Rating && - question.type !== SurveyQuestionType.SingleChoice - ) { - throw new Error( - `Survey question type must be ${SurveyQuestionType.Rating} or ${SurveyQuestionType.SingleChoice}` - ) - } - - if (value === SurveyQuestionBranchingType.NextQuestion) { + if (type === SurveyQuestionBranchingType.NextQuestion) { delete question.branching - } else if (value === SurveyQuestionBranchingType.ConfirmationMessage) { + } else if (type === SurveyQuestionBranchingType.ConfirmationMessage) { question.branching = { type: SurveyQuestionBranchingType.ConfirmationMessage, } - } else if (value === SurveyQuestionBranchingType.ResponseBased) { + } else if (type === SurveyQuestionBranchingType.ResponseBased) { + if ( + question.type !== SurveyQuestionType.Rating && + question.type !== SurveyQuestionType.SingleChoice + ) { + throw new Error( + `Survey question type must be ${SurveyQuestionType.Rating} or ${SurveyQuestionType.SingleChoice}` + ) + } + question.branching = { type: SurveyQuestionBranchingType.ResponseBased, - responseValue: {}, + responseValues: {}, } - } else if (value.startsWith(SurveyQuestionBranchingType.SpecificQuestion)) { - const nextQuestionIndex = parseInt(value.split(':')[1]) + } else if (type === SurveyQuestionBranchingType.SpecificQuestion) { question.branching = { type: SurveyQuestionBranchingType.SpecificQuestion, - index: nextQuestionIndex, + index: specificQuestionIndex, + } + } + + newQuestions[questionIndex] = question + return { + ...state, + questions: newQuestions, + } + }, + setResponseBasedBranchingForQuestion: ( + state, + { questionIndex, responseValue, nextStep, specificQuestionIndex } + ) => { + const newQuestions = [...state.questions] + const question = newQuestions[questionIndex] + + if ( + question.type !== SurveyQuestionType.Rating && + question.type !== SurveyQuestionType.SingleChoice + ) { + throw new Error( + `Survey question type must be ${SurveyQuestionType.Rating} or ${SurveyQuestionType.SingleChoice}` + ) + } + + if (question.branching?.type !== SurveyQuestionBranchingType.ResponseBased) { + throw new Error( + `Survey question branching type must be ${SurveyQuestionBranchingType.ResponseBased}` + ) + } + + if ('responseValues' in question.branching) { + if (nextStep === SurveyQuestionBranchingType.NextQuestion) { + delete question.branching.responseValues[responseValue] + } else if (nextStep === SurveyQuestionBranchingType.ConfirmationMessage) { + question.branching.responseValues[responseValue] = + SurveyQuestionBranchingType.ConfirmationMessage + } else if (nextStep === SurveyQuestionBranchingType.SpecificQuestion) { + question.branching.responseValues[responseValue] = specificQuestionIndex } } + newQuestions[questionIndex] = question + return { + ...state, + questions: newQuestions, + } + }, + resetBranchingForQuestion: (state, { questionIndex }) => { + const newQuestions = [...state.questions] + const question = newQuestions[questionIndex] + delete question.branching + newQuestions[questionIndex] = question return { ...state, @@ -943,6 +1003,32 @@ export const surveyLogic = kea([ return SurveyQuestionBranchingType.NextQuestion } + return SurveyQuestionBranchingType.ConfirmationMessage + }, + ], + getResponseBasedBranchingDropdownValue: [ + (s) => [s.survey], + (survey) => (questionIndex: number, question: RatingSurveyQuestion | MultipleSurveyQuestion, response) => { + if (!question.branching || !('responseValues' in question.branching)) { + return SurveyQuestionBranchingType.NextQuestion + } + + // If a value is mapped onto an integer, we're redirecting to a specific question + if (Number.isInteger(question.branching.responseValues[response])) { + const nextQuestionIndex = question.branching.responseValues[response] + return `${SurveyQuestionBranchingType.SpecificQuestion}:${nextQuestionIndex}` + } + + // If any other value is present (practically only Confirmation message), return that value + if (question.branching?.responseValues?.[response]) { + return question.branching.responseValues[response] + } + + // No branching specified, default to Next question / Confirmation message + if (questionIndex < survey.questions.length - 1) { + return SurveyQuestionBranchingType.NextQuestion + } + return SurveyQuestionBranchingType.ConfirmationMessage }, ], diff --git a/frontend/src/types.ts b/frontend/src/types.ts index b3cca7bd90f98..ed3339591df09 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2615,9 +2615,12 @@ export interface Survey { } export enum SurveyUrlMatchType { - Exact = 'exact', - Contains = 'icontains', - Regex = 'regex', + Exact = PropertyOperator.Exact, + IsNot = PropertyOperator.IsNot, + Contains = PropertyOperator.IContains, + NotIContains = PropertyOperator.NotIContains, + Regex = PropertyOperator.Regex, + NotRegex = PropertyOperator.NotRegex, } export enum SurveyType { @@ -2660,6 +2663,11 @@ export interface SurveyQuestionBase { descriptionContentType?: SurveyQuestionDescriptionContentType optional?: boolean buttonText?: string + branching?: + | NextQuestionBranching + | ConfirmationMessageBranching + | ResponseBasedBranching + | SpecificQuestionBranching } export interface BasicSurveyQuestion extends SurveyQuestionBase { @@ -2723,7 +2731,7 @@ interface ConfirmationMessageBranching { interface ResponseBasedBranching { type: SurveyQuestionBranchingType.ResponseBased - responseValue: Record + responseValues: Record } interface SpecificQuestionBranching { @@ -3686,6 +3694,7 @@ export enum ActivityScope { SURVEY = 'Survey', EARLY_ACCESS_FEATURE = 'EarlyAccessFeature', COMMENT = 'Comment', + COHORT = 'Cohort', TEAM = 'Team', } diff --git a/hogvm/__tests__/__snapshots__/ifJump.hoge b/hogvm/__tests__/__snapshots__/ifJump.hoge new file mode 100644 index 0000000000000..0e946f4101f07 --- /dev/null +++ b/hogvm/__tests__/__snapshots__/ifJump.hoge @@ -0,0 +1,3 @@ +["_h", 42, 0, 36, 0, 32, "email", 45, 32, "", 36, 1, 11, 40, 12, 32, "ERROR - Email not found!", 2, "print", 1, 35, 32, +"3", 2, "print", 1, 35, 32, "1", 2, "print", 1, 35, 32, "", 36, 1, 11, 40, 14, 32, "ERROR - Email not found!", 2, +"print", 1, 35, 32, "3", 2, "print", 1, 35, 39, 6, 32, "else", 2, "print", 1, 35, 32, "1", 2, "print", 1, 35, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/ifJump.stdout b/hogvm/__tests__/__snapshots__/ifJump.stdout new file mode 100644 index 0000000000000..27f921abd174f --- /dev/null +++ b/hogvm/__tests__/__snapshots__/ifJump.stdout @@ -0,0 +1,3 @@ +1 +else +1 diff --git a/hogvm/__tests__/__snapshots__/stl.hoge b/hogvm/__tests__/__snapshots__/stl.hoge index b3613e9644619..b55b0303ac63a 100644 --- a/hogvm/__tests__/__snapshots__/stl.hoge +++ b/hogvm/__tests__/__snapshots__/stl.hoge @@ -1,3 +1,25 @@ ["_h", 32, "-- empty, notEmpty, length, lower, upper, reverse --", 2, "print", 1, 35, 32, "234", 2, "notEmpty", 1, 32, -"", 2, "empty", 1, 3, 2, 40, 11, 32, "123", 2, "length", 1, 2, "print", 1, 35, 32, "tdd4gh", 32, "Tdd4gh", 2, "lower", -1, 11, 40, 11, 32, "test", 2, "upper", 1, 2, "print", 1, 35, 32, "spinner", 2, "reverse", 1, 2, "print", 1, 35] +"", 2, "empty", 1, 3, 2, 40, 9, 32, "123", 2, "length", 1, 2, "print", 1, 35, 32, "tdd4gh", 32, "Tdd4gh", 2, "lower", 1, +11, 40, 9, 32, "test", 2, "upper", 1, 2, "print", 1, 35, 32, "spinner", 2, "reverse", 1, 2, "print", 1, 35, 32, "", 2, +"print", 1, 35, 32, "-- encodeURLComponent, decodeURLComponent --", 2, "print", 1, 35, 32, "http://www.google.com", 2, +"encodeURLComponent", 1, 2, "print", 1, 35, 32, "tom & jerry", 2, "encodeURLComponent", 1, 2, "print", 1, 35, 32, +"http://www.google.com", 2, "encodeURLComponent", 1, 2, "decodeURLComponent", 1, 2, "print", 1, 35, 32, "tom & jerry", +2, "encodeURLComponent", 1, 2, "decodeURLComponent", 1, 2, "print", 1, 35, 32, "", 2, "print", 1, 35, 32, +"-- base64Encode, base64Decode --", 2, "print", 1, 35, 32, "http://www.google.com", 2, "base64Encode", 1, 2, "print", 1, +35, 32, "tom & jerry", 2, "base64Encode", 1, 2, "print", 1, 35, 32, "http://www.google.com", 2, "base64Encode", 1, 2, +"base64Decode", 1, 2, "print", 1, 35, 32, "tom & jerry", 2, "base64Encode", 1, 2, "base64Decode", 1, 2, "print", 1, 35, +32, "", 2, "print", 1, 35, 32, "-- empty --", 2, "print", 1, 35, 31, 2, "empty", 1, 2, "print", 1, 35, 33, 0, 2, +"empty", 1, 2, "print", 1, 35, 33, 1, 2, "empty", 1, 2, "print", 1, 35, 33, -1, 2, "empty", 1, 2, "print", 1, 35, 34, +0.0, 2, "empty", 1, 2, "print", 1, 35, 34, 0.01, 2, "empty", 1, 2, "print", 1, 35, 32, "", 2, "empty", 1, 2, "print", 1, +35, 32, "string", 2, "empty", 1, 2, "print", 1, 35, 32, "0", 2, "empty", 1, 2, "print", 1, 35, 43, 0, 2, "empty", 1, 2, +"print", 1, 35, 42, 0, 2, "empty", 1, 2, "print", 1, 35, 2, "tuple", 0, 2, "empty", 1, 2, "print", 1, 35, 33, 0, 2, +"tuple", 1, 2, "empty", 1, 2, "print", 1, 35, 33, 2, 33, 1, 2, "tuple", 2, 2, "empty", 1, 2, "print", 1, 35, 32, "", 2, +"print", 1, 35, 32, "-- notEmpty --", 2, "print", 1, 35, 31, 2, "notEmpty", 1, 2, "print", 1, 35, 33, 0, 2, "notEmpty", +1, 2, "print", 1, 35, 33, 1, 2, "notEmpty", 1, 2, "print", 1, 35, 33, -1, 2, "notEmpty", 1, 2, "print", 1, 35, 34, 0.0, +2, "notEmpty", 1, 2, "print", 1, 35, 34, 0.01, 2, "notEmpty", 1, 2, "print", 1, 35, 32, "", 2, "notEmpty", 1, 2, +"print", 1, 35, 32, "string", 2, "notEmpty", 1, 2, "print", 1, 35, 32, "0", 2, "notEmpty", 1, 2, "print", 1, 35, 43, 0, +2, "notEmpty", 1, 2, "print", 1, 35, 42, 0, 2, "notEmpty", 1, 2, "print", 1, 35, 2, "tuple", 0, 2, "notEmpty", 1, 2, +"print", 1, 35, 33, 0, 2, "tuple", 1, 2, "notEmpty", 1, 2, "print", 1, 35, 33, 2, 33, 1, 2, "tuple", 2, 2, "notEmpty", +1, 2, "print", 1, 35, 32, "", 2, "print", 1, 35, 32, "-- replaceAll, replaceOne --", 2, "print", 1, 35, 32, "L", 32, +"l", 32, "hello world", 2, "replaceAll", 3, 2, "print", 1, 35, 32, "L", 32, "l", 32, "hello world", 2, "replaceOne", 3, +2, "print", 1, 35] diff --git a/hogvm/__tests__/__snapshots__/stl.stdout b/hogvm/__tests__/__snapshots__/stl.stdout index fe9463feee70a..5d85f96594bf7 100644 --- a/hogvm/__tests__/__snapshots__/stl.stdout +++ b/hogvm/__tests__/__snapshots__/stl.stdout @@ -2,3 +2,51 @@ 3 TEST rennips + +-- encodeURLComponent, decodeURLComponent -- +http%3A%2F%2Fwww.google.com +tom%20%26%20jerry +http://www.google.com +tom & jerry + +-- base64Encode, base64Decode -- +aHR0cDovL3d3dy5nb29nbGUuY29t +dG9tICYgamVycnk= +http://www.google.com +tom & jerry + +-- empty -- +true +true +false +false +true +false +true +false +false +true +true +true +false +false + +-- notEmpty -- +false +false +true +true +false +true +false +true +true +false +false +false +true +true + +-- replaceAll, replaceOne -- +heLLo worLd +heLlo world diff --git a/hogvm/__tests__/__snapshots__/tuples.hoge b/hogvm/__tests__/__snapshots__/tuples.hoge index c2efc1fd7f680..e5577761448b6 100644 --- a/hogvm/__tests__/__snapshots__/tuples.hoge +++ b/hogvm/__tests__/__snapshots__/tuples.hoge @@ -1,6 +1,6 @@ -["_h", 33, 1, 33, 2, 33, 3, 44, 3, 2, "print", 1, 35, 33, 1, 32, "2", 33, 3, 44, 3, 2, "print", 1, 35, 33, 1, 33, 2, 33, -3, 44, 2, 33, 4, 44, 3, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 33, 4, 44, 2, 44, 2, 33, 5, 44, 3, 2, "print", 1, 35, -33, 1, 33, 2, 33, 3, 44, 3, 36, 0, 33, 1, 45, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 33, 4, 44, 2, 44, 2, 33, 5, 44, 3, -33, 1, 45, 33, 1, 45, 33, 1, 45, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 33, 4, 44, 2, 44, 2, 33, 5, 44, 3, 33, 1, 45, -33, 1, 45, 33, 1, 45, 2, "print", 1, 35, 33, 1, 33, 1, 33, 2, 33, 3, 33, 4, 44, 2, 44, 2, 33, 5, 44, 3, 33, 1, 45, 33, -1, 45, 33, 1, 45, 6, 2, "print", 1, 35, 35] +["_h", 2, "tuple", 0, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 44, 3, 2, "print", 1, 35, 33, 1, 32, "2", 33, 3, 44, 3, 2, +"print", 1, 35, 33, 1, 33, 2, 33, 3, 44, 2, 33, 4, 44, 3, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 33, 4, 44, 2, 44, 2, +33, 5, 44, 3, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 44, 3, 36, 0, 33, 1, 45, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, +33, 4, 44, 2, 44, 2, 33, 5, 44, 3, 33, 1, 45, 33, 1, 45, 33, 1, 45, 2, "print", 1, 35, 33, 1, 33, 2, 33, 3, 33, 4, 44, +2, 44, 2, 33, 5, 44, 3, 33, 1, 45, 33, 1, 45, 33, 1, 45, 2, "print", 1, 35, 33, 1, 33, 1, 33, 2, 33, 3, 33, 4, 44, 2, +44, 2, 33, 5, 44, 3, 33, 1, 45, 33, 1, 45, 33, 1, 45, 6, 2, "print", 1, 35, 35] diff --git a/hogvm/__tests__/__snapshots__/tuples.stdout b/hogvm/__tests__/__snapshots__/tuples.stdout index 182639c8af8be..a602254cd3101 100644 --- a/hogvm/__tests__/__snapshots__/tuples.stdout +++ b/hogvm/__tests__/__snapshots__/tuples.stdout @@ -1,3 +1,4 @@ +tuple() (1, 2, 3) (1, '2', 3) (1, (2, 3), 4) diff --git a/hogvm/__tests__/ifJump.hog b/hogvm/__tests__/ifJump.hog new file mode 100644 index 0000000000000..4e4e7084dc798 --- /dev/null +++ b/hogvm/__tests__/ifJump.hog @@ -0,0 +1,20 @@ +let props := { +} +let email := props.email + +if (email == '') { + print('ERROR - Email not found!') + print('3') +} + +print('1') + + +if (email == '') { + print('ERROR - Email not found!') + print('3') +} else { + print('else') +} + +print('1') diff --git a/hogvm/__tests__/stl.hog b/hogvm/__tests__/stl.hog index 111b42e3abff9..7cb6c0fdba908 100644 --- a/hogvm/__tests__/stl.hog +++ b/hogvm/__tests__/stl.hog @@ -2,3 +2,52 @@ print('-- empty, notEmpty, length, lower, upper, reverse --') if (empty('') and notEmpty('234')) print(length('123')) if (lower('Tdd4gh') == 'tdd4gh') print(upper('test')) print(reverse('spinner')) +print('') +print('-- encodeURLComponent, decodeURLComponent --') +print(encodeURLComponent('http://www.google.com')) +print(encodeURLComponent('tom & jerry')) +print(decodeURLComponent(encodeURLComponent('http://www.google.com'))) +print(decodeURLComponent(encodeURLComponent('tom & jerry'))) +print('') +print('-- base64Encode, base64Decode --') +print(base64Encode('http://www.google.com')) +print(base64Encode('tom & jerry')) +print(base64Decode(base64Encode('http://www.google.com'))) +print(base64Decode(base64Encode('tom & jerry'))) +print('') +print('-- empty --') +print(empty(null)) +print(empty(0)) +print(empty(1)) +print(empty(-1)) +print(empty(0.0)) +print(empty(0.01)) +print(empty('')) +print(empty('string')) +print(empty('0')) +print(empty([])) +print(empty({})) +print(empty(tuple())) +print(empty(tuple(0))) +print(empty(tuple(1,2))) +print('') +print('-- notEmpty --') +print(notEmpty(null)) +print(notEmpty(0)) +print(notEmpty(1)) +print(notEmpty(-1)) +print(notEmpty(0.0)) +print(notEmpty(0.01)) +print(notEmpty('')) +print(notEmpty('string')) +print(notEmpty('0')) +print(notEmpty([])) +print(notEmpty({})) +print(notEmpty(tuple())) +print(notEmpty(tuple(0))) +print(notEmpty(tuple(1,2))) +print('') +print('-- replaceAll, replaceOne --') +print(replaceAll('hello world', 'l', 'L')) +print(replaceOne('hello world', 'l', 'L')) + diff --git a/hogvm/__tests__/tuples.hog b/hogvm/__tests__/tuples.hog index e7bd601c7ed2f..da57b0bd497c5 100644 --- a/hogvm/__tests__/tuples.hog +++ b/hogvm/__tests__/tuples.hog @@ -1,3 +1,4 @@ +print(tuple()) print((1, 2, 3)) print((1, '2', 3)) print((1, (2, 3), 4)) diff --git a/hogvm/python/stl/__init__.py b/hogvm/python/stl/__init__.py index 134b7335f92ac..f61674dd3ed1a 100644 --- a/hogvm/python/stl/__init__.py +++ b/hogvm/python/stl/__init__.py @@ -64,6 +64,10 @@ def notEmpty(name: str, args: list[Any], team: Optional["Team"], stdout: Optiona return bool(args[0]) +def _tuple(name: str, args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int): + return tuple(args) + + def lower(name: str, args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int): return args[0].lower() @@ -107,6 +111,42 @@ def jsonStringify(name: str, args: list[Any], team: Optional["Team"], stdout: Op return json.dumps(args[0]) +def base64Encode(name: str, args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + import base64 + + return base64.b64encode(args[0].encode()).decode() + + +def base64Decode(name: str, args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + import base64 + + return base64.b64decode(args[0].encode()).decode() + + +def encodeURLComponent( + name: str, args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int +) -> str: + import urllib.parse + + return urllib.parse.quote(args[0], safe="") + + +def decodeURLComponent( + name: str, args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int +) -> str: + import urllib.parse + + return urllib.parse.unquote(args[0]) + + +def replaceOne(name: str, args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + return args[0].replace(args[1], args[2], 1) + + +def replaceAll(name: str, args: list[Any], team: Optional["Team"], stdout: Optional[list[str]], timeout: int) -> str: + return args[0].replace(args[1], args[2]) + + STL: dict[str, Callable[[str, list[Any], Optional["Team"], list[str] | None, int], Any]] = { "concat": concat, "match": match, @@ -118,6 +158,7 @@ def jsonStringify(name: str, args: list[Any], team: Optional["Team"], stdout: Op "length": length, "empty": empty, "notEmpty": notEmpty, + "tuple": _tuple, "lower": lower, "upper": upper, "reverse": reverse, @@ -126,4 +167,10 @@ def jsonStringify(name: str, args: list[Any], team: Optional["Team"], stdout: Op "run": run, "jsonParse": jsonParse, "jsonStringify": jsonStringify, + "base64Encode": base64Encode, + "base64Decode": base64Decode, + "encodeURLComponent": encodeURLComponent, + "decodeURLComponent": decodeURLComponent, + "replaceOne": replaceOne, + "replaceAll": replaceAll, } diff --git a/hogvm/typescript/package.json b/hogvm/typescript/package.json index 3080a86f544f9..53c5805ebe241 100644 --- a/hogvm/typescript/package.json +++ b/hogvm/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@posthog/hogvm", - "version": "1.0.11", + "version": "1.0.13", "description": "PostHog Hog Virtual Machine", "types": "dist/index.d.ts", "main": "dist/index.js", diff --git a/hogvm/typescript/src/stl/print.ts b/hogvm/typescript/src/stl/print.ts index 26fad2f4080e9..b70e554a7256e 100644 --- a/hogvm/typescript/src/stl/print.ts +++ b/hogvm/typescript/src/stl/print.ts @@ -42,6 +42,9 @@ export function escapeIdentifier(identifier: string | number): string { export function printHogValue(obj: any): string { if (Array.isArray(obj)) { if ((obj as any).__isHogTuple) { + if (obj.length < 2) { + return `tuple(${obj.map(printHogValue).join(', ')})` + } return `(${obj.map(printHogValue).join(', ')})` } else { return `[${obj.map(printHogValue).join(', ')}]` diff --git a/hogvm/typescript/src/stl/stl.ts b/hogvm/typescript/src/stl/stl.ts index cb358360ff754..7966e58ff2d93 100644 --- a/hogvm/typescript/src/stl/stl.ts +++ b/hogvm/typescript/src/stl/stl.ts @@ -27,10 +27,26 @@ export const STL: Record return args[0].length }, empty: (args) => { + if (typeof args[0] === 'object') { + if (Array.isArray(args[0])) { + return args[0].length === 0 + } else if (args[0] === null) { + return true + } else if (args[0] instanceof Map) { + return args[0].size === 0 + } else { + return Object.keys(args[0]).length === 0 + } + } return !args[0] }, notEmpty: (args) => { - return !!args[0] + return !STL.empty(args, 'empty', 0) + }, + tuple: (args) => { + const tuple = args.slice() + ;(tuple as any).__isHogTuple = true + return tuple }, lower: (args) => { return args[0].toLowerCase() @@ -86,6 +102,31 @@ export const STL: Record } return JSON.stringify(convert(args[0])) }, + base64Encode: (args) => { + return Buffer.from(args[0]).toString('base64') + }, + base64Decode: (args) => { + return Buffer.from(args[0], 'base64').toString() + }, + tryBase64Decode: (args) => { + try { + return Buffer.from(args[0], 'base64').toString() + } catch (e) { + return '' + } + }, + encodeURLComponent(args) { + return encodeURIComponent(args[0]) + }, + decodeURLComponent(args) { + return decodeURIComponent(args[0]) + }, + replaceOne(args) { + return args[0].replace(args[1], args[2]) + }, + replaceAll(args) { + return args[0].replaceAll(args[1], args[2]) + }, } export const ASYNC_STL: Record Promise> = { diff --git a/mypy-baseline.txt b/mypy-baseline.txt index 300c901c82196..00e42297ced4f 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -152,6 +152,15 @@ ee/billing/quota_limiting.py:0: error: "object" has no attribute "get" [attr-de ee/billing/quota_limiting.py:0: error: Unsupported target for indexed assignment ("object") [index] ee/billing/quota_limiting.py:0: error: Unsupported target for indexed assignment ("object") [index] posthog/tasks/email.py:0: error: Module "django.utils.timezone" does not explicitly export attribute "timedelta" [attr-defined] +posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_histogram_bin_count" [union-attr] +posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr] +posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr] +posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr] +posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument 1 to "parse_expr" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type] +posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr] +posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr] +posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument "breakdown_field" to "get_properties_chain" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type] +posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_group_type_index" [union-attr] posthog/hogql_queries/insights/trends/aggregation_operations.py:0: error: List item 1 has incompatible type "str | None"; expected "str" [list-item] posthog/hogql_queries/insights/trends/aggregation_operations.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type] posthog/hogql_queries/insights/trends/aggregation_operations.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance @@ -298,19 +307,6 @@ posthog/hogql/query.py:0: error: Subclass of "SelectQuery" and "SelectUnionQuery posthog/queries/person_query.py:0: error: Incompatible type for lookup 'pk': (got "str | int | list[str]", expected "str | int") [misc] posthog/queries/event_query/event_query.py:0: error: Incompatible type for lookup 'pk': (got "str | int | list[str]", expected "str | int") [misc] posthog/hogql_queries/sessions_timeline_query_runner.py:0: error: Statement is unreachable [unreachable] -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr] -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_histogram_bin_count" [union-attr] -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr] -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument "exprs" to "Or" has incompatible type "list[CompareOperation]"; expected "list[Expr]" [arg-type] -posthog/hogql_queries/insights/trends/breakdown.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance -posthog/hogql_queries/insights/trends/breakdown.py:0: note: Consider using "Sequence" instead, which is covariant -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr] -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr] -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument 1 to "parse_expr" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type] -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr] -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr] -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument "breakdown_field" to "get_properties_chain" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type] -posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_group_type_index" [union-attr] posthog/hogql_queries/hogql_query_runner.py:0: error: Statement is unreachable [unreachable] posthog/hogql_queries/hogql_query_runner.py:0: error: Incompatible return value type (got "SelectQuery | SelectUnionQuery", expected "SelectQuery") [return-value] posthog/hogql_queries/events_query_runner.py:0: error: Statement is unreachable [unreachable] diff --git a/package.json b/package.json index 1eb41c4aff559..0c31a4fba5e43 100644 --- a/package.json +++ b/package.json @@ -146,7 +146,7 @@ "pmtiles": "^2.11.0", "postcss": "^8.4.31", "postcss-preset-env": "^9.3.0", - "posthog-js": "1.139.1", + "posthog-js": "1.139.2", "posthog-js-lite": "3.0.0", "prettier": "^2.8.8", "prop-types": "^15.7.2", diff --git a/plugin-server/src/utils/event.ts b/plugin-server/src/utils/event.ts index 3f02106b3b8a6..d6f4392f26812 100644 --- a/plugin-server/src/utils/event.ts +++ b/plugin-server/src/utils/event.ts @@ -10,7 +10,6 @@ import { PostIngestionEvent, RawClickHouseEvent, } from '../types' -import { status } from '../utils/status' import { chainToElements } from './db/elements-chain' import { personInitialAndUTMProperties, sanitizeString } from './db/utils' import { @@ -257,13 +256,6 @@ export function formPipelineEvent(message: Message): PipelineEvent { '$unset' in combinedEvent.properties) ) { setUsageInNonPersonEventsCounter.inc() - if (Math.random() < 0.001) { - status.info('👀', 'Found $set usage in non-person event', { - event: combinedEvent.event, - team_id: combinedEvent.team_id, - token: combinedEvent.token, - }) - } } const event: PipelineEvent = normalizeEvent({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 04a1d312b247f..e2792e5699e33 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -260,8 +260,8 @@ dependencies: specifier: ^9.3.0 version: 9.3.0(postcss@8.4.31) posthog-js: - specifier: 1.139.1 - version: 1.139.1 + specifier: 1.139.2 + version: 1.139.2 posthog-js-lite: specifier: 3.0.0 version: 3.0.0 @@ -17707,8 +17707,8 @@ packages: resolution: {integrity: sha512-dyajjnfzZD1tht4N7p7iwf7nBnR1MjVaVu+MKr+7gBgA39bn28wizCIJZztZPtHy4PY0YwtSGgwfBCuG/hnHgA==} dev: false - /posthog-js@1.139.1: - resolution: {integrity: sha512-+JDu2S7z6sh9Q5kj0oh/W8PZJMQ1gSigWi7gbY4NwwCq2M3t0wNFjxlfHbAo1GncRWDxen+IC+3J7oJ8TJGnkA==} + /posthog-js@1.139.2: + resolution: {integrity: sha512-myyuOADqZvYwgqmriwlKDEUDwLhscivFLh67UWBj4Wt9kOlmklvJb36W0ES2GAS6IdojbnGZGH5lF3heqreLWQ==} dependencies: fflate: 0.4.8 preact: 10.22.0 diff --git a/posthog/api/activity_log.py b/posthog/api/activity_log.py index dd369f94c98fb..5645d038bd6a7 100644 --- a/posthog/api/activity_log.py +++ b/posthog/api/activity_log.py @@ -11,7 +11,7 @@ from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.shared import UserBasicSerializer -from posthog.models import ActivityLog, FeatureFlag, Insight, NotificationViewed, User +from posthog.models import ActivityLog, FeatureFlag, Insight, NotificationViewed, User, Cohort from posthog.models.comment import Comment from posthog.models.notebook.notebook import Notebook @@ -114,6 +114,7 @@ def important_changes(self, request: Request, *args: Any, **kwargs: Any) -> Resp my_comments = list( Comment.objects.filter(created_by=user, team_id=self.team.pk).values_list("id", flat=True) ) + my_cohorts = list(Cohort.objects.filter(created_by=user, team_id=self.team.pk).values_list("id", flat=True)) # then things they edited interesting_changes = [ @@ -168,6 +169,17 @@ def important_changes(self, request: Request, *args: Any, **kwargs: Any) -> Resp .values_list("item_id", flat=True) ) + my_changed_cohorts = list( + ActivityLog.objects.filter( + team_id=self.team.id, + activity__in=interesting_changes, + user_id=user.pk, + scope="Cohort", + ) + .exclude(item_id__in=my_cohorts) + .values_list("item_id", flat=True) + ) + last_read_date = NotificationViewed.objects.filter(user=user).first() last_read_filter = "" @@ -218,6 +230,7 @@ def important_changes(self, request: Request, *args: Any, **kwargs: Any) -> Resp & Q(id__in=deduplicated_notebook_activity_ids) ) | Q(Q(scope="Comment") & Q(item_id__in=my_comments)) + | Q(Q(scope="Cohort") & Q(item_id__in=my_cohorts)) ) | Q( # don't want to see creation of these things since that was before the user edited these things @@ -231,6 +244,7 @@ def important_changes(self, request: Request, *args: Any, **kwargs: Any) -> Resp & Q(id__in=deduplicated_notebook_activity_ids) ) | Q(Q(scope="Comment") & Q(item_id__in=my_changed_comments)) + | Q(Q(scope="Cohort") & Q(item_id__in=my_changed_cohorts)) ) ) ) diff --git a/posthog/api/cohort.py b/posthog/api/cohort.py index 139f8f3747d24..3e776167b8de4 100644 --- a/posthog/api/cohort.py +++ b/posthog/api/cohort.py @@ -2,9 +2,12 @@ from posthog.clickhouse.client.connection import Workload from django.db import DatabaseError +from loginas.utils import is_impersonated_session from sentry_sdk import start_span import structlog +from posthog.models.activity_logging.activity_log import log_activity, Detail, changes_between, load_activity +from posthog.models.activity_logging.activity_page import activity_page_response from posthog.models.feature_flag.flag_matching import ( FeatureFlagMatcher, FlagsMatcherCache, @@ -22,7 +25,7 @@ from django.db.models import QuerySet, Prefetch, prefetch_related_objects, OuterRef, Subquery from django.db.models.expressions import F from django.utils import timezone -from rest_framework import serializers, viewsets +from rest_framework import serializers, viewsets, request, status from rest_framework.decorators import action from rest_framework.exceptions import ValidationError from rest_framework.request import Request @@ -403,6 +406,69 @@ def persons(self, request: Request, **kwargs) -> Response: return Response({"results": serialized_actors, "next": next_url, "previous": previous_url}) + @action(methods=["GET"], url_path="activity", detail=False, required_scopes=["activity_log:read"]) + def all_activity(self, request: request.Request, **kwargs): + limit = int(request.query_params.get("limit", "10")) + page = int(request.query_params.get("page", "1")) + + activity_page = load_activity(scope="Cohort", team_id=self.team_id, limit=limit, page=page) + + return activity_page_response(activity_page, limit, page, request) + + @action(methods=["GET"], detail=True, required_scopes=["activity_log:read"]) + def activity(self, request: request.Request, **kwargs): + limit = int(request.query_params.get("limit", "10")) + page = int(request.query_params.get("page", "1")) + + item_id = kwargs["pk"] + if not Cohort.objects.filter(id=item_id, team_id=self.team_id).exists(): + return Response("", status=status.HTTP_404_NOT_FOUND) + + activity_page = load_activity( + scope="Cohort", + team_id=self.team_id, + item_ids=[str(item_id)], + limit=limit, + page=page, + ) + return activity_page_response(activity_page, limit, page, request) + + def perform_create(self, serializer): + serializer.save() + log_activity( + organization_id=self.organization.id, + team_id=self.team_id, + user=serializer.context["request"].user, + was_impersonated=is_impersonated_session(serializer.context["request"]), + item_id=serializer.instance.id, + scope="Cohort", + activity="created", + detail=Detail(name=serializer.instance.name), + ) + + def perform_update(self, serializer): + instance_id = serializer.instance.id + + try: + before_update = Cohort.objects.get(pk=instance_id) + except Cohort.DoesNotExist: + before_update = None + + serializer.save() + + changes = changes_between("Cohort", previous=before_update, current=serializer.instance) + + log_activity( + organization_id=self.organization.id, + team_id=self.team_id, + user=serializer.context["request"].user, + was_impersonated=is_impersonated_session(serializer.context["request"]), + item_id=instance_id, + scope="Cohort", + activity="updated", + detail=Detail(changes=changes, name=serializer.instance.name), + ) + class LegacyCohortViewSet(CohortViewSet): derive_current_team_from_user_only = True diff --git a/posthog/api/hog_function.py b/posthog/api/hog_function.py index 17ee1bb2b0995..9ddc46cc3b45f 100644 --- a/posthog/api/hog_function.py +++ b/posthog/api/hog_function.py @@ -1,12 +1,13 @@ import structlog from django_filters.rest_framework import DjangoFilterBackend +from django.db.models import QuerySet + from rest_framework import serializers, viewsets from rest_framework.serializers import BaseSerializer from rest_framework.decorators import action from rest_framework.request import Request from rest_framework.response import Response - from posthog.api.forbid_destroy_model import ForbidDestroyModel from posthog.api.hog_function_template import HogFunctionTemplateSerializer from posthog.api.log_entries import LogEntryMixin @@ -55,6 +56,7 @@ class Meta: "created_by", "updated_at", "enabled", + "deleted", "hog", "bytecode", "inputs_schema", @@ -74,6 +76,7 @@ class Meta: ] extra_kwargs = { "template_id": {"write_only": True}, + "deleted": {"write_only": True}, } def validate_inputs_schema(self, value): @@ -82,10 +85,17 @@ def validate_inputs_schema(self, value): def validate(self, attrs): team = self.context["get_team"]() attrs["team"] = team - attrs["inputs_schema"] = attrs.get("inputs_schema", []) - attrs["filters"] = attrs.get("filters", {}) - attrs["inputs"] = validate_inputs(attrs["inputs_schema"], attrs.get("inputs", {})) - attrs["bytecode"] = compile_hog(attrs["hog"]) + + if self.context["view"].action == "create": + # Ensure we have sensible defaults when created + attrs["filters"] = attrs.get("filters", {}) + attrs["inputs_schema"] = attrs.get("inputs_schema", []) + attrs["inputs"] = attrs.get("inputs", {}) + + if "inputs" in attrs: + attrs["inputs"] = validate_inputs(attrs["inputs_schema"], attrs["inputs"]) + if "hog" in attrs: + attrs["bytecode"] = compile_hog(attrs["hog"]) return attrs @@ -108,6 +118,12 @@ class HogFunctionViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, ForbidDestroyMod def get_serializer_class(self) -> type[BaseSerializer]: return HogFunctionMinimalSerializer if self.action == "list" else HogFunctionSerializer + def safely_get_queryset(self, queryset: QuerySet) -> QuerySet: + if self.action == "list": + queryset = queryset.filter(deleted=False) + + return queryset + @action(detail=False, methods=["GET"]) def icons(self, request: Request, *args, **kwargs): query = request.GET.get("query") diff --git a/posthog/api/ingestion_warnings.py b/posthog/api/ingestion_warnings.py index c246d990d61e2..a820c8c02fc3c 100644 --- a/posthog/api/ingestion_warnings.py +++ b/posthog/api/ingestion_warnings.py @@ -17,11 +17,25 @@ def list(self, request: Request, **kw) -> Response: start_date = now() - timedelta(days=30) warning_events = sync_execute( """ - SELECT type, timestamp, details - FROM ingestion_warnings - WHERE team_id = %(team_id)s - AND timestamp > %(start_date)s - ORDER BY timestamp DESC + SELECT + type, + count(details) as total_count, + arraySlice(groupArray((details, timestamp)), 1, 50) as top_50_recent_examples + FROM + (SELECT + type, + details, + timestamp + FROM + ingestion_warnings + WHERE + team_id = %(team_id)s + AND timestamp > %(start_date)s + ORDER BY + type, + timestamp DESC) + GROUP BY + type """, { "team_id": self.team_id, @@ -34,17 +48,19 @@ def list(self, request: Request, **kw) -> Response: def _calculate_summaries(warning_events): summaries = {} - for warning_type, timestamp, details in warning_events: - details = json.loads(details) - if warning_type not in summaries: - summaries[warning_type] = { - "type": warning_type, - "lastSeen": timestamp, - "warnings": [], - "count": 0, - } - - summaries[warning_type]["warnings"].append({"type": warning_type, "timestamp": timestamp, "details": details}) - summaries[warning_type]["count"] += 1 + for warning_type, count, examples in warning_events: + summaries[warning_type] = { + "type": warning_type, + "lastSeen": examples[0][1] if examples else None, + "warnings": [ + { + "type": warning_type, + "timestamp": timestamp, + "details": json.loads(details), + } + for details, timestamp in examples + ], + "count": count, + } return sorted(summaries.values(), key=lambda summary: summary["lastSeen"], reverse=True) diff --git a/posthog/api/plugin.py b/posthog/api/plugin.py index b568876eeecd4..65edc3419c874 100644 --- a/posthog/api/plugin.py +++ b/posthog/api/plugin.py @@ -909,6 +909,7 @@ def safely_get_queryset(self, queryset): class PipelineTransformationsConfigsViewSet(PluginConfigViewSet): def safely_get_queryset(self, queryset): + queryset = super().safely_get_queryset(queryset) return queryset.filter( Q(plugin__capabilities__has_key="methods") & Q(plugin__capabilities__methods__contains=["processEvent"]) ) @@ -925,6 +926,7 @@ def safely_get_queryset(self, queryset): class PipelineDestinationsConfigsViewSet(PluginConfigViewSet): def safely_get_queryset(self, queryset): + queryset = super().safely_get_queryset(queryset) return queryset.filter( Q(plugin__capabilities__has_key="methods") & ( @@ -942,6 +944,7 @@ def safely_get_queryset(self, queryset): class PipelineFrontendAppsConfigsViewSet(PluginConfigViewSet): def safely_get_queryset(self, queryset): + queryset = super().safely_get_queryset(queryset) return queryset.exclude( Q(plugin__capabilities__has_key="methods") | Q(plugin__capabilities__has_key="scheduled_tasks") ) @@ -964,6 +967,7 @@ def safely_get_queryset(self, queryset): class PipelineImportAppsConfigsViewSet(PluginConfigViewSet): def safely_get_queryset(self, queryset): + queryset = super().safely_get_queryset(queryset) return queryset.filter( Q(Q(plugin__capabilities__has_key="scheduled_tasks") & ~Q(plugin__capabilities__scheduled_tasks=[])) | Q( diff --git a/posthog/api/test/__snapshots__/test_api_docs.ambr b/posthog/api/test/__snapshots__/test_api_docs.ambr index 4327fbc67be9b..2ded9229008c7 100644 --- a/posthog/api/test/__snapshots__/test_api_docs.ambr +++ b/posthog/api/test/__snapshots__/test_api_docs.ambr @@ -110,6 +110,7 @@ 'Warning: operationId "batch_exports_pause_create" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/pause/\', \'post\'), (\'/api/projects/{project_id}/batch_exports/{id}/pause/\', \'post\')]. resolving with numeral suffixes.', 'Warning: operationId "batch_exports_unpause_create" has collisions [(\'/api/organizations/{organization_id}/batch_exports/{id}/unpause/\', \'post\'), (\'/api/projects/{project_id}/batch_exports/{id}/unpause/\', \'post\')]. resolving with numeral suffixes.', 'Warning: operationId "app_metrics_historical_exports_retrieve" has collisions [(\'/api/projects/{project_id}/app_metrics/{plugin_config_id}/historical_exports/\', \'get\'), (\'/api/projects/{project_id}/app_metrics/{plugin_config_id}/historical_exports/{id}/\', \'get\')]. resolving with numeral suffixes.', + 'Warning: operationId "cohorts_activity_retrieve" has collisions [(\'/api/projects/{project_id}/cohorts/{id}/activity/\', \'get\'), (\'/api/projects/{project_id}/cohorts/activity/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "event_definitions_retrieve" has collisions [(\'/api/projects/{project_id}/event_definitions/\', \'get\'), (\'/api/projects/{project_id}/event_definitions/{id}/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "feature_flags_activity_retrieve" has collisions [(\'/api/projects/{project_id}/feature_flags/{id}/activity/\', \'get\'), (\'/api/projects/{project_id}/feature_flags/activity/\', \'get\')]. resolving with numeral suffixes.', 'Warning: operationId "insights_activity_retrieve" has collisions [(\'/api/projects/{project_id}/insights/{id}/activity/\', \'get\'), (\'/api/projects/{project_id}/insights/activity/\', \'get\')]. resolving with numeral suffixes.', diff --git a/posthog/api/test/__snapshots__/test_insight.ambr b/posthog/api/test/__snapshots__/test_insight.ambr index 1376e752451f9..972bf8c24e289 100644 --- a/posthog/api/test/__snapshots__/test_insight.ambr +++ b/posthog/api/test/__snapshots__/test_insight.ambr @@ -1628,6 +1628,24 @@ LIMIT 21 ''' # --- +# name: TestInsight.test_listing_insights_does_not_nplus1.30 + ''' + SELECT "posthog_taggeditem"."id", + "posthog_taggeditem"."tag_id", + "posthog_taggeditem"."dashboard_id", + "posthog_taggeditem"."insight_id", + "posthog_taggeditem"."event_definition_id", + "posthog_taggeditem"."property_definition_id", + "posthog_taggeditem"."action_id", + "posthog_taggeditem"."feature_flag_id" + FROM "posthog_taggeditem" + WHERE "posthog_taggeditem"."insight_id" IN (1, + 2, + 3, + 4, + 5 /* ... */) + ''' +# --- # name: TestInsight.test_listing_insights_does_not_nplus1.4 ''' SELECT "posthog_team"."id", diff --git a/posthog/api/test/test_cohort.py b/posthog/api/test/test_cohort.py index 82740d73ed515..14b6b60b51484 100644 --- a/posthog/api/test/test_cohort.py +++ b/posthog/api/test/test_cohort.py @@ -1,6 +1,7 @@ import json from datetime import datetime, timedelta -from typing import Any +from typing import Optional, Any +from unittest import mock from unittest.mock import patch from django.core.files.uploadedfile import SimpleUploadedFile @@ -34,6 +35,31 @@ class TestCohort(TestExportMixin, ClickhouseTestMixin, APIBaseTest, QueryMatchin def capture_select_queries(self): return self.capture_queries(("INSERT INTO cohortpeople", "SELECT", "ALTER", "select", "DELETE")) + def _get_cohort_activity( + self, + flag_id: Optional[int] = None, + team_id: Optional[int] = None, + expected_status: int = status.HTTP_200_OK, + ): + if team_id is None: + team_id = self.team.id + + if flag_id: + url = f"/api/projects/{team_id}/cohorts/{flag_id}/activity" + else: + url = f"/api/projects/{team_id}/cohorts/activity" + + activity = self.client.get(url) + self.assertEqual(activity.status_code, expected_status) + return activity.json() + + def assert_cohort_activity(self, cohort_id: Optional[int], expected: list[dict]): + activity_response = self._get_cohort_activity(cohort_id) + + activity: list[dict] = activity_response["results"] + self.maxDiff = None + assert activity == expected + @patch("posthog.api.cohort.report_user_action") @patch("posthog.tasks.calculate_cohort.calculate_cohort_ch.delay", side_effect=calculate_cohort_ch) @patch("posthog.models.cohort.util.sync_execute", side_effect=sync_execute) @@ -281,6 +307,96 @@ def test_cohort_list(self): self.assertEqual(response["results"][0]["name"], "whatever") self.assertEqual(response["results"][0]["created_by"]["id"], self.user.id) + def test_cohort_activity_log(self): + self.team.app_urls = ["http://somewebsite.com"] + self.team.save() + Person.objects.create(team=self.team, properties={"prop": 5}) + Person.objects.create(team=self.team, properties={"prop": 6}) + + self.client.post( + f"/api/projects/{self.team.id}/cohorts", + data={"name": "whatever", "groups": [{"properties": {"prop": 5}}]}, + ) + + cohort = Cohort.objects.filter(team=self.team).last() + assert cohort is not None + + self.assert_cohort_activity( + cohort_id=cohort.pk, + expected=[ + { + "user": {"first_name": "", "email": "user1@posthog.com"}, + "activity": "created", + "scope": "Cohort", + "item_id": str(cohort.pk), + "detail": {"changes": None, "trigger": None, "name": "whatever", "short_id": None, "type": None}, + "created_at": mock.ANY, + } + ], + ) + + self.client.patch( + f"/api/projects/{self.team.id}/cohorts/{cohort.pk}", + data={"name": "woohoo", "groups": [{"properties": {"prop": 6}}]}, + ) + cohort.refresh_from_db() + assert cohort.name == "woohoo" + + self.assert_cohort_activity( + cohort_id=cohort.pk, + expected=[ + { + "user": {"first_name": "", "email": "user1@posthog.com"}, + "activity": "updated", + "scope": "Cohort", + "item_id": str(cohort.pk), + "detail": { + "changes": [ + { + "type": "Cohort", + "action": "changed", + "field": "name", + "before": "whatever", + "after": "woohoo", + }, + { + "type": "Cohort", + "action": "changed", + "field": "groups", + "before": [ + { + "days": None, + "count": None, + "label": None, + "end_date": None, + "event_id": None, + "action_id": None, + "properties": [{"key": "prop", "type": "person", "value": 5}], + "start_date": None, + "count_operator": None, + } + ], + "after": [{"properties": [{"key": "prop", "type": "person", "value": 6}]}], + }, + ], + "trigger": None, + "name": "woohoo", + "short_id": None, + "type": None, + }, + "created_at": mock.ANY, + }, + { + "user": {"first_name": "", "email": "user1@posthog.com"}, + "activity": "created", + "scope": "Cohort", + "item_id": str(cohort.pk), + "detail": {"changes": None, "trigger": None, "name": "whatever", "short_id": None, "type": None}, + "created_at": mock.ANY, + }, + ], + ) + def test_csv_export_new(self): # Test 100s of distinct_ids, we only want ~10 Person.objects.create( diff --git a/posthog/api/test/test_hog_function.py b/posthog/api/test/test_hog_function.py index 94c3b5b1c4d14..0e6eda95bbaf5 100644 --- a/posthog/api/test/test_hog_function.py +++ b/posthog/api/test/test_hog_function.py @@ -76,11 +76,7 @@ def test_create_hog_function_forbidden_if_not_in_flag(self, mock_feature_enabled def test_create_hog_function(self, *args): response = self.client.post( f"/api/projects/{self.team.id}/hog_functions/", - data={ - "name": "Fetch URL", - "description": "Test description", - "hog": "fetch(inputs.url);", - }, + data={"name": "Fetch URL", "description": "Test description", "hog": "fetch(inputs.url);", "inputs": {}}, ) assert response.status_code == status.HTTP_201_CREATED, response.json() assert response.json()["created_by"]["id"] == self.user.id @@ -124,6 +120,31 @@ def test_creates_with_template_id(self, *args): "filters": None, } + @patch("posthog.permissions.posthoganalytics.feature_enabled", return_value=True) + def test_deletes_via_update(self, *args): + response = self.client.post( + f"/api/projects/{self.team.id}/hog_functions/", + data={"name": "Fetch URL", "description": "Test description", "hog": "fetch(inputs.url);"}, + ) + assert response.status_code == status.HTTP_201_CREATED, response.json() + + list_res = self.client.get(f"/api/projects/{self.team.id}/hog_functions/") + assert list_res.status_code == status.HTTP_200_OK, list_res.json() + # Assert that it isn't in the list + assert ( + next((item for item in list_res.json()["results"] if item["id"] == response.json()["id"]), None) is not None + ) + + response = self.client.patch( + f"/api/projects/{self.team.id}/hog_functions/{response.json()['id']}/", + data={"deleted": True}, + ) + assert response.status_code == status.HTTP_200_OK, response.json() + + list_res = self.client.get(f"/api/projects/{self.team.id}/hog_functions/") + assert list_res.status_code == status.HTTP_200_OK, list_res.json() + assert next((item for item in list_res.json()["results"] if item["id"] == response.json()["id"]), None) is None + @patch("posthog.permissions.posthoganalytics.feature_enabled", return_value=True) def test_inputs_required(self, *args): payload = { diff --git a/posthog/api/test/test_ingestion_warnings.py b/posthog/api/test/test_ingestion_warnings.py index 05e893babfa3e..21a996c0236dd 100644 --- a/posthog/api/test/test_ingestion_warnings.py +++ b/posthog/api/test/test_ingestion_warnings.py @@ -1,4 +1,5 @@ import json +from math import floor from freezegun.api import freeze_time from rest_framework import status @@ -14,6 +15,7 @@ def create_ingestion_warning(team_id: int, type: str, details: dict, timestamp: str, source=""): timestamp = cast_timestamp_or_now(timestamp) + data = { "team_id": team_id, "type": type, @@ -29,6 +31,22 @@ def create_ingestion_warning(team_id: int, type: str, details: dict, timestamp: class TestIngestionWarningsAPI(ClickhouseTestMixin, APIBaseTest): @freeze_time("2021-12-04T19:20:00Z") def test_ingestion_warnings_api(self): + a_lot_of_ingestion_warning_timestamps: list[str] = [] + # more than the number the front end will show + # KLUDGE: it is 59 here so that timestamp creation can be naive + for i in range(59): + seconds = f"0{i}" if i < 10 else str(i) + minutes = floor(i / 10) + formatted_minutes = f"0{minutes}" if minutes < 10 else str(minutes) + timestamp = f"2021-12-04T13:{formatted_minutes}:{seconds}Z" + a_lot_of_ingestion_warning_timestamps.insert(0, timestamp) + create_ingestion_warning( + team_id=self.team.id, + type="replay_timestamp_too_far", + details={}, + timestamp=timestamp, + ) + create_ingestion_warning( team_id=self.team.id, type="cannot_merge_already_identified", @@ -69,10 +87,24 @@ def test_ingestion_warnings_api(self): response = self.client.get(f"/api/projects/{self.team.pk}/ingestion_warnings") self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual( - response.json(), - { + assert ( + response.json() + == { "results": [ + { + "count": 59, # count is correct and not limited like the examples are + "lastSeen": "2021-12-04T13:05:58Z", + "type": "replay_timestamp_too_far", + "warnings": [ + { + "details": {}, + "timestamp": t, + "type": "replay_timestamp_too_far", + # even though there are very many warnings we limit the number of examples we send to the frontend + } + for t in a_lot_of_ingestion_warning_timestamps[:50] + ], + }, { "type": "cannot_merge_already_identified", "lastSeen": "2021-12-03T00:00:00Z", @@ -108,5 +140,5 @@ def test_ingestion_warnings_api(self): "count": 1, }, ] - }, + } ) diff --git a/posthog/hogql/bytecode.py b/posthog/hogql/bytecode.py index 7a9ae7165f6f4..1a5933a88bc92 100644 --- a/posthog/hogql/bytecode.py +++ b/posthog/hogql/bytecode.py @@ -248,7 +248,7 @@ def visit_if_statement(self, node: ast.IfStatement): response = [] response.extend(expr) - response.extend([Operation.JUMP_IF_FALSE, len(then) + 2]) # + else's OP_JUMP + count + response.extend([Operation.JUMP_IF_FALSE, len(then) + (2 if else_ else 0)]) response.extend(then) if else_: response.extend([Operation.JUMP, len(else_)]) diff --git a/posthog/hogql/functions/mapping.py b/posthog/hogql/functions/mapping.py index bda37830c4b37..d079b7878534c 100644 --- a/posthog/hogql/functions/mapping.py +++ b/posthog/hogql/functions/mapping.py @@ -466,6 +466,7 @@ def compare_types(arg_types: list[ConstantType], sig_arg_types: tuple[ConstantTy "timeStampSub": HogQLFunctionMeta("timeStampSub", 2, 2), "now": HogQLFunctionMeta("now64", 0, 1, tz_aware=True, case_sensitive=False), "nowInBlock": HogQLFunctionMeta("nowInBlock", 1, 1), + "rowNumberInAllBlocks": HogQLFunctionMeta("rowNumberInAllBlocks", 0, 0), "today": HogQLFunctionMeta("today"), "yesterday": HogQLFunctionMeta("yesterday"), "timeSlot": HogQLFunctionMeta("timeSlot", 1, 1), diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index b44191b41095e..136e8ecab70fd 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -1145,7 +1145,10 @@ def visit_window_function(self, node: ast.WindowFunction): identifier = self._print_identifier(node.name) exprs = ", ".join(self.visit(expr) for expr in node.exprs or []) args = "(" + (", ".join(self.visit(arg) for arg in node.args or [])) + ")" if node.args else "" - over = f"({self.visit(node.over_expr)})" if node.over_expr else self._print_identifier(node.over_identifier) + if node.over_expr or node.over_identifier: + over = f"({self.visit(node.over_expr)})" if node.over_expr else self._print_identifier(node.over_identifier) + else: + over = "()" return f"{identifier}({exprs}){args} OVER {over}" def visit_window_frame_expr(self, node: ast.WindowFrameExpr): diff --git a/posthog/hogql_queries/insights/trends/breakdown.py b/posthog/hogql_queries/insights/trends/breakdown.py index 49491429cf54f..9c8af0064ba42 100644 --- a/posthog/hogql_queries/insights/trends/breakdown.py +++ b/posthog/hogql_queries/insights/trends/breakdown.py @@ -3,12 +3,6 @@ from posthog.hogql.constants import LimitContext from posthog.hogql.parser import parse_expr from posthog.hogql.timings import HogQLTimings -from posthog.hogql_queries.insights.trends.breakdown_values import ( - BREAKDOWN_NULL_STRING_LABEL, - BREAKDOWN_OTHER_STRING_LABEL, - BreakdownValues, -) -from posthog.hogql_queries.insights.trends.display import TrendsDisplay from posthog.hogql_queries.insights.trends.utils import ( get_properties_chain, ) @@ -17,6 +11,11 @@ from posthog.models.team.team import Team from posthog.schema import ActionsNode, EventsNode, DataWarehouseNode, HogQLQueryModifiers, InCohortVia, TrendsQuery +BREAKDOWN_OTHER_STRING_LABEL = "$$_posthog_breakdown_other_$$" +BREAKDOWN_NULL_STRING_LABEL = "$$_posthog_breakdown_null_$$" +BREAKDOWN_OTHER_DISPLAY = "Other (i.e. all remaining values)" +BREAKDOWN_NULL_DISPLAY = "None (i.e. no value)" + def hogql_to_string(expr: ast.Expr) -> ast.Call: return ast.Call(name="toString", args=[expr]) @@ -30,7 +29,6 @@ class Breakdown: timings: HogQLTimings modifiers: HogQLQueryModifiers events_filter: ast.Expr - breakdown_values_override: Optional[list[str | int]] limit_context: LimitContext def __init__( @@ -42,7 +40,6 @@ def __init__( timings: HogQLTimings, modifiers: HogQLQueryModifiers, events_filter: ast.Expr, - breakdown_values_override: Optional[list[str | int]] = None, limit_context: LimitContext = LimitContext.QUERY, ): self.team = team @@ -52,34 +49,19 @@ def __init__( self.timings = timings self.modifiers = modifiers self.events_filter = events_filter - self.breakdown_values_override = breakdown_values_override self.limit_context = limit_context @cached_property def enabled(self) -> bool: - return ( - self.query.breakdownFilter is not None - and self.query.breakdownFilter.breakdown is not None - and self.has_breakdown_values - ) - - @cached_property - def is_session_type(self) -> bool: - return self.enabled and self.query.breakdownFilter.breakdown_type == "session" + return self.query.breakdownFilter is not None and self.query.breakdownFilter.breakdown is not None @cached_property def is_histogram_breakdown(self) -> bool: return self.enabled and self.query.breakdownFilter.breakdown_histogram_bin_count is not None - def placeholders(self) -> dict[str, ast.Expr]: - values = self._breakdown_buckets_ast if self.is_histogram_breakdown else self._breakdown_values_ast - - return {"cross_join_breakdown_values": ast.Alias(alias="breakdown_value", expr=values)} - def column_expr(self) -> ast.Alias: if self.is_histogram_breakdown: - return ast.Alias(alias="breakdown_value", expr=self._get_breakdown_histogram_multi_if()) - + return ast.Alias(alias="breakdown_value", expr=ast.Field(chain=self._properties_chain)) if self.query.breakdownFilter.breakdown_type == "cohort": if self.modifiers.inCohortVia == InCohortVia.LEFTJOIN_CONJOINED: return ast.Alias( @@ -95,19 +77,15 @@ def column_expr(self) -> ast.Alias: expr=hogql_to_string(ast.Constant(value=cohort_breakdown)), ) - return ast.Alias(alias="breakdown_value", expr=self._get_breakdown_transform_func) + return ast.Alias(alias="breakdown_value", expr=self._get_breakdown_expression) - def events_where_filter(self) -> ast.Expr | None: + def events_where_filter(self, breakdown_values_override: Optional[str | int] = None) -> ast.Expr | None: if ( self.query.breakdownFilter is not None and self.query.breakdownFilter.breakdown is not None and self.query.breakdownFilter.breakdown_type == "cohort" ): - breakdown = ( - self.breakdown_values_override - if self.breakdown_values_override - else self.query.breakdownFilter.breakdown - ) + breakdown = breakdown_values_override if breakdown_values_override else self.query.breakdownFilter.breakdown if breakdown == "all": return None @@ -136,60 +114,33 @@ def events_where_filter(self) -> ast.Expr | None: right=ast.Constant(value=breakdown), ) - # No need to filter if we're showing the "other" bucket, as we need to look at all events anyway. - # Except when explicitly filtering - if ( - self.query.breakdownFilter is not None - and not self.query.breakdownFilter.breakdown_hide_other_aggregation - and len(self.breakdown_values_override or []) == 0 - ): - return ast.Constant(value=True) - - if ( - self.query.breakdownFilter is not None - and self.query.breakdownFilter.breakdown is not None - and self.query.breakdownFilter.breakdown_type == "hogql" - and isinstance(self.query.breakdownFilter.breakdown, str) - ): - left = parse_expr(self.query.breakdownFilter.breakdown) - else: - left = ast.Field(chain=self._properties_chain) - - if not self.is_histogram_breakdown: - left = hogql_to_string(left) - - compare_ops = [] - for _value in self._breakdown_values: - value: Optional[str] = str(_value) # non-cohorts are always strings - # If the value is one of the "other" values, then use the `transform()` func + if breakdown_values_override: + if ( + self.query.breakdownFilter is not None + and self.query.breakdownFilter.breakdown is not None + and self.query.breakdownFilter.breakdown_type == "hogql" + and isinstance(self.query.breakdownFilter.breakdown, str) + ): + left = parse_expr(self.query.breakdownFilter.breakdown) + else: + left = ast.Field(chain=self._properties_chain) + value: Optional[str] = str(breakdown_values_override) # non-cohorts are always strings if value == BREAKDOWN_OTHER_STRING_LABEL: - transform_func = self._get_breakdown_transform_func - compare_ops.append( - ast.CompareOperation( - left=transform_func, op=ast.CompareOperationOp.Eq, right=ast.Constant(value=value) - ) - ) + # TODO: Fix breaking down by other + return ast.Constant(value=True) elif value == BREAKDOWN_NULL_STRING_LABEL: - compare_ops.append( - ast.CompareOperation(left=left, op=ast.CompareOperationOp.Eq, right=ast.Constant(value=None)) - ) - compare_ops.append( - ast.CompareOperation(left=left, op=ast.CompareOperationOp.Eq, right=ast.Constant(value="")) + return ast.Or( + exprs=[ + ast.CompareOperation(left=left, op=ast.CompareOperationOp.Eq, right=ast.Constant(value=None)), + ast.CompareOperation(left=left, op=ast.CompareOperationOp.Eq, right=ast.Constant(value="")), + ] ) else: - compare_ops.append( - ast.CompareOperation(left=left, op=ast.CompareOperationOp.Eq, right=ast.Constant(value=value)) - ) - - if len(compare_ops) == 1: - return compare_ops[0] - elif len(compare_ops) == 0: - return parse_expr("1 = 1") - - return ast.Or(exprs=compare_ops) + return ast.CompareOperation(left=left, op=ast.CompareOperationOp.Eq, right=ast.Constant(value=value)) + return ast.Constant(value=True) @cached_property - def _get_breakdown_transform_func(self) -> ast.Call: + def _get_breakdown_expression(self) -> ast.Call: if self.query.breakdownFilter.breakdown_type == "hogql": return self._get_breakdown_values_transform(parse_expr(self.query.breakdownFilter.breakdown)) return self._get_breakdown_values_transform(ast.Field(chain=self._properties_chain)) @@ -202,115 +153,14 @@ def _get_breakdown_values_transform(self, node: ast.Expr) -> ast.Call: return cast( ast.Call, parse_expr( - "transform(ifNull(nullIf(toString({node}), ''), {nil}), {values}, {values}, {other})", + "ifNull(nullIf(toString({node}), ''), {nil})", placeholders={ "node": node, - "values": self._breakdown_values_ast, "nil": ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL), - "other": ast.Constant(value=BREAKDOWN_OTHER_STRING_LABEL), }, ), ) - @cached_property - def _breakdown_buckets_ast(self) -> ast.Array: - buckets = self._get_breakdown_histogram_buckets() - values = [f"[{t[0]},{t[1]}]" for t in buckets] - # TODO: add this only if needed - values.append('["",""]') - - return ast.Array(exprs=[ast.Constant(value=v) for v in values]) - - @property - def _breakdown_values_ast(self) -> ast.Array: - exprs: list[ast.Expr] = [] - for value in self._breakdown_values: - if isinstance(value, str): - exprs.append(ast.Constant(value=value)) - else: - exprs.append(hogql_to_string(ast.Constant(value=value))) - return ast.Array(exprs=exprs) - - @cached_property - def _all_breakdown_values(self) -> list[str | int | None]: - # Used in the actors query - if self.breakdown_values_override is not None: - return cast(list[str | int | None], self.breakdown_values_override) - - if self.query.breakdownFilter is None: - return [] - - with self.timings.measure("breakdown_values_query"): - breakdown = BreakdownValues( - team=self.team, - series=self.series, - events_filter=self.events_filter, - chart_display_type=self._trends_display().display_type, - breakdown_filter=self.query.breakdownFilter, - query_date_range=self.query_date_range, - modifiers=self.modifiers, - limit_context=self.limit_context, - ) - return cast(list[str | int | None], breakdown.get_breakdown_values()) - - @cached_property - def _breakdown_values(self) -> list[str | int]: - values = [BREAKDOWN_NULL_STRING_LABEL if v is None else v for v in self._all_breakdown_values] - return cast(list[str | int], values) - - @cached_property - def has_breakdown_values(self) -> bool: - return len(self._breakdown_values) > 0 - - def _get_breakdown_histogram_buckets(self) -> list[tuple[float, float]]: - buckets = [] - values = self._breakdown_values - - if len(values) == 1: - values = [values[0], values[0]] - - for i in range(len(values) - 1): - last_value = i == len(values) - 2 - - # Since we always `floor(x, 2)` the value, we add 0.01 to the last bucket - # to ensure it's always slightly greater than the maximum value - lower_bound = float(values[i]) - upper_bound = float(values[i + 1]) + 0.01 if last_value else float(values[i + 1]) - buckets.append((lower_bound, upper_bound)) - - return buckets - - def _get_breakdown_histogram_multi_if(self) -> ast.Expr: - multi_if_exprs: list[ast.Expr] = [] - - buckets = self._get_breakdown_histogram_buckets() - - for lower_bound, upper_bound in buckets: - multi_if_exprs.extend( - [ - ast.And( - exprs=[ - ast.CompareOperation( - left=ast.Field(chain=self._properties_chain), - op=ast.CompareOperationOp.GtEq, - right=ast.Constant(value=lower_bound), - ), - ast.CompareOperation( - left=ast.Field(chain=self._properties_chain), - op=ast.CompareOperationOp.Lt, - right=ast.Constant(value=upper_bound), - ), - ] - ), - ast.Constant(value=f"[{lower_bound},{upper_bound}]"), - ] - ) - - # `else` block of the multi-if - multi_if_exprs.append(ast.Constant(value='["",""]')) - - return ast.Call(name="multiIf", args=multi_if_exprs) - @cached_property def _properties_chain(self): return get_properties_chain( @@ -318,11 +168,3 @@ def _properties_chain(self): breakdown_field=self.query.breakdownFilter.breakdown, group_type_index=self.query.breakdownFilter.breakdown_group_type_index, ) - - def _trends_display(self) -> TrendsDisplay: - display = ( - self.query.trendsFilter.display - if self.query.trendsFilter is not None and self.query.trendsFilter.display is not None - else None - ) - return TrendsDisplay(display) diff --git a/posthog/hogql_queries/insights/trends/breakdown_values.py b/posthog/hogql_queries/insights/trends/breakdown_values.py deleted file mode 100644 index aee02dd9ccefb..0000000000000 --- a/posthog/hogql_queries/insights/trends/breakdown_values.py +++ /dev/null @@ -1,279 +0,0 @@ -from typing import Optional, Union, Any -from posthog.hogql import ast -from posthog.hogql.constants import LimitContext, get_breakdown_limit_for_context, BREAKDOWN_VALUES_LIMIT_FOR_COUNTRIES -from posthog.hogql.parser import parse_expr, parse_select -from posthog.hogql.placeholders import replace_placeholders, find_placeholders -from posthog.hogql.query import execute_hogql_query -from posthog.hogql_queries.insights.trends.aggregation_operations import AggregationOperations -from posthog.hogql_queries.insights.trends.utils import get_properties_chain -from posthog.hogql_queries.utils.query_date_range import QueryDateRange -from posthog.models.team.team import Team -from posthog.schema import ( - BreakdownFilter, - BreakdownType, - ChartDisplayType, - ActionsNode, - EventsNode, - DataWarehouseNode, - HogQLQueryModifiers, -) -from functools import cached_property - -BREAKDOWN_OTHER_STRING_LABEL = "$$_posthog_breakdown_other_$$" -BREAKDOWN_OTHER_NUMERIC_LABEL = 9007199254740991 # pow(2, 53) - 1, for JS compatibility -BREAKDOWN_OTHER_DISPLAY = "Other (i.e. all remaining values)" -BREAKDOWN_NULL_STRING_LABEL = "$$_posthog_breakdown_null_$$" -BREAKDOWN_NULL_NUMERIC_LABEL = 9007199254740990 # pow(2, 53) - 2, for JS compatibility -BREAKDOWN_NULL_DISPLAY = "None (i.e. no value)" - - -class BreakdownValues: - team: Team - series: Union[EventsNode, ActionsNode, DataWarehouseNode] - breakdown_field: Union[str, float, list[Union[str, float]]] - breakdown_type: BreakdownType - events_filter: ast.Expr - chart_display_type: ChartDisplayType - histogram_bin_count: Optional[int] - group_type_index: Optional[int] - hide_other_aggregation: Optional[bool] - normalize_url: Optional[bool] - breakdown_limit: int - query_date_range: QueryDateRange - modifiers: HogQLQueryModifiers - limit_context: LimitContext - - def __init__( - self, - team: Team, - series: Union[EventsNode, ActionsNode, DataWarehouseNode], - events_filter: ast.Expr, - chart_display_type: ChartDisplayType, - breakdown_filter: BreakdownFilter, - query_date_range: QueryDateRange, - modifiers: HogQLQueryModifiers, - limit_context: LimitContext = LimitContext.QUERY, - ): - self.team = team - self.series = series - self.breakdown_field = breakdown_filter.breakdown # type: ignore - self.breakdown_type = breakdown_filter.breakdown_type # type: ignore - self.events_filter = events_filter - self.chart_display_type = chart_display_type - self.histogram_bin_count = ( - int(breakdown_filter.breakdown_histogram_bin_count) - if breakdown_filter.breakdown_histogram_bin_count is not None - else None - ) - self.group_type_index = ( - int(breakdown_filter.breakdown_group_type_index) - if breakdown_filter.breakdown_group_type_index is not None - else None - ) - self.hide_other_aggregation = breakdown_filter.breakdown_hide_other_aggregation - self.normalize_url = breakdown_filter.breakdown_normalize_url - self.breakdown_limit = breakdown_filter.breakdown_limit or get_breakdown_limit_for_context(limit_context) - self.query_date_range = query_date_range - self.modifiers = modifiers - self.limit_context = limit_context - - def get_breakdown_values(self) -> list[str | int]: - if self.breakdown_type == "cohort": - if self.breakdown_field == "all": - return [0] - - if isinstance(self.breakdown_field, list): - return [value if isinstance(value, str) else int(value) for value in self.breakdown_field] - - return [self.breakdown_field if isinstance(self.breakdown_field, str) else int(self.breakdown_field)] - - if self.breakdown_type == "hogql": - select_field = ast.Alias( - alias="value", - expr=parse_expr(str(self.breakdown_field)), - ) - else: - select_field = ast.Alias( - alias="value", - expr=ast.Field( - chain=get_properties_chain( - breakdown_type=self.breakdown_type, - breakdown_field=str(self.breakdown_field), - group_type_index=self.group_type_index, - ) - ), - ) - - if not self.histogram_bin_count: - if self.normalize_url: - select_field.expr = parse_expr( - "empty(trimRight({node}, '/?#')) ? '/' : trimRight({node}, '/?#')", - placeholders={"node": select_field.expr}, - ) - - select_field.expr = ast.Call(name="toString", args=[select_field.expr]) - - if self.chart_display_type == ChartDisplayType.WORLD_MAP: - breakdown_limit = BREAKDOWN_VALUES_LIMIT_FOR_COUNTRIES - else: - breakdown_limit = int(self.breakdown_limit) - - aggregation_expression: ast.Expr - if self._aggregation_operation.aggregating_on_session_duration(): - aggregation_expression = ast.Call(name="max", args=[ast.Field(chain=["session", "$session_duration"])]) - elif self.series.math == "dau": - # When aggregating by (daily) unique users, run the breakdown aggregation on count(e.uuid). - # This retains legacy compatibility and should be removed once we have the new trends in production. - aggregation_expression = parse_expr("count({id_field})", placeholders={"id_field": self._id_field}) - else: - aggregation_expression = self._aggregation_operation.select_aggregation() - # Take a shortcut with WAU and MAU queries. Get the total AU-s for the period instead. - if "replaced" in find_placeholders(aggregation_expression): - actor = "e.distinct_id" if self.team.aggregate_users_by_distinct_id else "e.person_id" - replaced = parse_expr(f"count(DISTINCT {actor})") - aggregation_expression = replace_placeholders(aggregation_expression, {"replaced": replaced}) - - timestamp_field = self.series.timestamp_field if hasattr(self.series, "timestamp_field") else "timestamp" - date_filter = ast.And( - exprs=[ - parse_expr( - "{timestamp} >= {date_from_with_adjusted_start_of_interval}", - placeholders={ - **self.query_date_range.to_placeholders(), - "timestamp": ast.Field(chain=[timestamp_field]), - }, - ), - parse_expr( - "{timestamp} <= {date_to}", - placeholders={ - **self.query_date_range.to_placeholders(), - "timestamp": ast.Field(chain=[timestamp_field]), - }, - ), - ] - ) - - inner_events_query = parse_select( - """ - SELECT - {select_field}, - {aggregation_expression} as count - FROM {table} e - WHERE - {date_filter} and {events_where} - GROUP BY - value - ORDER BY - count DESC, - value DESC - LIMIT {breakdown_limit_plus_one} - """, - placeholders={ - "select_field": select_field, - "aggregation_expression": aggregation_expression, - "table": self._table, - "date_filter": date_filter, - "events_where": self.events_filter, - "breakdown_limit_plus_one": ast.Constant(value=breakdown_limit + 1), - }, - ) - - # Reverse the order if looking at the smallest values - if self.series.math_property is not None and self.series.math == "min": - if ( - isinstance(inner_events_query, ast.SelectQuery) - and inner_events_query.order_by is not None - and isinstance(inner_events_query.order_by[0], ast.OrderExpr) - ): - inner_events_query.order_by[0].order = "ASC" - - values: list[Any] - if self.histogram_bin_count is not None: - query = parse_select( - """ - SELECT {expr} FROM ({inner_events_query}) - """, - placeholders={ - "inner_events_query": inner_events_query, - "expr": self._to_bucketing_expression(), - }, - ) - response = execute_hogql_query( - query_type="TrendsQueryBreakdownValues", - query=query, - team=self.team, - modifiers=self.modifiers, - limit_context=self.limit_context, - ) - if response.results and len(response.results) > 0: - values = response.results[0][0] - else: - values = [] - else: - # We're not running this through groupArray, as that eats NULL values. - query = inner_events_query - response = execute_hogql_query( - query_type="TrendsQueryBreakdownValues", - query=query, - team=self.team, - modifiers=self.modifiers, - limit_context=self.limit_context, - ) - value_index = (response.columns or []).index("value") - values = [row[value_index] for row in response.results or []] - - needs_other = False - if len(values) == breakdown_limit + 1: - needs_other = True - values = values[:-1] - - # Add "other" value if "other" is not hidden and we're not bucketing numeric values - if self.hide_other_aggregation is not True and self.histogram_bin_count is None: - values = [BREAKDOWN_NULL_STRING_LABEL if value in (None, "") else value for value in values] - if needs_other: - values = [BREAKDOWN_OTHER_STRING_LABEL, *values] - - if len(values) == 0: - values.insert(0, None) - return values - - return values - - def _to_bucketing_expression(self) -> ast.Expr: - assert isinstance(self.histogram_bin_count, int) - - if self.histogram_bin_count <= 1: - qunatile_expression = "quantiles(0,1)(value)" - else: - quantiles = [] - bin_size = 1.0 / self.histogram_bin_count - for i in range(self.histogram_bin_count + 1): - quantiles.append(i * bin_size) - - qunatile_expression = f"quantiles({','.join([f'{quantile:.2f}' for quantile in quantiles])})(value)" - - return parse_expr(f"arrayCompact(arrayMap(x -> floor(x, 2), {qunatile_expression}))") - - @cached_property - def _id_field(self) -> ast.Field: - if isinstance(self.series, DataWarehouseNode): - return ast.Field(chain=["e", self.series.id_field]) - - return ast.Field(chain=["e", "uuid"]) - - @cached_property - def _table(self) -> ast.Field: - if isinstance(self.series, DataWarehouseNode): - return ast.Field(chain=[self.series.table_name]) - - return ast.Field(chain=["events"]) - - @cached_property - def _aggregation_operation(self) -> AggregationOperations: - return AggregationOperations( - self.team, - self.series, - self.chart_display_type, - self.query_date_range, - is_total_value=True, # doesn't matter in this case - ) diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr index 80d7c6c77345e..f7cca83fa28a4 100644 --- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr +++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr @@ -184,61 +184,42 @@ # --- # name: TestTrends.test_breakdown_by_group_props_person_on_events ''' - SELECT toString(e__group_0.properties___industry) AS value, - count(e.uuid) AS count - FROM events AS e - LEFT JOIN - (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 - ''' -# --- -# name: TestTrends.test_breakdown_by_group_props_person_on_events.1 - ''' - SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))))), 1))) AS date, - arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) - and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, - ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value FROM - (SELECT sum(total) AS count, - day_start AS day_start, - breakdown_value AS breakdown_value + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number FROM - (SELECT count(e.uuid) AS total, - toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, - transform(ifNull(nullIf(toString(e__group_0.properties___industry), ''), '$$_posthog_breakdown_null_$$'), ['finance', 'technology'], ['finance', 'technology'], '$$_posthog_breakdown_other_$$') AS breakdown_value - FROM events AS e SAMPLE 1 - LEFT JOIN - (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.uuid) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(e__group_0.properties___industry), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY day_start, + breakdown_value) GROUP BY day_start, - breakdown_value) - GROUP BY day_start, - breakdown_value - ORDER BY day_start ASC, breakdown_value ASC) + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)), - arraySum(total) DESC, breakdown_value ASC + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC LIMIT 50000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -248,7 +229,7 @@ max_query_size=524288 ''' # --- -# name: TestTrends.test_breakdown_by_group_props_person_on_events.2 +# name: TestTrends.test_breakdown_by_group_props_person_on_events.1 ''' SELECT persons.id AS id, toTimeZone(persons.created_at, 'UTC') AS created_at, @@ -273,7 +254,7 @@ WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0)) GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 2), equals(e.event, 'sign up'), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-02 00:00:00.000000', 6, 'UTC')), less(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-03 00:00:00.000000', 6, 'UTC')), ifNull(equals(toString(e__group_0.properties___industry), 'technology'), 0))) + WHERE and(equals(e.team_id, 2), equals(e.event, 'sign up'), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-02 00:00:00.000000', 6, 'UTC')), less(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-03 00:00:00.000000', 6, 'UTC')), ifNull(equals(e__group_0.properties___industry, 'technology'), 0))) GROUP BY actor_id) AS source INNER JOIN (SELECT argMax(person.created_at, person.version) AS created_at, @@ -293,6 +274,20 @@ max_query_size=524288 ''' # --- +# name: TestTrends.test_breakdown_by_group_props_person_on_events.2 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 + ''' +# --- # name: TestTrends.test_breakdown_by_group_props_person_on_events.3 ''' SELECT DISTINCT session_replay_events.session_id AS session_id @@ -309,27 +304,49 @@ # --- # name: TestTrends.test_breakdown_by_group_props_with_person_filter_person_on_events ''' - SELECT toString(e__group_0.properties___industry) AS value, - count(e.uuid) AS count - FROM events AS e - LEFT JOIN - (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'key'), ''), 'null'), '^"|"$', ''), 'value'), 0))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.uuid) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(e__group_0.properties___industry), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'key'), ''), 'null'), '^"|"$', ''), 'value'), 0), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_breakdown_by_group_props_with_person_filter_person_on_events.1 @@ -375,45 +392,73 @@ # --- # name: TestTrends.test_breakdown_filtering_with_properties_in_new_format ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.uuid) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_breakdown_filtering_with_properties_in_new_format.1 ''' - SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))))), 1))) AS date, - arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) - and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, - ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value FROM - (SELECT sum(total) AS count, - day_start AS day_start, - breakdown_value AS breakdown_value + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number FROM - (SELECT count(e.uuid) AS total, - toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, - transform(ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$'), ['second url'], ['second url'], '$$_posthog_breakdown_other_$$') AS breakdown_value - FROM events AS e SAMPLE 1 - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0), true) + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.uuid) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0), true) + GROUP BY day_start, + breakdown_value) GROUP BY day_start, - breakdown_value) - GROUP BY day_start, - breakdown_value - ORDER BY day_start ASC, breakdown_value ASC) + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)), - arraySum(total) DESC, breakdown_value ASC + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC LIMIT 50000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -475,26 +520,44 @@ # --- # name: TestTrends.test_breakdown_weekly_active_users_aggregated ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value, - count(DISTINCT e__pdi.person_id) AS count - FROM events AS e - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC')))), equals(e.event, '$pageview')) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT count(DISTINCT actor_id) AS total, + breakdown_value AS breakdown_value + FROM + (SELECT d.timestamp AS timestamp, + e.actor_id AS actor_id, + e.breakdown_value AS breakdown_value + FROM + (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp + FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC'))), toIntervalDay(7)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC')))) AS numbers) AS d + CROSS JOIN + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e__pdi.person_id AS actor_id, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + WHERE and(equals(e.team_id, 2), and(equals(e.event, '$pageview'), true), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0)) + GROUP BY timestamp, actor_id, + breakdown_value) AS e + WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(6))), 0)) + GROUP BY d.timestamp, + e.actor_id, + e.breakdown_value + ORDER BY d.timestamp ASC) + WHERE and(ifNull(greaterOrEquals(timestamp, toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0)) + GROUP BY breakdown_value + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_breakdown_weekly_active_users_aggregated.1 @@ -541,26 +604,44 @@ # --- # name: TestTrends.test_breakdown_weekly_active_users_aggregated_materialized ''' - SELECT toString(nullIf(nullIf(e.mat_key, ''), 'null')) AS value, - count(DISTINCT e__pdi.person_id) AS count - FROM events AS e - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC')))), equals(e.event, '$pageview')) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT count(DISTINCT actor_id) AS total, + breakdown_value AS breakdown_value + FROM + (SELECT d.timestamp AS timestamp, + e.actor_id AS actor_id, + e.breakdown_value AS breakdown_value + FROM + (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp + FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC'))), toIntervalDay(7)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC')))) AS numbers) AS d + CROSS JOIN + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e__pdi.person_id AS actor_id, + ifNull(nullIf(toString(nullIf(nullIf(e.mat_key, ''), 'null')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + WHERE and(equals(e.team_id, 2), and(equals(e.event, '$pageview'), true), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0)) + GROUP BY timestamp, actor_id, + breakdown_value) AS e + WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(6))), 0)) + GROUP BY d.timestamp, + e.actor_id, + e.breakdown_value + ORDER BY d.timestamp ASC) + WHERE and(ifNull(greaterOrEquals(timestamp, toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0)) + GROUP BY breakdown_value + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_breakdown_weekly_active_users_aggregated_materialized.1 @@ -627,40 +708,78 @@ # --- # name: TestTrends.test_breakdown_weekly_active_users_daily_based_on_action.2 ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value, - count(DISTINCT e__pdi.person_id) AS count - FROM events AS e - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'name'), ''), 'null'), '^"|"$', '') AS properties___name - FROM person - WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 2) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC')))), and(equals(e.event, '$pageview'), and(or(ifNull(equals(e__pdi__person.properties___name, 'p1'), 0), ifNull(equals(e__pdi__person.properties___name, 'p2'), 0), ifNull(equals(e__pdi__person.properties___name, 'p3'), 0)), ifNull(in(e__pdi.person_id, - (SELECT cohortpeople.person_id AS person_id - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0)))), 0)))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT counts AS total, + toStartOfDay(timestamp) AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT d.timestamp AS timestamp, + count(DISTINCT e.actor_id) AS counts, + e.breakdown_value AS breakdown_value + FROM + (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp + FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(7)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC')))) AS numbers) AS d + CROSS JOIN + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e__pdi.person_id AS actor_id, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'name'), ''), 'null'), '^"|"$', '') AS properties___name + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) + WHERE and(equals(e.team_id, 2), and(and(equals(e.event, '$pageview'), and(or(ifNull(equals(e__pdi__person.properties___name, 'p1'), 0), ifNull(equals(e__pdi__person.properties___name, 'p2'), 0), ifNull(equals(e__pdi__person.properties___name, 'p3'), 0)), ifNull(in(e__pdi.person_id, + (SELECT cohortpeople.person_id AS person_id + FROM cohortpeople + WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0)))), 0))), true), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0)) + GROUP BY timestamp, actor_id, + breakdown_value) AS e + WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(6))), 0)) + GROUP BY d.timestamp, + e.breakdown_value + ORDER BY d.timestamp ASC) + WHERE and(ifNull(greaterOrEquals(timestamp, toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0))) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_breakdown_weekly_active_users_daily_based_on_action.3 @@ -735,27 +854,49 @@ # --- # name: TestTrends.test_breakdown_with_filter_groups_person_on_events ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - LEFT JOIN - (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.uuid) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_breakdown_with_filter_groups_person_on_events.1 @@ -810,27 +951,56 @@ # --- # name: TestTrends.test_breakdown_with_filter_groups_person_on_events_v2.1 ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - LEFT JOIN - (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0)) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_breakdown_with_filter_groups_person_on_events_v2.2 @@ -883,52 +1053,87 @@ # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT e__pdi.person_id) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1.0 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.1 ''' - SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, - arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) - and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, - ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value FROM - (SELECT sum(total) AS count, - day_start AS day_start, - breakdown_value AS breakdown_value + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number FROM - (SELECT count(DISTINCT e__pdi.person_id) AS total, - toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, - transform(ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$'), ['other_value', '$$_posthog_breakdown_null_$$', 'value'], ['other_value', '$$_posthog_breakdown_null_$$', 'value'], '$$_posthog_breakdown_other_$$') AS breakdown_value - FROM events AS e SAMPLE 1.0 - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT e__pdi.person_id) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1.0 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY day_start, + breakdown_value) GROUP BY day_start, - breakdown_value) - GROUP BY day_start, - breakdown_value - ORDER BY day_start ASC, breakdown_value ASC) + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)), - arraySum(total) DESC, breakdown_value ASC + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC LIMIT 50000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -1290,38 +1495,76 @@ # --- # name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter ''' - SELECT toString(e__pdi__person.`properties___$some_prop`) AS value, - count(DISTINCT e__pdi.person_id) AS count - FROM events AS e - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop`, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'filter_prop'), ''), 'null'), '^"|"$', '') AS properties___filter_prop - FROM person - WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 2) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(equals(e.event, 'sign up'), ifNull(equals(e__pdi__person.properties___filter_prop, 'filter_val'), 0))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT counts AS total, + toStartOfDay(timestamp) AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT d.timestamp AS timestamp, + count(DISTINCT e.actor_id) AS counts, + e.breakdown_value AS breakdown_value + FROM + (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp + FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(30)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))) AS numbers) AS d + CROSS JOIN + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e__pdi.person_id AS actor_id, + ifNull(nullIf(toString(e__pdi__person.`properties___$some_prop`), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop`, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'filter_prop'), ''), 'null'), '^"|"$', '') AS properties___filter_prop + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) + WHERE and(equals(e.team_id, 2), and(equals(e.event, 'sign up'), ifNull(equals(e__pdi__person.properties___filter_prop, 'filter_val'), 0), true), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), toIntervalDay(30))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) + GROUP BY timestamp, actor_id, + breakdown_value) AS e + WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(29))), 0)) + GROUP BY d.timestamp, + e.breakdown_value + ORDER BY d.timestamp ASC) + WHERE and(ifNull(greaterOrEquals(timestamp, toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0))) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter.1 @@ -1394,26 +1637,64 @@ # --- # name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter_poe_v2 ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')) AS value, - count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS count - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 2) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0)) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'filter_prop'), ''), 'null'), '^"|"$', ''), 'filter_val'), 0))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT counts AS total, + toStartOfDay(timestamp) AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT d.timestamp AS timestamp, + count(DISTINCT e.actor_id) AS counts, + e.breakdown_value AS breakdown_value + FROM + (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS timestamp + FROM numbers(dateDiff('day', minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(30)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))) AS numbers) AS d + CROSS JOIN + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS actor_id, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0)) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'filter_prop'), ''), 'null'), '^"|"$', ''), 'filter_val'), 0), true), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), toIntervalDay(30))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) + GROUP BY timestamp, actor_id, + breakdown_value) AS e + WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(29))), 0)) + GROUP BY d.timestamp, + e.breakdown_value + ORDER BY d.timestamp ASC) + WHERE and(ifNull(greaterOrEquals(timestamp, toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0))) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter_poe_v2.1 @@ -1520,29 +1801,51 @@ # --- # name: TestTrends.test_person_filtering_in_cohort_in_action.2 ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), and(equals(e.event, 'sign up'), ifNull(in(e__pdi.person_id, - (SELECT cohortpeople.person_id AS person_id - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0)))), 0)))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.uuid) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), and(equals(e.event, 'sign up'), ifNull(in(e__pdi.person_id, + (SELECT cohortpeople.person_id AS person_id + FROM cohortpeople + WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0)))), 0)), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_person_filtering_in_cohort_in_action.3 @@ -1610,29 +1913,51 @@ # --- # name: TestTrends.test_person_filtering_in_cohort_in_action_poe_v2.2 ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 2) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0)) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), and(equals(e.event, 'sign up'), ifNull(in(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), - (SELECT cohortpeople.person_id AS person_id - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0)))), 0)))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.uuid) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0)) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), and(equals(e.event, 'sign up'), ifNull(in(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), + (SELECT cohortpeople.person_id AS person_id + FROM cohortpeople + WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0)))), 0)), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_person_filtering_in_cohort_in_action_poe_v2.3 @@ -2233,19 +2558,48 @@ # --- # name: TestTrends.test_timezones_daily.4 ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT e__pdi.person_id) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_timezones_daily.5 @@ -2421,19 +2775,48 @@ # --- # name: TestTrends.test_timezones_daily_minus_utc.4 ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT e__pdi.person_id) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'America/Phoenix')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), equals(e.event, 'sign up'), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_timezones_daily_minus_utc.5 @@ -2609,19 +2992,48 @@ # --- # name: TestTrends.test_timezones_daily_plus_utc.4 ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT e__pdi.person_id) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'Asia/Tokyo')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), equals(e.event, 'sign up'), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_timezones_daily_plus_utc.5 @@ -2999,85 +3411,113 @@ # --- # name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns ''' - SELECT toString(e__pdi__person.properties___email) AS value, - count(e.uuid) AS count - FROM events AS e - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, - argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$os'), ''), 'null'), '^"|"$', '') AS `properties___$os`, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` - FROM person - WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 2) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(or(ifNull(notILike(e__pdi__person.properties___email, '%@posthog.com%'), 1), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), or(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'safari'), 0))))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 - ''' -# --- -# name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns.1 - ''' - SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))))), 1))) AS date, - arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) - and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, - ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value FROM - (SELECT sum(total) AS count, - day_start AS day_start, - breakdown_value AS breakdown_value + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number FROM - (SELECT count(e.uuid) AS total, - toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, - transform(ifNull(nullIf(toString(e__pdi__person.properties___email), ''), '$$_posthog_breakdown_null_$$'), ['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com'], ['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com'], '$$_posthog_breakdown_other_$$') AS breakdown_value - FROM events AS e SAMPLE 1 - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, - argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$os'), ''), 'null'), '^"|"$', '') AS `properties___$os`, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` - FROM person - WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 2) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(or(ifNull(notILike(e__pdi__person.properties___email, '%@posthog.com%'), 1), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), or(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'safari'), 0))), true) + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.uuid) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(e__pdi__person.properties___email), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, + argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$os'), ''), 'null'), '^"|"$', '') AS `properties___$os`, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(or(ifNull(notILike(e__pdi__person.properties___email, '%@posthog.com%'), 1), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), or(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'safari'), 0))), true) + GROUP BY day_start, + breakdown_value) GROUP BY day_start, - breakdown_value) - GROUP BY day_start, - breakdown_value - ORDER BY day_start ASC, breakdown_value ASC) + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)), - arraySum(total) DESC, breakdown_value ASC + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 + ''' +# --- +# name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns.1 + ''' + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.uuid) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(e__pdi__person.properties___email), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, + argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$os'), ''), 'null'), '^"|"$', '') AS `properties___$os`, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'chrome'), 0)), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0), ifNull(ilike(e__pdi__person.properties___email, '%@posthog.com%'), 0)), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC LIMIT 50000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -3249,81 +3689,52 @@ # --- # name: TestTrends.test_trends_aggregate_by_distinct_id.2 ''' - SELECT toString(e__pdi__person.`properties___$some_prop`) AS value, - count(e.uuid) AS count - FROM events AS e - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, - argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` - FROM person - WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 2) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 - ''' -# --- -# name: TestTrends.test_trends_aggregate_by_distinct_id.3 - ''' - SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))))), 1))) AS date, - arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) - and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, - ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value FROM - (SELECT sum(total) AS count, - day_start AS day_start, - breakdown_value AS breakdown_value + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number FROM - (SELECT count(DISTINCT e.distinct_id) AS total, - toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, - transform(ifNull(nullIf(toString(e__pdi__person.`properties___$some_prop`), ''), '$$_posthog_breakdown_null_$$'), ['some_val', '$$_posthog_breakdown_null_$$'], ['some_val', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value - FROM events AS e SAMPLE 1 - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, - argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` - FROM person - WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 2) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT e.distinct_id) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(e__pdi__person.`properties___$some_prop`), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, + argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY day_start, + breakdown_value) GROUP BY day_start, - breakdown_value) - GROUP BY day_start, - breakdown_value - ORDER BY day_start ASC, breakdown_value ASC) + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)), - arraySum(total) DESC, breakdown_value ASC + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC LIMIT 50000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -3333,7 +3744,7 @@ max_query_size=524288 ''' # --- -# name: TestTrends.test_trends_aggregate_by_distinct_id.4 +# name: TestTrends.test_trends_aggregate_by_distinct_id.3 ''' SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))))), 1))) AS date, arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) @@ -3372,7 +3783,7 @@ max_query_size=524288 ''' # --- -# name: TestTrends.test_trends_aggregate_by_distinct_id.5 +# name: TestTrends.test_trends_aggregate_by_distinct_id.4 ''' SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))))), 1))) AS date, arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) @@ -3411,6 +3822,45 @@ max_query_size=524288 ''' # --- +# name: TestTrends.test_trends_aggregate_by_distinct_id.5 + ''' + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT e.distinct_id) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 + ''' +# --- # name: TestTrends.test_trends_aggregate_by_distinct_id.6 ''' SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', '')) AS value, @@ -3515,19 +3965,54 @@ # --- # name: TestTrends.test_trends_breakdown_cumulative ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayFill(x -> ifNull(greater(x, 0), 0), arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date)) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT day_start AS day_start, + sum(count) OVER (PARTITION BY breakdown_value + ORDER BY day_start ASC) AS count, + breakdown_value AS breakdown_value + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT e__pdi.person_id) AS total, + min(toStartOfDay(toTimeZone(e.timestamp, 'UTC'))) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY e__pdi.person_id, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + ORDER BY day_start ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_trends_breakdown_cumulative.1 @@ -3578,19 +4063,54 @@ # --- # name: TestTrends.test_trends_breakdown_cumulative_poe_v2 ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayFill(x -> ifNull(greater(x, 0), 0), arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date)) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT day_start AS day_start, + sum(count) OVER (PARTITION BY breakdown_value + ORDER BY day_start ASC) AS count, + breakdown_value AS breakdown_value + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + min(toStartOfDay(toTimeZone(e.timestamp, 'UTC'))) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0)) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + ORDER BY day_start ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_trends_breakdown_cumulative_poe_v2.1 @@ -3628,8 +4148,62 @@ ORDER BY day_start ASC, breakdown_value ASC) ORDER BY day_start ASC) GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)), - arraySum(total) DESC, breakdown_value ASC + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)), + arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 + ''' +# --- +# name: TestTrends.test_trends_breakdown_normalize_url + ''' + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayFill(x -> ifNull(greater(x, 0), 0), arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date)) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT day_start AS day_start, + sum(count) OVER (PARTITION BY breakdown_value + ORDER BY day_start ASC) AS count, + breakdown_value AS breakdown_value + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + min(toStartOfDay(toTimeZone(e.timestamp, 'UTC'))) AS day_start, + ifNull(nullIf(toString(if(empty(trim(TRAILING '/?#' + FROM replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''))), '/', trim(TRAILING '/?#' + FROM replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')))), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0)) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + ORDER BY day_start ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC LIMIT 50000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -3639,25 +4213,6 @@ max_query_size=524288 ''' # --- -# name: TestTrends.test_trends_breakdown_normalize_url - ''' - SELECT toString(if(empty(trim(TRAILING '/?#' - FROM replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''))), '/', trim(TRAILING '/?#' - FROM replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')))) AS value, - count(e.uuid) AS count - FROM events AS e - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 - ''' -# --- # name: TestTrends.test_trends_breakdown_normalize_url.1 ''' SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, @@ -3708,26 +4263,31 @@ # --- # name: TestTrends.test_trends_breakdown_with_session_property_single_aggregate_math_and_breakdown ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value, - max(e__session.`$session_duration`) AS count - FROM events AS e - LEFT JOIN - (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, - sessions.session_id AS session_id - FROM sessions - WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) - GROUP BY sessions.session_id, - sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT quantile(0.5)(session_duration) AS total, + breakdown_value AS breakdown_value + FROM + (SELECT any(e__session.`$session_duration`) AS session_duration, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + LEFT JOIN + (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, + sessions.session_id AS session_id + FROM sessions + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) + GROUP BY sessions.session_id, + sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY e.`$session_id`, + breakdown_value + ORDER BY 1 DESC, breakdown_value DESC) + GROUP BY breakdown_value + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_trends_breakdown_with_session_property_single_aggregate_math_and_breakdown.1 @@ -3736,18 +4296,19 @@ breakdown_value AS breakdown_value FROM (SELECT any(e__session.`$session_duration`) AS session_duration, - transform(ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$'), ['value2', 'value1', '$$_posthog_breakdown_null_$$'], ['value2', 'value1', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value FROM events AS e SAMPLE 1 LEFT JOIN (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) GROUP BY e.`$session_id`, - breakdown_value) + breakdown_value + ORDER BY 1 DESC, breakdown_value DESC) GROUP BY breakdown_value LIMIT 50000 SETTINGS readonly=2, max_execution_time=60, @@ -3942,19 +4503,33 @@ # --- # name: TestTrends.test_trends_count_per_user_average_aggregated_with_event_property_breakdown_with_sampling ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')) AS value, - count(e.uuid) AS count - FROM events AS e - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC')))), equals(e.event, 'viewed video')) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT total AS total, + breakdown_value AS breakdown_value + FROM + (SELECT avg(total) AS total, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.uuid) AS total, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1.0 + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + WHERE and(equals(e.team_id, 2), and(equals(e.event, 'viewed video'), true), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), toIntervalDay(0))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC')))) + GROUP BY e__pdi.person_id, + breakdown_value) + GROUP BY breakdown_value) + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_trends_count_per_user_average_aggregated_with_event_property_breakdown_with_sampling.1 @@ -4198,44 +4773,49 @@ # --- # name: TestTrends.test_trends_person_breakdown_with_session_property_single_aggregate_math_and_breakdown ''' - SELECT toString(e__pdi__person.`properties___$some_prop`) AS value, - max(e__session.`$session_duration`) AS count - FROM events AS e - LEFT JOIN - (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, - sessions.session_id AS session_id - FROM sessions - WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) - GROUP BY sessions.session_id, - sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) - INNER JOIN - (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, - argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, - person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 2) - GROUP BY person_distinct_id2.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` - FROM person - WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 2) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT quantile(0.5)(session_duration) AS total, + breakdown_value AS breakdown_value + FROM + (SELECT any(e__session.`$session_duration`) AS session_duration, + ifNull(nullIf(toString(e__pdi__person.`properties___$some_prop`), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1 + LEFT JOIN + (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, + sessions.session_id AS session_id + FROM sessions + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) + GROUP BY sessions.session_id, + sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, + argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY e.`$session_id`, + breakdown_value + ORDER BY 1 DESC, breakdown_value DESC) + GROUP BY breakdown_value + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_trends_person_breakdown_with_session_property_single_aggregate_math_and_breakdown.1 @@ -4324,7 +4904,8 @@ GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) - GROUP BY e.`$session_id`) + GROUP BY e.`$session_id` + ORDER BY 1 DESC) LIMIT 50000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -4348,7 +4929,8 @@ GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) - GROUP BY e.`$session_id`) + GROUP BY e.`$session_id` + ORDER BY 1 DESC) LIMIT 50000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -4438,67 +5020,103 @@ # --- # name: TestTrends.test_trends_with_session_property_total_volume_math_with_breakdowns ''' - SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value, - max(e__session.`$session_duration`) AS count - FROM events AS e - LEFT JOIN - (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, - sessions.session_id AS session_id - FROM sessions - WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) - GROUP BY sessions.session_id, - sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) - WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0), toIntervalWeek(number)), range(0, plus(coalesce(dateDiff('week', toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')), 0))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT quantile(0.5)(session_duration) AS total, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT any(e__session.`$session_duration`) AS session_duration, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value, + toStartOfWeek(toTimeZone(e.timestamp, 'UTC'), 0) AS day_start + FROM events AS e SAMPLE 1 + LEFT JOIN + (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, + sessions.session_id AS session_id + FROM sessions + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) + GROUP BY sessions.session_id, + sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY day_start, + e.`$session_id`, + breakdown_value, + day_start) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrends.test_trends_with_session_property_total_volume_math_with_breakdowns.1 ''' - SELECT arrayMap(number -> plus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0), toIntervalWeek(number)), range(0, plus(coalesce(dateDiff('week', toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')), 0))), 1))) AS date, - arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) - and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, - ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value FROM - (SELECT sum(total) AS count, - day_start AS day_start, - breakdown_value AS breakdown_value + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number FROM - (SELECT quantile(0.5)(session_duration) AS total, + (SELECT sum(total) AS count, day_start AS day_start, breakdown_value AS breakdown_value FROM - (SELECT any(e__session.`$session_duration`) AS session_duration, - transform(ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$'), ['value2', 'value1'], ['value2', 'value1'], '$$_posthog_breakdown_other_$$') AS breakdown_value, - toStartOfWeek(toTimeZone(e.timestamp, 'UTC'), 0) AS day_start - FROM events AS e SAMPLE 1 - LEFT JOIN - (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, - sessions.session_id AS session_id - FROM sessions - WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) - GROUP BY sessions.session_id, - sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) - WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + (SELECT quantile(0.5)(session_duration) AS total, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT any(e__session.`$session_duration`) AS session_duration, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start + FROM events AS e SAMPLE 1 + LEFT JOIN + (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, + sessions.session_id AS session_id + FROM sessions + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) + GROUP BY sessions.session_id, + sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) + GROUP BY day_start, + e.`$session_id`, + breakdown_value, + day_start) GROUP BY day_start, - e.`$session_id`, - breakdown_value, - day_start) + breakdown_value) GROUP BY day_start, - breakdown_value) - GROUP BY day_start, - breakdown_value - ORDER BY day_start ASC, breakdown_value ASC) + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) GROUP BY breakdown_value - ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)), - arraySum(total) DESC, breakdown_value ASC + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC LIMIT 50000 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr index 916b036f00e17..5b65891935f0c 100644 --- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr +++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr @@ -1,19 +1,41 @@ # serializer version: 1 # name: TestTrendsDataWarehouseQuery.test_trends_breakdown ''' - SELECT toString(e.prop_1) AS value, - count(e.id) AS count - FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', '`id` String, `prop_1` String, `prop_2` String, `created` DateTime64(3, \'UTC\')') AS e - WHERE and(and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)), and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.id) AS total, + toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start, + ifNull(nullIf(toString(e.prop_1), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', '`id` String, `prop_1` String, `prop_2` String, `created` DateTime64(3, \'UTC\')') AS e + WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrendsDataWarehouseQuery.test_trends_breakdown.1 @@ -51,19 +73,41 @@ # --- # name: TestTrendsDataWarehouseQuery.test_trends_breakdown_with_property ''' - SELECT toString(e.prop_1) AS value, - count(e.id) AS count - FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', '`id` String, `prop_1` String, `prop_2` String, `created` DateTime64(3, \'UTC\')') AS e - WHERE and(and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)), and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'))) - GROUP BY value - ORDER BY count DESC, value DESC - LIMIT 26 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=1000000, - max_expanded_ast_elements=1000000, - max_query_size=524288 + SELECT groupArray(1)(date)[1] AS date, + arrayMap(i -> arraySum(arrayMap(x -> arrayElement(x, i), groupArray(total))), arrayEnumerate(date)) AS total, + if(ifNull(greaterOrEquals(row_number, 25), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(e.id) AS total, + toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start, + ifNull(nullIf(toString(e.prop_1), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', '`id` String, `prop_1` String, `prop_2` String, `created` DateTime64(3, \'UTC\')') AS e + WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'), true) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY arraySum(total) DESC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=1000000, + max_expanded_ast_elements=1000000, + max_query_size=524288 ''' # --- # name: TestTrendsDataWarehouseQuery.test_trends_breakdown_with_property.1 diff --git a/posthog/hogql_queries/insights/trends/test/test_trends.py b/posthog/hogql_queries/insights/trends/test/test_trends.py index 26a1b7f1a6d4f..c3838fa8fd63c 100644 --- a/posthog/hogql_queries/insights/trends/test/test_trends.py +++ b/posthog/hogql_queries/insights/trends/test/test_trends.py @@ -1117,11 +1117,7 @@ def test_unique_session_with_session_breakdown(self): self.assertEqual( [(item["breakdown_value"], item["count"], item["data"]) for item in response], - [ - ("[4.95,10.05]", 2.0, [2, 0, 0, 0]), - ("[0.0,4.95]", 1.0, [1, 0, 0, 0]), - ("[10.05,15.01]", 1.0, [0, 1, 0, 0]), - ], + [("[10,15.01]", 2.0, [1, 1, 0, 0]), ("[0,5]", 1.0, [1, 0, 0, 0]), ("[5,10]", 1.0, [1, 0, 0, 0])], ) @also_test_with_person_on_events_v2 @@ -6503,9 +6499,9 @@ def test_breakdown_filtering_bar_chart_by_value(self): self.team, ) - self.assertEqual(response[0]["aggregated_value"], 1) + self.assertEqual(response[0]["aggregated_value"], 2) self.assertEqual(response[1]["aggregated_value"], 1) - self.assertEqual(response[2]["aggregated_value"], 2) # the events without breakdown value + self.assertEqual(response[2]["aggregated_value"], 1) # the events without breakdown value self.assertEqual(response[0]["days"], []) @also_test_with_materialized_columns(person_properties=["key", "key_2"], verify_no_jsonextract=False) diff --git a/posthog/hogql_queries/insights/trends/test/test_trends_persons.py b/posthog/hogql_queries/insights/trends/test/test_trends_persons.py index 1751c7482dd40..ff1991ff537d8 100644 --- a/posthog/hogql_queries/insights/trends/test/test_trends_persons.py +++ b/posthog/hogql_queries/insights/trends/test/test_trends_persons.py @@ -343,7 +343,6 @@ def test_trends_breakdown_hogql_persons(self): ) result = self._get_actors(trends_query=source_query, day="2023-05-01", breakdown=20) - self.assertEqual(len(result), 1) self.assertEqual(get_distinct_id(result[0]), "person1") self.assertEqual(get_event_count(result[0]), 1) diff --git a/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py b/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py index c87d55643d771..5ac3c588fd970 100644 --- a/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py +++ b/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py @@ -9,8 +9,7 @@ from posthog.hogql import ast from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS, LimitContext from posthog.hogql.modifiers import create_default_modifiers_for_team -from posthog.hogql_queries.insights.trends.breakdown_values import BREAKDOWN_OTHER_DISPLAY -from posthog.hogql_queries.insights.trends.trends_query_runner import TrendsQueryRunner +from posthog.hogql_queries.insights.trends.trends_query_runner import TrendsQueryRunner, BREAKDOWN_OTHER_DISPLAY from posthog.models.cohort.cohort import Cohort from posthog.models.property_definition import PropertyDefinition @@ -811,7 +810,7 @@ def test_breakdown_is_context_aware(self, mock_sync_execute: MagicMock): limit_context=LimitContext.QUERY_ASYNC, ) - self.assertEqual(mock_sync_execute.call_count, 4) + self.assertEqual(mock_sync_execute.call_count, 2) for mock_execute_call_args in mock_sync_execute.call_args_list: self.assertIn(f" max_execution_time={HOGQL_INCREASED_MAX_EXECUTION_TIME},", mock_execute_call_args[0][0]) @@ -981,11 +980,11 @@ def test_trends_breakdowns_histogram(self): breakdown_labels = [result["breakdown_value"] for result in response.results] assert len(response.results) == 4 - assert breakdown_labels == ["[10.0,17.5]", "[17.5,25.0]", "[25.0,32.5]", "[32.5,40.01]"] + assert breakdown_labels == ["[10,17.5]", "[17.5,25]", "[25,32.5]", "[32.5,40.01]"] - assert response.results[0]["label"] == "[10.0,17.5]" - assert response.results[1]["label"] == "[17.5,25.0]" - assert response.results[2]["label"] == "[25.0,32.5]" + assert response.results[0]["label"] == "[10,17.5]" + assert response.results[1]["label"] == "[17.5,25]" + assert response.results[2]["label"] == "[25,32.5]" assert response.results[3]["label"] == "[32.5,40.01]" assert response.results[0]["data"] == [0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0] @@ -1852,7 +1851,7 @@ def test_to_actors_query_options(self): IntervalType.DAY, [EventsNode(event="$pageview")], None, - None, + BreakdownFilter(breakdown_type=None, breakdown=None), ) response = runner.to_actors_query_options() @@ -1992,10 +1991,10 @@ def test_to_actors_query_options_breakdowns(self): assert response.day is not None assert response.series == [InsightActorsQuerySeries(label="$pageview", value=0)] assert response.breakdown == [ - BreakdownItem(label=BREAKDOWN_OTHER_DISPLAY, value="$$_posthog_breakdown_other_$$"), BreakdownItem(label="Chrome", value="Chrome"), BreakdownItem(label="Firefox", value="Firefox"), - BreakdownItem(label="Safari", value="Safari"), + BreakdownItem(label="Edge", value="Edge"), + BreakdownItem(label=BREAKDOWN_OTHER_DISPLAY, value="$$_posthog_breakdown_other_$$"), ] def test_to_actors_query_options_breakdowns_boolean(self): @@ -2042,9 +2041,9 @@ def test_to_actors_query_options_breakdowns_histogram(self): assert response.series == [InsightActorsQuerySeries(label="$pageview", value=0)] assert response.breakdown == [ - BreakdownItem(label="[10.0,17.5]", value="[10.0,17.5]"), - BreakdownItem(label="[17.5,25.0]", value="[17.5,25.0]"), - BreakdownItem(label="[25.0,32.5]", value="[25.0,32.5]"), + BreakdownItem(label="[10,17.5]", value="[10,17.5]"), + BreakdownItem(label="[17.5,25]", value="[17.5,25]"), + BreakdownItem(label="[25,32.5]", value="[25,32.5]"), BreakdownItem(label="[32.5,40.01]", value="[32.5,40.01]"), BreakdownItem(label='["",""]', value='["",""]'), ] @@ -2105,8 +2104,8 @@ def test_to_actors_query_options_breakdowns_hogql(self): assert response.breakdown == [ BreakdownItem(label="Chrome", value="Chrome"), BreakdownItem(label="Firefox", value="Firefox"), - BreakdownItem(label="Safari", value="Safari"), BreakdownItem(label="Edge", value="Edge"), + BreakdownItem(label="Safari", value="Safari"), ] def test_to_actors_query_options_bar_value(self): diff --git a/posthog/hogql_queries/insights/trends/trends_actors_query_builder.py b/posthog/hogql_queries/insights/trends/trends_actors_query_builder.py index 2d404f0a8c092..bf55a3eb82489 100644 --- a/posthog/hogql_queries/insights/trends/trends_actors_query_builder.py +++ b/posthog/hogql_queries/insights/trends/trends_actors_query_builder.py @@ -384,12 +384,11 @@ def _breakdown_where_expr(self) -> list[ast.Expr]: timings=self.timings, modifiers=self.modifiers, events_filter=self._events_where_expr(with_breakdown_expr=False), - breakdown_values_override=[self.breakdown_value] if self.breakdown_value is not None else None, limit_context=self.limit_context, ) if breakdown.enabled and not breakdown.is_histogram_breakdown: - breakdown_filter = breakdown.events_where_filter() + breakdown_filter = breakdown.events_where_filter(breakdown_values_override=self.breakdown_value) if breakdown_filter is not None: conditions.append(breakdown_filter) diff --git a/posthog/hogql_queries/insights/trends/trends_query_builder.py b/posthog/hogql_queries/insights/trends/trends_query_builder.py index 015e269e5628e..75418ffba2a3e 100644 --- a/posthog/hogql_queries/insights/trends/trends_query_builder.py +++ b/posthog/hogql_queries/insights/trends/trends_query_builder.py @@ -8,13 +8,13 @@ from posthog.hogql_queries.insights.trends.aggregation_operations import ( AggregationOperations, ) -from posthog.hogql_queries.insights.trends.breakdown import Breakdown -from posthog.hogql_queries.insights.trends.breakdown_values import BREAKDOWN_OTHER_STRING_LABEL +from posthog.hogql_queries.insights.trends.breakdown import Breakdown, BREAKDOWN_OTHER_STRING_LABEL from posthog.hogql_queries.insights.trends.display import TrendsDisplay from posthog.hogql_queries.insights.trends.utils import series_event_name from posthog.hogql_queries.utils.query_date_range import QueryDateRange from posthog.models.action.action import Action from posthog.models.filters.mixins.utils import cached_property +from posthog.hogql.constants import get_breakdown_limit_for_context from posthog.models.team.team import Team from posthog.queries.trends.breakdown import BREAKDOWN_NULL_STRING_LABEL from posthog.schema import ( @@ -98,14 +98,12 @@ def _get_events_subquery( no_modifications: Optional[bool], is_actors_query: bool, breakdown: Breakdown, - breakdown_values_override: Optional[str | int] = None, actors_query_time_frame: Optional[str] = None, ) -> ast.SelectQuery: events_filter = self._events_filter( ignore_breakdowns=False, breakdown=breakdown, is_actors_query=is_actors_query, - breakdown_values_override=breakdown_values_override, actors_query_time_frame=actors_query_time_frame, ) @@ -132,7 +130,13 @@ def _get_events_subquery( ), ) - if not self._trends_display.is_total_value(): # TODO: remove: and not is_actors_query + # If it's total value, we should order the results as there's no outer query to do the ordering + if self._trends_display.is_total_value(): + default_query.order_by = [ast.OrderExpr(expr=parse_expr("1"), order="DESC")] + if breakdown.enabled: + default_query.order_by.append(ast.OrderExpr(expr=ast.Field(chain=["breakdown_value"]), order="DESC")) + + else: # For cumulative unique users or groups, we want to count each user or group once per query, not per day if ( self.query.trendsFilter @@ -197,12 +201,15 @@ def _get_events_subquery( wrapper.group_by.append(ast.Field(chain=["day_start"])) wrapper.select.append(ast.Field(chain=["breakdown_value"])) - wrapper.group_by.append(ast.Field(chain=["breakdown_value"])) + if not breakdown.is_histogram_breakdown: + wrapper.group_by.append(ast.Field(chain=["breakdown_value"])) return wrapper + # Just breakdowns elif breakdown.enabled: breakdown_expr = breakdown.column_expr() + default_query.select.append(breakdown_expr) default_query.group_by.append(ast.Field(chain=["breakdown_value"])) # Just session duration math property @@ -236,7 +243,9 @@ def _get_events_subquery( return default_query - def _outer_select_query(self, breakdown: Breakdown, inner_query: ast.SelectQuery) -> ast.SelectQuery: + def _outer_select_query( + self, breakdown: Breakdown, inner_query: ast.SelectQuery + ) -> ast.SelectQuery | ast.SelectUnionQuery: total_array = parse_expr( """ arrayMap( @@ -322,24 +331,47 @@ def _outer_select_query(self, breakdown: Breakdown, inner_query: ast.SelectQuery ), ) ) + query.select.append(ast.Alias(alias="row_number", expr=parse_expr("rowNumberInAllBlocks()"))) query.group_by = [ast.Field(chain=["breakdown_value"])] - query.order_by.insert( - 0, - cast( - ast.OrderExpr, - parse_expr( - "breakdown_value = {other} ? 2 : breakdown_value = {nil} ? 1 : 0", - placeholders={ - "other": ast.Constant(value=BREAKDOWN_OTHER_STRING_LABEL), - "nil": ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL), - }, - ), - ), - ) + query.order_by.append(ast.OrderExpr(expr=ast.Field(chain=["breakdown_value"]), order="ASC")) + # TODO: What happens with cohorts and this limit? + if not breakdown.is_histogram_breakdown: + return parse_select( + """ + SELECT + groupArray(1)(date)[1] as date, + arrayMap( + i -> + arraySum(arrayMap( + x -> arrayElement(x, i), + groupArray(total) + )), + arrayEnumerate(date) + ) as total, + if(row_number >= {breakdown_limit}, {other}, breakdown_value) as breakdown_value + FROM {outer_query} + GROUP BY breakdown_value + ORDER BY + breakdown_value = {other} ? 2 : breakdown_value = {nil} ? 1 : 0, + arraySum(total) DESC, + breakdown_value ASC + """, + { + "outer_query": query, + "breakdown_limit": ast.Constant(value=self._get_breakdown_limit()), + "other": ast.Constant(value=BREAKDOWN_OTHER_STRING_LABEL), + "nil": ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL), + }, + ) return query + def _get_breakdown_limit(self) -> int: + return ( + self.query.breakdownFilter and self.query.breakdownFilter.breakdown_limit + ) or get_breakdown_limit_for_context(self.limit_context) + def _inner_select_query( self, breakdown: Breakdown, inner_query: ast.SelectQuery | ast.SelectUnionQuery ) -> ast.SelectQuery: @@ -364,7 +396,47 @@ def _inner_select_query( query.order_by.append(ast.OrderExpr(expr=ast.Field(chain=["day_start"]), order="ASC")) if breakdown.enabled: - query.select.append(ast.Field(chain=["breakdown_value"])) + if breakdown.is_histogram_breakdown: + histogram_bin_count = ( + self.query.breakdownFilter.breakdown_histogram_bin_count if self.query.breakdownFilter else None + ) + query.ctes = { + "min_max": ast.CTE( + name="min_max", + expr=self._get_events_subquery( + no_modifications=False, is_actors_query=False, breakdown=breakdown + ), + cte_type="subquery", + ) + } + query.select.extend( + [ + # Using arrays would be more efficient here, _but_ only if there's low cardinality in breakdown_values + # If cardinality is high it'd blow up memory + # Clickhouse is reasonably clever not rereading the same data + parse_expr("(select max(breakdown_value) from min_max) as max_num"), + parse_expr("(select min(breakdown_value) from min_max) as min_num"), + parse_expr("max_num - min_num as diff"), + parse_expr(f"{histogram_bin_count} as bins"), + parse_expr(""" + arrayMap( + x -> [ + ((diff / bins) * x) + min_num, + ((diff / bins) * (x + 1)) + min_num + if(x + 1 = bins, 0.01, 0) + ], + range(bins) + ) as buckets + """), + parse_expr("""arrayFilter( + x -> + x[1] <= breakdown_value and breakdown_value < x[2], + buckets + )[1] as breakdown_value + """), + ] + ) + else: + query.select.append(ast.Field(chain=["breakdown_value"])) query.group_by.append(ast.Field(chain=["breakdown_value"])) query.order_by.append(ast.OrderExpr(expr=ast.Field(chain=["breakdown_value"]), order="ASC")) @@ -380,7 +452,6 @@ def _events_filter( is_actors_query: bool, breakdown: Breakdown | None, ignore_breakdowns: bool = False, - breakdown_values_override: Optional[str | int] = None, actors_query_time_frame: Optional[str] = None, ) -> ast.Expr: series = self.series @@ -479,7 +550,7 @@ def session_duration_math_property_wrapper(self, default_query: ast.SelectQuery) query.group_by = [] return query - def _breakdown(self, is_actors_query: bool, breakdown_values_override: Optional[str] = None): + def _breakdown(self, is_actors_query: bool): return Breakdown( team=self.team, query=self.query, @@ -491,9 +562,7 @@ def _breakdown(self, is_actors_query: bool, breakdown_values_override: Optional[ breakdown=None, # Passing in None because we know we dont actually need it ignore_breakdowns=True, is_actors_query=is_actors_query, - breakdown_values_override=breakdown_values_override, ), - breakdown_values_override=[breakdown_values_override] if breakdown_values_override is not None else None, limit_context=self.limit_context, ) diff --git a/posthog/hogql_queries/insights/trends/trends_query_runner.py b/posthog/hogql_queries/insights/trends/trends_query_runner.py index 50d7db4160255..18374b0677ce4 100644 --- a/posthog/hogql_queries/insights/trends/trends_query_runner.py +++ b/posthog/hogql_queries/insights/trends/trends_query_runner.py @@ -22,13 +22,13 @@ from posthog.hogql.printer import to_printed_hogql from posthog.hogql.query import execute_hogql_query from posthog.hogql.timings import HogQLTimings -from posthog.hogql_queries.insights.trends.breakdown_values import ( +from posthog.hogql_queries.insights.trends.display import TrendsDisplay +from posthog.hogql_queries.insights.trends.breakdown import ( BREAKDOWN_NULL_DISPLAY, BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_DISPLAY, BREAKDOWN_OTHER_STRING_LABEL, ) -from posthog.hogql_queries.insights.trends.display import TrendsDisplay from posthog.hogql_queries.insights.trends.trends_query_builder import TrendsQueryBuilder from posthog.hogql_queries.insights.trends.trends_actors_query_builder import TrendsActorsQueryBuilder from posthog.hogql_queries.insights.trends.series_with_extras import SeriesWithExtras @@ -215,60 +215,65 @@ def to_actors_query_options(self) -> InsightActorsQueryOptionsResponse: ] # Breakdowns - for series in self.query.series: - # TODO: Add support for DataWarehouseNode - if isinstance(series, DataWarehouseNode): - continue + if self.query.breakdownFilter is not None and self.query.breakdownFilter.breakdown is not None: + res_breakdown = [] + if self.query.breakdownFilter.breakdown_type == "cohort": + assert isinstance(self.query.breakdownFilter.breakdown, list) + for value in self.query.breakdownFilter.breakdown: + if value != "all" and str(value) != "0": + res_breakdown.append( + BreakdownItem(label=Cohort.objects.get(pk=int(value), team=self.team).name, value=value) + ) + else: + res_breakdown.append(BreakdownItem(label="all users", value="all")) + else: + # TODO: Work out if we will have issues only getting breakdown values for + # the "current" period and not "previous" period for when "compare" is turned on + query_date_range = self.query_date_range - # TODO: Work out if we will have issues only getting breakdown values for - # the "current" period and not "previous" period for when "compare" is turned on - query_date_range = self.query_date_range + query_builder = TrendsQueryBuilder( + trends_query=self.query, + team=self.team, + query_date_range=query_date_range, + series=series, + timings=self.timings, + modifiers=self.modifiers, + limit_context=self.limit_context, + ) - query_builder = TrendsQueryBuilder( - trends_query=self.query, - team=self.team, - query_date_range=query_date_range, - series=series, - timings=self.timings, - modifiers=self.modifiers, - limit_context=self.limit_context, - ) + query = query_builder.build_query() - breakdown = query_builder._breakdown(is_actors_query=False) - if not breakdown.enabled: - break + breakdown = query_builder._breakdown(is_actors_query=False) - is_boolean_breakdown = self._is_breakdown_field_boolean() - is_histogram_breakdown = breakdown.is_histogram_breakdown - breakdown_values: list[str | int] - res_breakdown = [] + results = execute_hogql_query( + query_type="TrendsActorsQueryOptions", + query=query, + team=self.team, + # timings=timings, + # modifiers=modifiers, + ) + breakdown_values = [ + row[results.columns.index("breakdown_value") if results.columns else 2] for row in results.results + ] - if is_histogram_breakdown: - buckets = breakdown._get_breakdown_histogram_buckets() - breakdown_values = [f"[{t[0]},{t[1]}]" for t in buckets] - # TODO: append this only if needed - breakdown_values.append('["",""]') - else: - breakdown_values = breakdown._breakdown_values - - for value in breakdown_values: - if self.query.breakdownFilter is not None and self.query.breakdownFilter.breakdown_type == "cohort": - is_all = value == "all" or str(value) == "0" - label = "all users" if is_all else Cohort.objects.get(pk=value).name - value = "all" if is_all else value - elif value == BREAKDOWN_OTHER_STRING_LABEL: - label = BREAKDOWN_OTHER_DISPLAY - elif value == BREAKDOWN_NULL_STRING_LABEL: - label = BREAKDOWN_NULL_DISPLAY - elif is_boolean_breakdown: - label = self._convert_boolean(value) - else: - label = str(value) + if breakdown.is_histogram_breakdown: + breakdown_values.append('["",""]') + is_boolean_breakdown = self._is_breakdown_field_boolean() + + for value in breakdown_values: + if value == BREAKDOWN_OTHER_STRING_LABEL: + label = BREAKDOWN_OTHER_DISPLAY + elif value == BREAKDOWN_NULL_STRING_LABEL: + label = BREAKDOWN_NULL_DISPLAY + elif is_boolean_breakdown: + label = self._convert_boolean(value) + else: + label = str(value) - item = BreakdownItem(label=label, value=value) + item = BreakdownItem(label=label, value=value) - if item not in res_breakdown: - res_breakdown.append(item) + if item not in res_breakdown: + res_breakdown.append(item) return InsightActorsQueryOptionsResponse( series=res_series, breakdown=res_breakdown, day=res_days, compare=res_compare diff --git a/posthog/models/activity_logging/activity_log.py b/posthog/models/activity_logging/activity_log.py index 141130ea4f80e..c05b0cac48bbd 100644 --- a/posthog/models/activity_logging/activity_log.py +++ b/posthog/models/activity_logging/activity_log.py @@ -18,6 +18,7 @@ logger = structlog.get_logger(__name__) ActivityScope = Literal[ + "Cohort", "FeatureFlag", "Person", "Insight", @@ -133,6 +134,14 @@ class Meta: field_exclusions: dict[ActivityScope, list[str]] = { + "Cohort": [ + "version", + "pending_version", + "count", + "is_calculating", + "last_calculation", + "errors_calculating", + ], "Notebook": [ "text_content", ], diff --git a/posthog/models/cohort/util.py b/posthog/models/cohort/util.py index 118c0baec74eb..a505fd4e8489f 100644 --- a/posthog/models/cohort/util.py +++ b/posthog/models/cohort/util.py @@ -8,6 +8,7 @@ from django.utils import timezone from rest_framework.exceptions import ValidationError +from posthog.clickhouse.client.connection import Workload from posthog.clickhouse.query_tagging import tag_queries from posthog.client import sync_execute from posthog.constants import PropertyOperatorType @@ -324,6 +325,7 @@ def recalculate_cohortpeople( "new_version": pending_version, }, settings={"optimize_on_insert": 0}, + workload=Workload.OFFLINE, ) count = get_cohort_size(cohort, override_version=pending_version) @@ -370,6 +372,7 @@ def get_cohort_size(cohort: Cohort, override_version: Optional[int] = None) -> O "version": override_version if override_version is not None else cohort.version, "team_id": cohort.team_id, }, + workload=Workload.OFFLINE, ) if count_result and len(count_result) and len(count_result[0]): diff --git a/posthog/queries/trends/breakdown.py b/posthog/queries/trends/breakdown.py index e0bab69fe666d..663ccd42ccff6 100644 --- a/posthog/queries/trends/breakdown.py +++ b/posthog/queries/trends/breakdown.py @@ -18,7 +18,6 @@ PropertyOperatorType, TREND_FILTER_TYPE_EVENTS, ) -from posthog.hogql_queries.insights.trends.breakdown_values import BREAKDOWN_NULL_DISPLAY, BREAKDOWN_OTHER_DISPLAY from posthog.models.action.util import format_action_filter from posthog.models.entity import Entity from posthog.models.event.sql import EVENT_JOIN_PERSON_SQL @@ -82,6 +81,10 @@ from posthog.utils import encode_get_request_params, generate_short_id from posthog.queries.person_on_events_v2_sql import PERSON_DISTINCT_ID_OVERRIDES_JOIN_SQL +BREAKDOWN_OTHER_DISPLAY = "Other (i.e. all remaining values)" +BREAKDOWN_NULL_DISPLAY = "None (i.e. no value)" + + BREAKDOWN_OTHER_STRING_LABEL = "$$_posthog_breakdown_other_$$" BREAKDOWN_OTHER_NUMERIC_LABEL = 9007199254740991 # pow(2, 53) - 1, for JS compatibility BREAKDOWN_NULL_STRING_LABEL = "$$_posthog_breakdown_null_$$" diff --git a/posthog/queries/trends/test/test_paging_breakdowns.py b/posthog/queries/trends/test/test_paging_breakdowns.py index 47ea447005c1a..b7bee93f31dc6 100644 --- a/posthog/queries/trends/test/test_paging_breakdowns.py +++ b/posthog/queries/trends/test/test_paging_breakdowns.py @@ -2,12 +2,13 @@ from freezegun import freeze_time -from posthog.hogql_queries.insights.trends.breakdown_values import BREAKDOWN_OTHER_DISPLAY from posthog.models import Filter from posthog.queries.trends.trends import Trends from posthog.test.base import APIBaseTest from posthog.test.test_journeys import journeys_for +BREAKDOWN_OTHER_DISPLAY = "Other (i.e. all remaining values)" + class TestPagingBreakdowns(APIBaseTest): """ diff --git a/posthog/session_recordings/queries/session_recording_list_from_filters.py b/posthog/session_recordings/queries/session_recording_list_from_filters.py index fc0c3781946d5..f6b2652c9b2aa 100644 --- a/posthog/session_recordings/queries/session_recording_list_from_filters.py +++ b/posthog/session_recordings/queries/session_recording_list_from_filters.py @@ -252,7 +252,12 @@ def _where_predicates(self) -> ast.And: return ast.And(exprs=exprs) def _having_predicates(self) -> ast.And | Constant: - exprs: list[ast.Expr] = [] + exprs: list[ast.Expr] = [ + # a missing first url indicate delayed or incomplete ingestion and we can ignore those + ast.CompareOperation( + op=ast.CompareOperationOp.NotEq, left=ast.Field(chain=["first_url"]), right=ast.Constant(value=None) + ) + ] if self._filter.recording_duration_filter: op = ( diff --git a/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py b/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py index 78b204d872314..1e34d247a42d3 100644 --- a/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py +++ b/posthog/session_recordings/queries/session_recording_list_from_replay_summary.py @@ -642,7 +642,7 @@ def ttl_days(self): {provided_session_ids_clause} {log_matching_session_ids_clause} GROUP BY session_id - HAVING 1=1 {duration_clause} {console_log_clause} + HAVING s.first_url is not null {duration_clause} {console_log_clause} {order_by_clause} LIMIT %(limit)s OFFSET %(offset)s """ diff --git a/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_filters.ambr b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_filters.ambr index d620633d8d16f..3dfc09ffd8e85 100644 --- a/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_filters.ambr +++ b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_filters.ambr @@ -24,7 +24,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['custom-event'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -61,7 +61,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['custom-event'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -98,7 +98,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['custom-event'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -135,7 +135,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['custom-event'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -186,7 +186,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person_id, '00000000-0000-0000-0000-000000000000'), 0)))) GROUP BY s.session_id - HAVING ifNull(greaterOrEquals(duration, 60), 0) + HAVING and(isNotNull(first_url), ifNull(greaterOrEquals(duration, 60), 0)) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -223,7 +223,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -260,7 +260,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -297,7 +297,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -334,7 +334,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -371,7 +371,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -408,7 +408,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -440,7 +440,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -472,7 +472,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING ifNull(greaterOrEquals(duration, 60), 0) + HAVING and(isNotNull(first_url), ifNull(greaterOrEquals(duration, 60), 0)) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -504,7 +504,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING ifNull(greaterOrEquals(active_seconds, 60), 0) + HAVING and(isNotNull(first_url), ifNull(greaterOrEquals(active_seconds, 60), 0)) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -536,7 +536,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING ifNull(greaterOrEquals(inactive_seconds, 60), 0) + HAVING and(isNotNull(first_url), ifNull(greaterOrEquals(inactive_seconds, 60), 0)) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -568,7 +568,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY active_seconds DESC LIMIT 4 OFFSET 0 SETTINGS readonly=2, @@ -600,7 +600,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY console_error_count DESC LIMIT 4 OFFSET 0 SETTINGS readonly=2, @@ -632,7 +632,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 4 OFFSET 0 SETTINGS readonly=2, @@ -664,7 +664,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 2 OFFSET 0 SETTINGS readonly=2, @@ -696,7 +696,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 2 OFFSET 1 SETTINGS readonly=2, @@ -728,7 +728,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 2 OFFSET 2 SETTINGS readonly=2, @@ -760,7 +760,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -792,7 +792,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-30 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -824,7 +824,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 12:46:00.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-12 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 12:46:00.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -856,7 +856,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 12:46:00.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 12:46:00.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -888,7 +888,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 12:46:00.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-10 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 12:46:00.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -920,7 +920,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-28 23:59:59.999999', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -952,7 +952,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-29 23:59:59.999999', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -984,7 +984,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING ifNull(greaterOrEquals(duration, 60), 0) + HAVING and(isNotNull(first_url), ifNull(greaterOrEquals(duration, 60), 0)) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1016,7 +1016,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING ifNull(lessOrEquals(duration, 60), 0) + HAVING and(isNotNull(first_url), ifNull(lessOrEquals(duration, 60), 0)) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1053,7 +1053,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1090,7 +1090,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$autocapture'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1127,7 +1127,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1159,7 +1159,7 @@ FROM session_replay_events AS s WHERE and(equals(s.team_id, 2), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-11 13:46:23.000000', 6, 'UTC')), 0), ifNull(greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-25 00:00:00.000000', 6, 'UTC')), 0), ifNull(lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-01 13:46:23.000000', 6, 'UTC')), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1196,7 +1196,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING ifNull(greaterOrEquals(duration, 60), 0) + HAVING and(isNotNull(first_url), ifNull(greaterOrEquals(duration, 60), 0)) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1233,7 +1233,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING ifNull(greaterOrEquals(active_seconds, 60), 0) + HAVING and(isNotNull(first_url), ifNull(greaterOrEquals(active_seconds, 60), 0)) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1275,7 +1275,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1312,7 +1312,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1366,7 +1366,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person.properties___email, 'bla'), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1420,7 +1420,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person.properties___email, 'bla'), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1457,7 +1457,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1511,7 +1511,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person.properties___email, 'bla'), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1565,7 +1565,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person.properties___email, 'bla'), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1617,7 +1617,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1669,7 +1669,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1706,7 +1706,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1743,7 +1743,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1780,7 +1780,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1817,7 +1817,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1854,7 +1854,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1891,7 +1891,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$autocapture'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1928,7 +1928,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -1965,7 +1965,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2002,7 +2002,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['a_different_event'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2039,7 +2039,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['a_different_event'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2076,7 +2076,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2113,7 +2113,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2150,7 +2150,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['a_different_event'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2187,7 +2187,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['a_different_event'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2241,7 +2241,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(notILike(person_distinct_ids__person.properties___email, '%@posthog.com%'), 1)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2278,7 +2278,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2332,7 +2332,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(notILike(person_distinct_ids__person.properties___email, '%@posthog.com%'), 1)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2369,7 +2369,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2406,7 +2406,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageleave', '$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2444,7 +2444,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE and(ifNull(in(console_logs_log_entries.level, ['warn', 'error']), 0), ifNull(greater(positionCaseInsensitive(console_logs_log_entries.message, 'message 4'), 0), 0))))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2482,7 +2482,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE and(ifNull(in(console_logs_log_entries.level, ['warn', 'error']), 0), ifNull(greater(positionCaseInsensitive(console_logs_log_entries.message, 'message 5'), 0), 0))))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2520,7 +2520,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE and(ifNull(in(console_logs_log_entries.level, ['warn', 'error']), 0), ifNull(greater(positionCaseInsensitive(console_logs_log_entries.message, 'MESSAGE 5'), 0), 0))))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2558,7 +2558,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE and(ifNull(in(console_logs_log_entries.level, ['info']), 0), ifNull(greater(positionCaseInsensitive(console_logs_log_entries.message, 'message 5'), 0), 0))))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2596,7 +2596,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE ifNull(in(console_logs_log_entries.level, ['error']), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2634,7 +2634,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE ifNull(in(console_logs_log_entries.level, ['info']), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2672,7 +2672,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE ifNull(in(console_logs_log_entries.level, ['info']), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2710,7 +2710,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE ifNull(in(console_logs_log_entries.level, ['warn']), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2748,7 +2748,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE ifNull(in(console_logs_log_entries.level, ['warn']), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2786,7 +2786,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE ifNull(in(console_logs_log_entries.level, ['info']), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2824,7 +2824,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE ifNull(in(console_logs_log_entries.level, ['warn', 'error']), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2862,7 +2862,7 @@ WHERE and(equals(log_entries.team_id, 2), equals(log_entries.log_source, 'session_replay'))) AS console_logs_log_entries WHERE ifNull(in(console_logs_log_entries.level, ['info']), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2899,7 +2899,7 @@ in(s.session_id, ['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001' /* ... */]) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2936,7 +2936,7 @@ in(s.session_id, ['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001' /* ... */]) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -2998,7 +2998,7 @@ FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0)))), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3065,7 +3065,7 @@ FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0)))), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3112,7 +3112,7 @@ FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0)))), 0)) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3161,7 +3161,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person.properties___email, 'bla@gmail.com'), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3210,7 +3210,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(notILike(person_distinct_ids__person.properties___email, '%gmail.com%'), 1)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3247,7 +3247,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview', 'new-event'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3284,7 +3284,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview', 'new-event2'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3330,7 +3330,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person_id, '00000000-0000-0000-0000-000000000000'), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3367,7 +3367,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3404,7 +3404,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3441,7 +3441,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3478,7 +3478,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3515,7 +3515,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3552,7 +3552,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3589,7 +3589,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3626,7 +3626,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3663,7 +3663,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3700,7 +3700,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3737,7 +3737,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3774,7 +3774,7 @@ GROUP BY events.`$session_id` HAVING true))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3811,7 +3811,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3860,7 +3860,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person.properties___email, 'bla'), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3897,7 +3897,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3946,7 +3946,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person.properties___email, 'bla'), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -3983,7 +3983,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -4032,7 +4032,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person.properties___email, 'bla'), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -4069,7 +4069,7 @@ GROUP BY events.`$session_id` HAVING hasAll(groupUniqArray(events.event), ['$pageview'])))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, @@ -4118,7 +4118,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(person.created_at, person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS person_distinct_ids__person ON equals(person_distinct_ids.person_distinct_ids___person_id, person_distinct_ids__person.id) WHERE ifNull(equals(person_distinct_ids__person.properties___email, 'bla'), 0)))) GROUP BY s.session_id - HAVING true + HAVING isNotNull(first_url) ORDER BY start_time DESC LIMIT 51 OFFSET 0 SETTINGS readonly=2, diff --git a/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr index 45277b504d4b8..58c6d9940430b 100644 --- a/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr +++ b/posthog/session_recordings/queries/test/__snapshots__/test_session_recording_list_from_session_replay.ambr @@ -41,7 +41,7 @@ HAVING 1=1 AND hasAll(event_names, ['custom-event'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -88,7 +88,7 @@ HAVING 1=1 AND hasAll(event_names, ['custom-event'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -136,7 +136,7 @@ HAVING 1=1 AND hasAll(event_names, ['custom-event'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -184,7 +184,7 @@ HAVING 1=1 AND hasAll(event_names, ['custom-event'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -260,7 +260,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview', 'custom-event'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND duration > 60 ORDER BY start_time DESC LIMIT 51 @@ -304,7 +304,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -348,7 +348,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -392,7 +392,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -435,7 +435,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -479,7 +479,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -523,7 +523,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -553,7 +553,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -583,7 +583,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND duration > 60 ORDER BY start_time DESC LIMIT 51 @@ -614,7 +614,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND active_seconds > 60 ORDER BY start_time DESC LIMIT 51 @@ -645,7 +645,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND inactive_seconds > 60 ORDER BY start_time DESC LIMIT 51 @@ -676,7 +676,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY active_seconds DESC LIMIT 4 OFFSET 0 @@ -706,7 +706,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY console_error_count DESC LIMIT 4 OFFSET 0 @@ -736,7 +736,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 4 OFFSET 0 @@ -766,7 +766,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 2 OFFSET 0 @@ -796,7 +796,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 2 OFFSET 1 @@ -826,7 +826,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 2 OFFSET 2 @@ -856,7 +856,7 @@ AND s.min_first_timestamp >= '2021-01-01 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -886,7 +886,7 @@ AND s.min_first_timestamp >= '2020-12-30 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -916,7 +916,7 @@ AND s.min_first_timestamp >= '2020-12-12 00:00:00' AND s.min_first_timestamp <= '2021-01-01 12:46:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -946,7 +946,7 @@ AND s.min_first_timestamp >= '2020-12-11 00:00:00' AND s.min_first_timestamp <= '2021-01-01 12:46:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -976,7 +976,7 @@ AND s.min_first_timestamp >= '2020-12-10 00:00:00' AND s.min_first_timestamp <= '2021-01-01 12:46:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1006,7 +1006,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2020-12-28 23:59:59' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1036,7 +1036,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2020-12-29 23:59:59' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1066,7 +1066,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND duration > 60 ORDER BY start_time DESC LIMIT 51 @@ -1097,7 +1097,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND duration < 60 ORDER BY start_time DESC LIMIT 51 @@ -1143,7 +1143,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1188,7 +1188,7 @@ HAVING 1=1 AND hasAll(event_names, ['$autocapture'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1233,7 +1233,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1263,7 +1263,7 @@ AND s.min_first_timestamp >= '2020-12-25 00:00:00' AND s.min_first_timestamp <= '2021-01-01 13:46:23' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1308,7 +1308,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND duration > 60 ORDER BY start_time DESC LIMIT 51 @@ -1354,7 +1354,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND active_seconds > 60 ORDER BY start_time DESC LIMIT 51 @@ -1408,7 +1408,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1453,7 +1453,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1534,7 +1534,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1613,7 +1613,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1658,7 +1658,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1737,7 +1737,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1814,7 +1814,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1892,7 +1892,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -1970,7 +1970,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2016,7 +2016,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2062,7 +2062,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2108,7 +2108,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2154,7 +2154,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2199,7 +2199,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2244,7 +2244,7 @@ HAVING 1=1 AND hasAll(event_names, ['$autocapture'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2290,7 +2290,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2336,7 +2336,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2382,7 +2382,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2428,7 +2428,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2517,7 +2517,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2562,7 +2562,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2651,7 +2651,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2696,7 +2696,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2742,7 +2742,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview', '$pageleave'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -2782,7 +2782,7 @@ AND level in ['warn', 'error'] AND positionCaseInsensitive(message, 'message 4') > 0 ) as log_text_matching GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_warn_count > 0 OR console_error_count > 0) ORDER BY start_time DESC @@ -2824,7 +2824,7 @@ AND level in ['warn', 'error'] AND positionCaseInsensitive(message, 'message 5') > 0 ) as log_text_matching GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_warn_count > 0 OR console_error_count > 0) ORDER BY start_time DESC @@ -2866,7 +2866,7 @@ AND level in ['warn', 'error'] AND positionCaseInsensitive(message, 'MESSAGE 5') > 0 ) as log_text_matching GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_warn_count > 0 OR console_error_count > 0) ORDER BY start_time DESC @@ -2908,7 +2908,7 @@ AND level in ['info'] AND positionCaseInsensitive(message, 'message 5') > 0 ) as log_text_matching GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_log_count > 0) ORDER BY start_time DESC LIMIT 51 @@ -2939,7 +2939,7 @@ AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.min_first_timestamp <= '2021-01-21 20:00:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_error_count > 0) ORDER BY start_time DESC LIMIT 51 @@ -2970,7 +2970,7 @@ AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.min_first_timestamp <= '2021-01-21 20:00:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_log_count > 0) ORDER BY start_time DESC LIMIT 51 @@ -3001,7 +3001,7 @@ AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.min_first_timestamp <= '2021-01-21 20:00:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_log_count > 0) ORDER BY start_time DESC LIMIT 51 @@ -3032,7 +3032,7 @@ AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.min_first_timestamp <= '2021-01-21 20:00:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_warn_count > 0) ORDER BY start_time DESC LIMIT 51 @@ -3063,7 +3063,7 @@ AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.min_first_timestamp <= '2021-01-21 20:00:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_warn_count > 0) ORDER BY start_time DESC LIMIT 51 @@ -3094,7 +3094,7 @@ AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.min_first_timestamp <= '2021-01-21 20:00:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_log_count > 0) ORDER BY start_time DESC LIMIT 51 @@ -3125,7 +3125,7 @@ AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.min_first_timestamp <= '2021-01-21 20:00:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_warn_count > 0 OR console_error_count > 0) ORDER BY start_time DESC @@ -3157,7 +3157,7 @@ AND s.min_first_timestamp >= '2021-01-14 00:00:00' AND s.min_first_timestamp <= '2021-01-21 20:00:00' GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null AND (console_log_count > 0) ORDER BY start_time DESC LIMIT 51 @@ -3229,7 +3229,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3336,7 +3336,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3423,7 +3423,7 @@ HAVING 1=1 AND hasAll(event_names, ['custom_event'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3474,7 +3474,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3525,7 +3525,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3571,7 +3571,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview', 'new-event'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3617,7 +3617,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview', 'new-event2'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3662,7 +3662,7 @@ HAVING argMax(is_deleted, version) = 0 AND current_person_id = '00000000-0000-0000-0000-000000000000') as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3707,7 +3707,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3750,7 +3750,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3795,7 +3795,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3838,7 +3838,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3883,7 +3883,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3926,7 +3926,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -3971,7 +3971,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4014,7 +4014,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4059,7 +4059,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4102,7 +4102,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4147,7 +4147,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4190,7 +4190,7 @@ GROUP BY `$session_id` HAVING 1=1) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4235,7 +4235,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4283,7 +4283,7 @@ HAVING argMax(is_deleted, version) = 0 AND (ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'email'), ''), 'null'), '^"|"$', ''), 'bla'), 0))) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4328,7 +4328,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4375,7 +4375,7 @@ HAVING argMax(is_deleted, version) = 0 AND (ifNull(equals(nullIf(nullIf(pmat_email, ''), 'null'), 'bla'), 0))) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4420,7 +4420,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4471,7 +4471,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4516,7 +4516,7 @@ HAVING 1=1 AND hasAll(event_names, ['$pageview'])) as session_events_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 @@ -4567,7 +4567,7 @@ GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0) as session_persons_sub_query) GROUP BY session_id - HAVING 1=1 + HAVING s.first_url is not null ORDER BY start_time DESC LIMIT 51 OFFSET 0 diff --git a/posthog/session_recordings/queries/test/session_replay_sql.py b/posthog/session_recordings/queries/test/session_replay_sql.py index d2d0edb16c35a..8be7d3b88b1da 100644 --- a/posthog/session_recordings/queries/test/session_replay_sql.py +++ b/posthog/session_recordings/queries/test/session_replay_sql.py @@ -105,7 +105,7 @@ def produce_replay_summary( distinct_id: Optional[str] = None, first_timestamp: Optional[str | datetime] = None, last_timestamp: Optional[str | datetime] = None, - first_url: Optional[str | None] = None, + first_url: Optional[str | None] = "https://not-provided-by-test.com", click_count: Optional[int] = None, keypress_count: Optional[int] = None, mouse_activity_count: Optional[int] = None, diff --git a/posthog/session_recordings/queries/test/test_session_recording_list_from_filters.py b/posthog/session_recordings/queries/test/test_session_recording_list_from_filters.py index 9f09d82f8f034..2cc1aa2e4fae4 100644 --- a/posthog/session_recordings/queries/test/test_session_recording_list_from_filters.py +++ b/posthog/session_recordings/queries/test/test_session_recording_list_from_filters.py @@ -132,7 +132,7 @@ def test_basic_query(self): first_timestamp=(self.an_hour_ago + relativedelta(seconds=20)), last_timestamp=(self.an_hour_ago + relativedelta(seconds=2000)), distinct_id=user, - first_url=None, + first_url="https://another-url.com", click_count=2, keypress_count=2, mouse_activity_count=2, @@ -154,7 +154,7 @@ def test_basic_query(self): "inactive_seconds": 1188.0, "start_time": self.an_hour_ago + relativedelta(seconds=20), "end_time": self.an_hour_ago + relativedelta(seconds=2000), - "first_url": None, + "first_url": "https://another-url.com", "console_log_count": 0, "console_warn_count": 0, "console_error_count": 0, @@ -313,7 +313,7 @@ def test_basic_query_with_paging(self): first_timestamp=(self.an_hour_ago + relativedelta(seconds=20)), last_timestamp=(self.an_hour_ago + relativedelta(seconds=2000)), distinct_id=user, - first_url=None, + first_url="https://another-url.com", click_count=2, keypress_count=2, mouse_activity_count=2, @@ -337,7 +337,7 @@ def test_basic_query_with_paging(self): "inactive_seconds": 1188.0, "start_time": self.an_hour_ago + relativedelta(seconds=20), "end_time": self.an_hour_ago + relativedelta(seconds=2000), - "first_url": None, + "first_url": "https://another-url.com", "console_log_count": 0, "console_warn_count": 0, "console_error_count": 0, @@ -563,17 +563,17 @@ def test_first_url_selection(self): "session_id": session_id_two, "first_url": "https://first-is-on-second-event.com", }, - { - "session_id": session_id_three, - "first_url": None, - }, + # sessions without urls are not included + # { + # "session_id": session_id_three, + # "first_url": None, + # }, { "session_id": session_id_four, "first_url": "https://on-second-received-event-but-actually-first.com", }, ], - # mypy unhappy about this lambda 🤷️ - key=lambda x: x["session_id"], # type: ignore + key=lambda x: x["session_id"], ) def test_recordings_dont_leak_data_between_teams(self): @@ -591,7 +591,6 @@ def test_recordings_dont_leak_data_between_teams(self): distinct_id=user, first_timestamp=self.an_hour_ago, last_timestamp=self.an_hour_ago + relativedelta(seconds=20), - first_url=None, click_count=2, keypress_count=2, mouse_activity_count=2, @@ -604,7 +603,6 @@ def test_recordings_dont_leak_data_between_teams(self): distinct_id=user, first_timestamp=self.an_hour_ago, last_timestamp=self.an_hour_ago + relativedelta(seconds=20), - first_url=None, click_count=2, keypress_count=2, mouse_activity_count=2, @@ -1230,6 +1228,7 @@ def test_all_sessions_recording_object_keys_with_entity_filter(self): first_timestamp=self.an_hour_ago, last_timestamp=(self.an_hour_ago + relativedelta(seconds=60)), team_id=self.team.id, + first_url="https://recieved-out-of-order.com/second", ) self.create_event( user, @@ -1242,6 +1241,7 @@ def test_all_sessions_recording_object_keys_with_entity_filter(self): first_timestamp=self.an_hour_ago, last_timestamp=(self.an_hour_ago + relativedelta(seconds=30)), team_id=self.team.id, + first_url="https://recieved-out-of-order.com/first", ) (session_recordings, _, _) = self._filter_recordings_by( @@ -1266,7 +1266,7 @@ def test_all_sessions_recording_object_keys_with_entity_filter(self): "end_time": self.an_hour_ago + relativedelta(seconds=60), "active_seconds": 0.0, "click_count": 0, - "first_url": None, + "first_url": "https://recieved-out-of-order.com/first", "inactive_seconds": 60.0, "keypress_count": 0, "mouse_activity_count": 0, diff --git a/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py b/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py index 7bfbd6fcb3f67..09b1c68cfe122 100644 --- a/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py +++ b/posthog/session_recordings/queries/test/test_session_recording_list_from_session_replay.py @@ -1151,7 +1151,7 @@ def test_all_sessions_recording_object_keys_with_entity_filter(self): "end_time": self.an_hour_ago + relativedelta(seconds=60), "active_seconds": 0.0, "click_count": 0, - "first_url": None, + "first_url": "https://not-provided-by-test.com", "inactive_seconds": 60.0, "keypress_count": 0, "mouse_activity_count": 0, diff --git a/posthog/session_recordings/test/test_session_recordings.py b/posthog/session_recordings/test/test_session_recordings.py index 401eaa01365ed..eee3c288c3be9 100644 --- a/posthog/session_recordings/test/test_session_recordings.py +++ b/posthog/session_recordings/test/test_session_recordings.py @@ -254,7 +254,7 @@ def test_session_recordings_dont_leak_teams(self) -> None: "recording_duration": ANY, "snapshot_source": "web", "start_time": ANY, - "start_url": None, + "start_url": "https://not-provided-by-test.com", "storage": "object_storage", "viewed": False, }, @@ -399,7 +399,7 @@ def test_get_single_session_recording_metadata(self): "end_time": (base_time + relativedelta(seconds=30)).strftime("%Y-%m-%dT%H:%M:%SZ"), "click_count": 0, "keypress_count": 0, - "start_url": None, + "start_url": "https://not-provided-by-test.com", "mouse_activity_count": 0, "inactive_seconds": 30, "active_seconds": 0, diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py index 75e5b9c3ea93b..140fa36dd44f9 100644 --- a/posthog/settings/temporal.py +++ b/posthog/settings/temporal.py @@ -27,6 +27,7 @@ ) CLICKHOUSE_MAX_EXECUTION_TIME: int = get_from_env("CLICKHOUSE_MAX_EXECUTION_TIME", 0, type_cast=int) +CLICKHOUSE_MAX_MEMORY_USAGE: int = get_from_env("CLICKHOUSE_MAX_MEMORY_USAGE", 100 * 1000 * 1000 * 1000, type_cast=int) CLICKHOUSE_MAX_BLOCK_SIZE_DEFAULT: int = get_from_env("CLICKHOUSE_MAX_BLOCK_SIZE_DEFAULT", 10000, type_cast=int) # Comma separated list of overrides in the format "team_id:block_size" CLICKHOUSE_MAX_BLOCK_SIZE_OVERRIDES: dict[int, int] = dict( diff --git a/posthog/tasks/tasks.py b/posthog/tasks/tasks.py index 351f99878fcff..a9cf873fdbd9b 100644 --- a/posthog/tasks/tasks.py +++ b/posthog/tasks/tasks.py @@ -47,6 +47,7 @@ def redis_heartbeat() -> None: retry_backoff_max=10, max_retries=3, expires=60 * 10, # Do not run queries that got stuck for more than this + reject_on_worker_lost=True, ) @limit_concurrency(90) # Do not go above what CH can handle (max_concurrent_queries) @limit_concurrency( @@ -508,7 +509,7 @@ def monitoring_check_clickhouse_schema_drift() -> None: check_clickhouse_schema_drift() -@shared_task(ignore_result=True, queue=CeleryQueue.LONG_RUNNING) +@shared_task(ignore_result=True, queue=CeleryQueue.LONG_RUNNING.value) def calculate_cohort() -> None: from posthog.tasks.calculate_cohort import calculate_cohorts @@ -631,7 +632,7 @@ def schedule_cache_updates_task() -> None: retry_backoff_max=30, max_retries=3, retry_jitter=True, - queue=CeleryQueue.LONG_RUNNING, + queue=CeleryQueue.LONG_RUNNING.value, ) def update_cache_task(caching_state_id: UUID) -> None: from posthog.caching.insight_cache import update_cache diff --git a/posthog/temporal/common/clickhouse.py b/posthog/temporal/common/clickhouse.py index 2640bf95c1f97..ad8bfe8173e82 100644 --- a/posthog/temporal/common/clickhouse.py +++ b/posthog/temporal/common/clickhouse.py @@ -417,6 +417,7 @@ async def get_client( password=settings.CLICKHOUSE_PASSWORD, database=settings.CLICKHOUSE_DATABASE, max_execution_time=settings.CLICKHOUSE_MAX_EXECUTION_TIME, + max_memory_usage=settings.CLICKHOUSE_MAX_MEMORY_USAGE, max_block_size=max_block_size, output_format_arrow_string_as_string="true", **kwargs,