Merge branch 'master' into stateless_dag

* master: (103 commits)
  feat(postgres-estimated-rows): pg Estimated Rows on Data Warehouse Sync (#27634)
  fix: revert darkmode class toggle, updated content on fills (#27783)
  chore: upgrade posthog-js (#27790)
  chore(editor-3001): add back join actions (#27740)
  feat: Add person distinct ID overrides squash job (as dagster job) (#27710)
  fix(created-by-sources): Adding `created_by` to sources (#27751)
  Revert "feat(data-warehouse): V2 pipeline release " (#27791)
  fix: typo for feature flags (#27786)
  fix(defer-unmounting): Defer unmounting of react elements (#27742)
  feat(data-warehouse): V2 pipeline release (#27732)
  fix(data-warehouse): Ensure dates are actual datetime formats (#27777)
  fix: enable hot reload for the products dir (#27746)
  fix: assignee selector when null (#27737)
  chore: clarify rrweb imports (#27776)
  chore(deps): Update posthog-js to 1.207.3 (#27779)
  feat(retention): filters on start/return event (#27770)
  fix(experiments): only show supported math functions (#27589)
  feat(web-analytics): Set unique conversions graph when adding conversions goal (#27774)
  chore: color design system part 1: banner and accents (#27756)
  chore(experiments): Add tests for funnel attribution options (#27752)
  ...

.dockerignore

@@ -17,6 +17,7 @@
 !.kearc
 !.storybook
 !tsconfig.json
+!products
 !frontend/@posthog
 !frontend/src
 !frontend/types
@@ -32,10 +33,10 @@
 !plugin-server/.eslintrc.js
 !plugin-server/.prettierrc
 !share/GeoLite2-City.mmdb
-!hogvm/python
+!common/hogvm/python
 !unit.json.tpl
-!plugin-transpiler/src
-!plugin-transpiler/*.*
+!common/plugin_transpiler/src
+!common/plugin_transpiler/*.*
 !test-runner-jest.config.js
 !test-runner-jest-environment.js
 !patches

.eslintignore

@@ -3,3 +3,8 @@ jest.config.ts
 node_modules
 rust
 livestream
+common/hogvm/__tests__/__snapshots__
+common/hogvm/typescript/dist
+common/hogvm/typescript/jest.config.js
+common/plugin_transpiler/dist
+common/plugin_transpiler/build.mjs

.github/actions/run-backend-tests/action.yml

@@ -63,7 +63,7 @@ runs:
           id: hogql-parser-diff
           run: |
               git fetch --no-tags --prune --depth=1 origin master
-              changed=$(git diff --quiet HEAD origin/master -- hogql_parser/ && echo "false" || echo "true")
+              changed=$(git diff --quiet HEAD origin/master -- common/hogql_parser/ && echo "false" || echo "true")
               echo "changed=$changed" >> $GITHUB_OUTPUT
 
         - name: Install SAML (python3-saml) dependencies
@@ -86,10 +86,10 @@ runs:
         # absolute wild tbh https://stackoverflow.com/a/75503402
         - uses: tlambert03/setup-qt-libs@v1
 
-        - name: Install plugin-transpiler
+        - name: Install plugin_transpiler
           shell: bash
           run: |
-              cd plugin-transpiler
+              cd common/plugin_transpiler
               pnpm install
               pnpm run build
 
@@ -122,7 +122,7 @@ runs:
               sudo cp out/usr/local/lib/libantlr4-runtime.so* /usr/lib/
               sudo ldconfig
               cd ..
-              uv pip install --system ./hogql_parser
+              uv pip install --system ./common/hogql_parser
 
         - name: Set up needed files
           shell: bash
@@ -164,7 +164,7 @@ runs:
                   || 'posthog'
               }} ${{ inputs.person-on-events == 'true' && 'ee/clickhouse/' || 'ee/' }} -m "not async_migrations" \
                   --ignore=posthog/temporal \
-                  --ignore=hogvm/python/test \
+                  --ignore=common/hogvm/python/test \
                   --splits ${{ inputs.concurrency }} --group ${{ inputs.group }} \
                   --durations=100 --durations-min=1.0 --store-durations \
                   $PYTEST_ARGS

.github/pr-deploy/hobby.yaml.tmpl

@@ -20,8 +20,8 @@ spec:
           privileged: true
         resources:
           requests:
-            cpu: 2
-            memory: 500M          
+            cpu: 4
+            memory: 4G          
         ports:
         - containerPort: 2375
         - containerPort: 80

.github/workflows/build-hogql-parser.yml

@@ -3,7 +3,7 @@ name: Release hogql-parser
 on:
     pull_request:
         paths:
-            - hogql_parser/**
+            - common/hogql_parser/**
             - .github/workflows/build-hogql-parser.yml
 
 concurrency:
@@ -21,14 +21,14 @@ jobs:
               with:
                   fetch-depth: 0 # Fetching all for comparison since last push (not just last commit)
 
-            - name: Check if hogql_parser/ has changed
+            - name: Check if common/hogql_parser/ has changed
               id: changed-files
               uses: tj-actions/changed-files@v43
               with:
                   since_last_remote_commit: true
                   files_yaml: |
                       parser:
-                      - hogql_parser/**
+                      - common/hogql_parser/**
 
             - name: Check if version was bumped
               shell: bash
@@ -37,7 +37,7 @@ jobs:
                   parser_release_needed='false'
                   if [[ ${{ steps.changed-files.outputs.parser_any_changed }} == 'true' ]]; then
                     published=$(curl -fSsl https://pypi.org/pypi/hogql-parser/json | jq -r '.info.version')
-                    local=$(python hogql_parser/setup.py --version)
+                    local=$(python common/hogql_parser/setup.py --version)
                     if [[ "$published" != "$local" ]]; then
                       parser_release_needed='true'
                     else
@@ -59,7 +59,7 @@ jobs:
                 # As of October 2023, GitHub doesn't have ARM Actions runners… and ARM emulation is insanely slow
                 # (20x longer) on the Linux runners (while being reasonable on the macOS runners). Hence, we use
                 # BuildJet as a provider of ARM runners - this solution saves a lot of time and consequently some money.
-                os: [ubuntu-22.04, buildjet-2vcpu-ubuntu-2204-arm, macos-12]
+                os: [ubuntu-22.04, buildjet-2vcpu-ubuntu-2204-arm, macos-13]
 
         steps:
             - uses: actions/checkout@v4
@@ -86,21 +86,23 @@ jobs:
 
             - name: Build sdist
               if: matrix.os == 'ubuntu-22.04' # Only build the sdist once
-              run: cd hogql_parser && python setup.py sdist
+              run: cd common/hogql_parser && python setup.py sdist
 
             - name: Install cibuildwheel
               run: pip install cibuildwheel==2.16.*
 
             - name: Build wheels
-              run: cd hogql_parser && python -m cibuildwheel --output-dir dist
+              run: cd common/hogql_parser && python -m cibuildwheel --output-dir dist
               env:
-                  MACOSX_DEPLOYMENT_TARGET: '12' # A modern target allows us to use C++20
+                  MACOSX_DEPLOYMENT_TARGET: '13' # A modern target allows us to use C++20
 
-            - uses: actions/upload-artifact@v4
+            - name: Upload wheels artifact
+              uses: actions/upload-artifact@v4
               with:
+                  name: wheels-${{ matrix.os }}
                   path: |
-                      hogql_parser/dist/*.whl
-                      hogql_parser/dist/*.tar.gz
+                      common/hogql_parser/dist/*.whl
+                      common/hogql_parser/dist/*.tar.gz
                   if-no-files-found: error
 
     publish:
@@ -111,11 +113,23 @@ jobs:
             id-token: write
         runs-on: ubuntu-22.04
         steps:
-            - name: Fetch wheels
+            - name: Download wheels from ubuntu-22.04
               uses: actions/download-artifact@v4
               with:
-                  name: artifact
-                  path: dist/
+                  name: wheels-ubuntu-22.04
+                  path: dist
+
+            - name: Download wheels from buildjet-2vcpu-ubuntu-2204-arm
+              uses: actions/download-artifact@v4
+              with:
+                  name: wheels-buildjet-2vcpu-ubuntu-2204-arm
+                  path: dist
+
+            - name: Download wheels from macos-13
+              uses: actions/download-artifact@v4
+              with:
+                  name: wheels-macos-13
+                  path: dist
 
             - name: Publish package to PyPI
               uses: pypa/gh-action-pypi-publish@release/v1
@@ -128,7 +142,7 @@ jobs:
             - name: Update hogql-parser in requirements
               shell: bash
               run: |
-                  local=$(python hogql_parser/setup.py --version)
+                  local=$(python common/hogql_parser/setup.py --version)
                   sed -i "s/hogql-parser==.*/hogql-parser==${local}/g" requirements.in
                   sed -i "s/hogql-parser==.*/hogql-parser==${local}/g" requirements.txt
 

.github/workflows/ci-backend.yml

@@ -68,15 +68,15 @@ jobs:
                         # including the negated rule appears to work
                         # but makes it always match because the checked file always isn't `ee/frontend/**` 🙈
                         - 'ee/**/*'
-                        - 'hogvm/**/*'
+                        - 'common/hogvm/**/*'
                         - 'posthog/**/*'
                         - 'bin/*.py'
                         - requirements.txt
                         - requirements-dev.txt
                         - mypy.ini
                         - pytest.ini
                         - frontend/src/queries/schema.json # Used for generating schema.py
-                        - plugin-transpiler/src # Used for transpiling plugins
+                        - common/plugin_transpiler/src # Used for transpiling plugins
                         # Make sure we run if someone is explicitly change the workflow
                         - .github/workflows/ci-backend.yml
                         - .github/actions/run-backend-tests/action.yml
@@ -316,7 +316,7 @@ jobs:
               uses: actions/upload-artifact@v4
               if: needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' && matrix.person-on-events == false
               with:
-                  name: email_renders
+                  name: email_renders-${{ matrix.segment }}-${{ matrix.person-on-events }}
                   path: posthog/tasks/test/__emails__
                   retention-days: 5
 

.github/workflows/ci-e2e.yml

@@ -242,29 +242,29 @@ jobs:
             - name: Archive test screenshots
               uses: actions/upload-artifact@v4
               with:
-                  name: screenshots
+                  name: screenshots-${{ strategy.job-index }}
                   path: cypress/screenshots
               if: ${{ failure() }}
 
             - name: Archive test downloads
               uses: actions/upload-artifact@v4
               with:
-                  name: downloads
+                  name: downloads-${{ strategy.job-index }}
                   path: cypress/downloads
               if: ${{ failure() }}
 
             - name: Archive test videos
               uses: actions/upload-artifact@v4
               with:
-                  name: videos
+                  name: videos-${{ strategy.job-index }}
                   path: cypress/videos
               if: ${{ failure() }}
 
             - name: Archive accessibility violations
               if: needs.changes.outputs.shouldTriggerCypress == 'true'
               uses: actions/upload-artifact@v4
               with:
-                  name: accessibility-violations
+                  name: accessibility-violations-${{ strategy.job-index }}
                   path: '**/a11y/'
                   if-no-files-found: 'ignore'
 

.github/workflows/ci-hog.yml

@@ -37,7 +37,7 @@ jobs:
                   filters: |
                       hog:
                         # Avoid running tests for irrelevant changes
-                        - 'hogvm/**/*'
+                        - 'common/hogvm/**/*'
                         - 'posthog/hogql/**/*'
                         - 'bin/hog'
                         - 'bin/hoge'
@@ -111,27 +111,27 @@ jobs:
                   npm run grammar:build && git diff --exit-code
               env:
                   # Installing a version of ANTLR compatible with what's in Homebrew as of August 2024 (version 4.13.2),
-                  # as apt-get is quite out of date. The same version must be set in hogql_parser/pyproject.toml
+                  # as apt-get is quite out of date. The same version must be set in common/hogql_parser/pyproject.toml
                   ANTLR_VERSION: '4.13.2'
 
             - name: Check if STL bytecode is up to date
               run: |
-                  python -m hogvm.stl.compile
+                  python -m common.hogvm.stl.compile
                   git diff --exit-code
 
             - name: Run HogVM Python tests
               run: |
-                  pytest hogvm
+                  pytest common/hogvm
 
             - name: Run HogVM TypeScript tests
               run: |
-                  cd hogvm/typescript
+                  cd common/hogvm/typescript
                   pnpm install --frozen-lockfile
                   pnpm run test
 
             - name: Run Hog tests
               run: |
-                  cd hogvm/typescript
+                  cd common/hogvm/typescript
                   pnpm run build 
                   cd ../
                   ./test.sh && git diff --exit-code
@@ -152,7 +152,7 @@ jobs:
               id: check-package-version
               uses: PostHog/check-package-version@v2
               with:
-                  path: hogvm/typescript
+                  path: common/hogvm/typescript
 
     release-hogvm:
         name: Release new HogVM TypeScript version
@@ -191,9 +191,9 @@ jobs:
                   node-version: 18
                   registry-url: https://registry.npmjs.org
             - name: Install package.json dependencies
-              run: cd hogvm/typescript && pnpm install
+              run: cd common/hogvm/typescript && pnpm install
             - name: Publish the package in the npm registry
-              run: cd hogvm/typescript && npm publish --access public
+              run: cd common/hogvm/typescript && npm publish --access public
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
             - name: Sleep 60 seconds to allow npm to update the package
@@ -225,7 +225,7 @@ jobs:
               id: check-mismatch
               run: |
                   # Extract committed version
-                  HOGVM_VERSION=$(jq -r '.version' hogvm/typescript/package.json)
+                  HOGVM_VERSION=$(jq -r '.version' common/hogvm/typescript/package.json)
 
                   # Compare dependencies in package.json
                   MAIN_VERSION=$(jq -r '.dependencies."@posthog/hogvm"' package.json | tr -d '^')
@@ -245,7 +245,7 @@ jobs:
             - name: Update package.json versions
               if: env.mismatch == 'true'
               run: |
-                  VERSION=$(jq ".version" hogvm/typescript/package.json -r)
+                  VERSION=$(jq ".version" common/hogvm/typescript/package.json -r)
 
                   retry_pnpm_install() {
                       local retries=0
@@ -283,7 +283,7 @@ jobs:
               if: env.mismatch == 'true'
               uses: EndBug/add-and-commit@v9
               with:
-                  add: '["package.json", "pnpm-lock.yaml", "plugin-server/package.json", "plugin-server/pnpm-lock.yaml", "hogvm/typescript/package.json"]'
+                  add: '["package.json", "pnpm-lock.yaml", "plugin-server/package.json", "plugin-server/pnpm-lock.yaml", "common/hogvm/typescript/package.json"]'
                   message: 'Update @posthog/hogvm version in package.json'
                   default_author: github_actions
                   github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

.github/workflows/ci-plugin-server.yml

@@ -188,7 +188,7 @@ jobs:
                   DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/test_posthog'
                   REDIS_URL: 'redis://localhost'
                   NODE_OPTIONS: '--max_old_space_size=4096'
-              run: cd plugin-server && pnpm test -- --runInBand --forceExit tests/ --shard=${{matrix.shard}}
+              run: cd plugin-server && pnpm test -- --runInBand --forceExit --shard=${{matrix.shard}}
 
     functional-tests:
         name: Functional tests

.github/workflows/rust-docker-build.yml

@@ -33,6 +33,8 @@ jobs:
                       dockerfile: ./rust/Dockerfile
                     - image: cymbal
                       dockerfile: ./rust/Dockerfile
+                    - image: batch-import-worker
+                      dockerfile: ./rust/Dockerfile
         runs-on: depot-ubuntu-22.04-4
         permissions:
             id-token: write # allow issuing OIDC tokens for this workflow run
@@ -44,6 +46,7 @@ jobs:
             cyclotron-fetch_digest: ${{ steps.digest.outputs.cyclotron-fetch_digest }}
             cyclotron-janitor_digest: ${{ steps.digest.outputs.cyclotron-janitor_digest }}
             property-defs-rs_digest: ${{ steps.digest.outputs.property-defs-rs_digest }}
+            batch-import-worker_digest: ${{ steps.digest.outputs.batch-import-worker_digest }}
             hook-api_digest: ${{ steps.digest.outputs.hook-api_digest }}
             hook-janitor_digest: ${{ steps.digest.outputs.hook-janitor_digest }}
             hook-worker_digest: ${{ steps.digest.outputs.hook-worker_digest }}
@@ -142,6 +145,10 @@ jobs:
                       values:
                           image:
                               sha: '${{ needs.build.outputs.property-defs-rs_digest }}'
+                    - release: batch-import-worker
+                      values:
+                          image:
+                              sha: '${{ needs.build.outputs.batch-import-worker_digest }}'
                     - release: cymbal
                       values:
                           image:

.gitignore

@@ -56,10 +56,10 @@ gen/
 # Local files generated for the ANTLR VS Code extension (https://github.com/mike-lischke/vscode-antlr4)
 .antlr
 upgrade/
-hogvm/typescript/dist
-hogvm/typescript/.parcel-cache
+common/hogvm/typescript/dist
+common/hogvm/typescript/.parcel-cache
 .wokeignore
-plugin-transpiler/dist
+common/plugin_transpiler/dist
 *-esbuild-meta.json
 *-esbuild-bundle-visualization.html
 .dlt