Merge pull request #2 from PostHog/master

Pull from master

.dockerignore

@@ -18,3 +18,4 @@
 !frontend/src
 !frontend/types
 !frontend/public
+!ee

.github/workflows/e2e.yml

@@ -76,7 +76,7 @@ jobs:
             - name: Yarn install deps
               run: |
                   yarn install --frozen-lockfile
-                  yarn add [email protected]
+                  yarn add [email protected] [email protected]
             - uses: actions/cache@v1
               name: Setup Yarn build cache
               id: yarn-build-cache

.github/workflows/prod-container.yml

@@ -0,0 +1,59 @@
+name: Build & Deploy Production Containers
+
+on:
+    push:
+        branches: master
+
+jobs:
+    build:
+        name: Build & Deploy Production Docker image
+        runs-on: ubuntu-20.04
+        steps:
+            - name: Configure AWS credentials
+              uses: aws-actions/configure-aws-credentials@v1
+              with:
+                  aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+                  aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                  aws-region: us-east-1
+
+            - name: Login to Amazon ECR
+              id: login-ecr
+              uses: aws-actions/amazon-ecr-login@v1
+
+            - name: Fetch posthog-production
+              run: |
+                  curl -L https://github.com/posthog/posthog-production/tarball/master | tar --strip-components=1 -xz --
+                  mkdir deploy/
+
+            - name: Checkout master
+              uses: actions/checkout@v2
+              with:
+                  ref: 'master'
+                  path: 'deploy/'
+
+            - name: Build, tag, and push image to Amazon ECR
+              id: build-image
+              env:
+                  ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+                  ECR_REPOSITORY: posthog-production
+                  IMAGE_TAG: ${{ github.sha }}
+              run: |
+                  docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f prod.web.Dockerfile .
+                  docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+                  echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG"
+
+            - name: Fill in the new image ID in the Amazon ECS task definition
+              id: task-def
+              uses: aws-actions/amazon-ecs-render-task-definition@v1
+              with:
+                  task-definition: deploy/task-definition.json
+                  container-name: posthog-production
+                  image: ${{ steps.build-image.outputs.image }}
+
+            - name: Deploy Amazon ECS task definition
+              uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+              with:
+                  task-definition: ${{ steps.task-def.outputs.task-definition }}
+                  service: posthog-production
+                  cluster: posthog-production-cluster
+                  wait-for-service-stability: true

.gitignore

@@ -28,5 +28,4 @@ yarn-error.log
 yalc.lock
 cypress/screenshots/*
 docker-compose.prod.yml
-posthog.json
 .python-version

CHANGELOG.md

@@ -1,5 +1,72 @@
 # Changelog
 
+### 1.17.0 - Tuesday 17 November 2020
+
+- [Sentry Integration](https://github.com/PostHog/posthog/pull/1833)
+
+![Sentry Screenshot](https://posthog.com/static/85a8c81d33e2e3647657b389c0b12814/2cefc/sentry.png)
+
+An important part of devolping a great user experience is identifying, tracking, and fixing bugs. 
+
+With our new [Sentry](https://sentry.io/) integration, you can leverage PostHog data to help your debugging, and Sentry exception data to track relevant UX metrics. As a two-way integration, it:
+- Adds a direct link in Sentry to the profile of the person affected in PostHog
+- Sends an `$exception` event to PostHog with a direct link to Sentry
+
+If you're unfamiliar with Sentry, we highly recommend you to check it out - it is an awesome application monitoring platform of which we're avid users at PostHog.
+
+To set up the integration you can read the step-by-step instructions on the dedicated [Sentry Integration page](https://posthog.com/docs/integrations/sentry-integration).
+
+- [RudderStack Integration](https://docs.rudderstack.com/destinations/posthog)
+
+RudderStack is an open-source, warehouse-first, customer data platform for developers. It allows you to collect and deliver customer event data to a variety of destinations such as data warehouses and analytics platforms.
+
+As of last week, PostHog is now available as a destination on RudderStack, allowing you to send your event data from various sources into PostHog for performing product analytics. 
+
+You can read more about RudderStack on [their website](https://rudderstack.com/), and learn how to integrate PostHog through their [comprehensive integration docs](https://docs.rudderstack.com/destinations/posthog). 
+
+- [Plugin Attachments and GeoIP Plugin](https://github.com/PostHog/posthog/pull/2263)
+
+![MaxMind Plugin Page Screenshot](https://posthog.com/static/db00f5bcf26ff68ad3e8fe14cde54dcb/2cefc/maxmind-plugin.png)
+
+Over the past two weeks, our [Plugins](https://posthog.com/docs/plugins/overview) feature was extensively worked on to improve the experience of using and developing plugins for PostHog. 
+
+One of the main changes was the addition of plugin attachments, which allow you to upload files that are used in the configuration of the plugin, vastly expanding the realm of possibilities of what plugins can do. 
+
+As a result of this, we built the [PostHog MaxMind Plugin](https://posthog.com/docs/plugins/maxmind), leveraging attachments to allow GeoIP data to be used for enriching your events. Once configured, the plugin adds IP-based location information as properties on your events, such as what country and city your users are located in, making it possible to create charts and tables filtered based on the location of your users.
+
+> **Note:** Plugins are currently only available on self-hosted instances. If you're self-hosting and want to use the PostHog MaxMind Plugin, please follow [these instructions](https://posthog.com/docs/plugins/maxmind). If you want to build your own plugin, check out our [fresh new guide](https://posthog.com/docs/plugins/build) on how to do so.
+
+- [Retentions & Paths Dashboard Panels](https://github.com/PostHog/posthog/pull/2201)
+
+![Retention Panel Screenshot](https://posthog.com/static/adc21b7a7d974cc268481fd4d55b2c29/2cefc/retention-panel.png)
+
+Dashboards are a key part of PostHog, so it's important to us that you can have an overview of as many as possible of your metrics in them.
+
+As such, the user paths graph and the retention table can now be added as panels on dashboards, making it so that every single chart, table, funnel, and graph you create in PostHog can make it to your dashboards now. 
+
+- [First Time Retention](https://github.com/PostHog/posthog/pull/2325)
+
+![First Time Retention Screenshot](https://posthog.com/static/61a2f75d668da309c8800cfc2b4478c7/2cefc/first-time-retention.png)
+
+Following some feedback from our own Growth Engineer on what functionality we need for ourselves at PostHog, we have now extended the functionality of our 'Retention' view, adding first time retention and differentiating between 'Cohortizing' and 'Retaining' events.
+
+In short, first time retention cohortizes users based on when they did an event for the **first time**, rather than adding a user to each cohort they had the event in. Additionally, by being able to have different target events for the cohort and the retention, you are able to track the impact of 'Event A' on the retention of 'Event B', exploring hypotheses such as how users who read your documentation retain on product pageviews when compared to other users.
+
+- [New Events & Actions View](https://github.com/PostHog/posthog/pull/2319)
+
+![Manage Events View Screenshot](https://posthog.com/static/73e3d54092192d20c9f686152685a82e/2cefc/manage-events.png)
+
+In an effort to make it easier to filter through your events in PostHog and tag events that you find useful, we have now consolidated 'Events' and 'Actions' into one single view, found on the left sidebar as 'Events & Actions'.
+
+On this page, you'll be able to manage everything related to your events, from inspecting their properties, to tagging them as actions. In addition, we have also added stats for your event and property volumes, so you can dig deeper into your analytics data collection, and optimize it to your needs.
+
+- [Improved AWS CloudFormation Deployment](https://github.com/PostHog/deployment/pulls?q=is%3Apr+is%3Aclosed)
+
+Following a lot of great user feedback, we have now significantly improved our [AWS CloudFormation Deployment](https://posthog.com/docs/deployment/deploy-aws).
+
+We have now added configuration for relevant alerts and RDS disk size, as well as improved the setup flow and added automatic `SECRET_KEY` generation. If you're happy with the standard config, deploying with AWS is now just a matter of "click, click, click", as described by Karl, one of our engineers.
+
+
 ### 1.16.0 - Wednesday 4 November 2020
 
 - [Session Recording (Beta)](https://github.com/PostHog/posthog/issues/1846)
@@ -409,9 +476,9 @@ To ensure the security of your PostHog instance, it's important that you use a r
 
 Prior to this version, we denoted the importance of this in our Docs, but did not enforce it in our software. Now, to enhance security, PostHog will not allow you to run the server without setting it.
 
-Many of our deployments generate and set this key by default, so that you will not need to worry about it. This is the case with our [Heroku One-Click deployment](/docs/deployment/deploy-heroku), for example. However, other methods may not automatically do this (we're working on it!). As such, if you run into any issues when updating PostHog, make sure you have a unique `SECRET_KEY` set. 
+Many of our deployments generate and set this key by default, so that you will not need to worry about it. This is the case with our [Heroku One-Click deployment](https://posthog.com/docs/deployment/deploy-heroku), for example. However, other methods may not automatically do this (we're working on it!). As such, if you run into any issues when updating PostHog, make sure you have a unique `SECRET_KEY` set. 
 
-You can find more information about this on our ['Securing PostHog' page](/docs/configuring-posthog/securing-posthog#secret-key) and should always feel welcome to ask any questions on our [community Slack group](https://join.slack.com/t/posthogusers/shared_invite/enQtOTY0MzU5NjAwMDY3LTc2MWQ0OTZlNjhkODk3ZDI3NDVjMDE1YjgxY2I4ZjI4MzJhZmVmNjJkN2NmMGJmMzc2N2U3Yjc3ZjI5NGFlZDQ).
+You can find more information about this on our ['Securing PostHog' page](https://posthog.com/docs/configuring-posthog/securing-posthog#secret-key) and should always feel welcome to ask any questions on our [community Slack group](https://join.slack.com/t/posthogusers/shared_invite/enQtOTY0MzU5NjAwMDY3LTc2MWQ0OTZlNjhkODk3ZDI3NDVjMDE1YjgxY2I4ZjI4MzJhZmVmNjJkN2NmMGJmMzc2N2U3Yjc3ZjI5NGFlZDQ).
 
 
 ## Bug Fixes and Performance Improvements
@@ -841,7 +908,7 @@ This should really help with debugging, or just trying to get a detailed view of
 - We fixed [property filter array issue](https://github.com/PostHog/posthog/pull/769)
 - [Optimize funnel rendering](https://github.com/PostHog/posthog/pull/792) is a major improvement in speed for those with many events - now 1 order of magnitude faster. 
 - [Multiple filters with same key](https://github.com/PostHog/posthog/pull/738), fixed a bug that means you can now have multiple filters that are the same ie $current_url doesn't equal A and $current_url doesn't equal B
-- [Event partioning](https://github.com/PostHog/posthog/pull/733), which speeds up trends and paths pages in particular. Learn more about [scaling PostHog](/docs/scaling-posthog).
+- [Event partioning](https://github.com/PostHog/posthog/pull/733), which speeds up trends and paths pages in particular. Learn more about [scaling PostHog](https://posthog.com/docs/scaling-posthog).
 - The component Deletewithundo wasn't working because of property mixup, [now it is](https://github.com/PostHog/posthog/pull/750)!
 - [Funnels](https://github.com/PostHog/posthog/pull/751) and [Actions](https://github.com/PostHog/posthog/pull/757) now use Ant Design
 - We temporarily [removed stickiness breakdowns](https://github.com/PostHog/posthog/pull/774), as they were causing issues.

Procfile

@@ -1,3 +1,5 @@
 release: REDIS_URL='redis://' python manage.py migrate
 web: gunicorn posthog.wsgi --log-file -
 worker: ./bin/docker-worker
+celeryworker: ./bin/docker-worker-celery --with-beat
+pluginworker: ./bin/plugin-server

app.json

@@ -2,7 +2,7 @@
     "name": "PostHog - Product Analytics",
     "website": "https://www.posthog.com",
     "repository": "https://github.com/posthog/posthog",
-    "logo": "https://posthog.com/images/98771d009c04ff64fc4ba3b31cc46fe8/posthog-icon-color.svg",
+    "logo": "https://posthog.com/images/36f8ad157f42d4d48dc1de1db45802e1/posthog-icon-color.svg",
     "environments": {
         "review": {
             "scripts": {
@@ -21,6 +21,12 @@
                 },
                 "worker": {
                     "quantity": 1
+                },
+                "celeryworker": {
+                    "quantity": 0
+                },
+                "pluginworker": {
+                    "quantity": 0
                 }
             }
         },
@@ -48,6 +54,12 @@
         },
         "worker": {
             "quantity": 1
+        },
+        "celeryworker": {
+            "quantity": 0
+        },
+        "pluginworker": {
+            "quantity": 0
         }
     },
     "env": {

bin/docker-preview

@@ -9,6 +9,5 @@ export REDIS_URL=redis://localhost
 /etc/init.d/postgresql start > /dev/null
 /etc/init.d/redis-server start > /dev/null
 
-DEBUG=1 celery -A posthog worker --loglevel=info &
-DEBUG=1 ./bin/plugin-server &
+DEBUG=1 ./bin/docker-worker &
 DEBUG=1 gunicorn posthog.wsgi --config gunicorn.config.py --bind 0.0.0.0:8000 --log-file - -e DEBUG=1 -e DATABASE_URL=postgres://posthog:posthog@localhost:5432/posthog

bin/docker-worker

@@ -14,8 +14,6 @@ then
   echo "⚠️ --> https://posthog.com/docs/deployment/upgrading-posthog#upgrading-from-before-1011"
   echo "⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️"
 else
-  rm celerybeat.pid || echo "celerybeat.pid not found, proceeding"
-  ./bin/docker-worker-plugins &
-  ./bin/docker-worker-beat &
-  ./bin/docker-worker-celery
+  ./bin/plugin-server &
+  ./bin/docker-worker-celery --with-beat
 fi

bin/docker-worker-beat

@@ -1,4 +1,5 @@
 #!/bin/bash
 set -e
 
+rm celerybeat.pid || echo "celerybeat.pid not found, proceeding"
 celery -A posthog beat -S redbeat.RedBeatScheduler

bin/docker-worker-celery

@@ -1,10 +1,17 @@
 #!/bin/bash
 set -e
 
+if [ "$1" == "--with-beat" ]; then
+  ./bin/docker-worker-beat &
+fi
+
 # On heroku $WEB_CONCURRENCY contains suggested nr of forks per dyno type
 # https://github.com/heroku/heroku-buildpack-python/blob/main/vendor/WEB_CONCURRENCY.sh
 if [[ -z "${WEB_CONCURRENCY}" ]]; then
   celery -A posthog worker
 else
   celery -A posthog worker --concurrency $WEB_CONCURRENCY
 fi
+
+# Stop the beat!
+trap 'kill $(jobs -p)' EXIT

bin/frontend-test-runner

@@ -3,7 +3,8 @@ set -e
 
 dropdb --if-exists posthog_e2e_test
 createdb posthog_e2e_test
-DEBUG=1 DATABASE_URL=postgres://localhost:5432/posthog_e2e_test python manage.py migrate
+DEBUG=1 DATABASE_URL=postgres://localhost:5432/posthog_e2e_test python manage.py migrate & yarn add cypress-terminal-report
 ./bin/start-frontend &
 CYPRESS_BASE_URL=http://localhost:8080 npx cypress open &
 DEBUG=1 TEST=1 DATABASE_URL=postgres://localhost:5432/posthog_e2e_test python manage.py runserver 8080
+yarn remove cypress-terminal-report

bin/tests

@@ -1,5 +1,11 @@
 #!/bin/bash
 set -x
 
+if ! command -v COMMAND &> /dev/null
+then
+    echo "Please install nodemon (npm install -g nodemon) to automatically run tests."
+    exit
+fi
+
 REDIS_URL='redis:///' OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES python manage.py test $@ --noinput
-REDIS_URL='redis:///' npx nodemon -w ./posthog -w ./ee --ext py --exec "OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES python manage.py test --noinput --keepdb $@; mypy posthog ee"
+REDIS_URL='redis:///' nodemon -w ./posthog -w ./ee --ext py --exec "OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES python manage.py test --noinput --keepdb $@; mypy posthog ee"

cypress/integration/cohorts.js

@@ -1,7 +1,7 @@
 describe('Cohorts', () => {
     beforeEach(() => {
-        cy.get('[data-attr=menu-item-people]').click()
-        cy.get('[data-attr=menu-item-people-cohorts]').click()
+        cy.get('[data-attr=menu-item-people]').click() // TODO: Remove when releasing navigation-1775
+        cy.get('[data-attr=menu-item-cohorts]').click()
     })
     it('Cohorts new and list', () => {
         // load an empty page
@@ -29,7 +29,7 @@ describe('Cohorts', () => {
 
         // back to cohorts
         cy.get('h1').should('contain', 'Persons')
-        cy.get('[data-attr=menu-item-people-cohorts]').click()
+        cy.get('[data-attr=menu-item-cohorts]').click()
 
         cy.get('h1').should('contain', 'Cohorts')
         cy.get('.ant-empty').should('not.exist')

cypress/integration/featureFlags.js

@@ -8,9 +8,23 @@ describe('Feature Flags', () => {
         cy.get('[data-attr=new-feature-flag]').click()
         cy.get('[data-attr=feature-flag-name').type('beta feature').should('have.value', 'beta feature')
         cy.get('[data-attr=feature-flag-key').should('have.value', 'beta-feature')
+
+        // select "add filter" and "property"
+        cy.get('[data-attr=new-prop-filter-feature-flag').click()
+
+        // select the first property
+        cy.get('[data-attr=property-filter-dropdown]').click()
+        cy.get('[data-attr=prop-filter-person-0]').click({ force: true })
+
+        // selects the first value
+        cy.get('[data-attr=prop-val]').click()
+        cy.get('[data-attr=prop-val-0]').click({ force: true })
+
         cy.get('[data-attr=feature-flag-switch').click()
         cy.get('[data-attr=feature-flag-submit').click()
         cy.get('[data-attr=feature-flag-table').should('contain', 'beta feature')
+        cy.get('[data-attr=rollout-precentage').should('contain', '30%')
+        cy.get('[data-attr=feature-flag-table').should('contain', 'is_demo')
 
         cy.get('[data-attr=feature-flag-table] tr:first-child td:first-child').click()
         cy.get('[data-attr=feature-flag-name').type(' updated').should('have.value', 'beta feature updated')

cypress/integration/person.js

@@ -1,6 +1,6 @@
 describe('Person', () => {
     beforeEach(() => {
-        cy.get('[data-attr=menu-item-people]').click()
+        cy.get('[data-attr=menu-item-people]').click() // TODO: Adjust when releasing navigation-1775
         cy.contains('[email protected]').click()
     })
 

cypress/integration/people.js → cypress/integration/persons.js

@@ -1,5 +1,6 @@
-describe('People', () => {
+describe('Persons', () => {
     beforeEach(() => {
+        // TODO: Remove when releasing navigation-1775
         cy.get('[data-attr=menu-item-people]').click()
     })
 
@@ -17,11 +18,11 @@ describe('People', () => {
         cy.get('.ant-empty-img-simple').should('exist')
     })
 
-    it('All people route works', () => {
-        cy.get('[data-attr=menu-item-people-cohorts]').click()
+    it('Person & cohorts routes works', () => {
+        cy.get('[data-attr=menu-item-cohorts]').click()
         cy.get('h1').should('contain', 'Cohorts')
 
-        cy.get('[data-attr=menu-item-people-persons]').click()
+        cy.get('[data-attr=menu-item-persons]').click()
         cy.get('h1').should('contain', 'Persons')
     })
 })

docker-compose-config.py

@@ -2,8 +2,13 @@
 import string
 from typing import Optional
 
-import yaml
-from yaml.loader import FullLoader
+try:
+    import yaml
+    from yaml.loader import FullLoader
+except ImportError:
+    raise ImportError(
+        "Missing pyyaml! Install it with command `python3 -m pip install pyyaml` to run this configurator."
+    )
 
 FILE_PATH = "docker-compose.yml"
 
@@ -76,4 +81,7 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except KeyboardInterrupt:
+        print("\nConfiguration canceled.")

docker-compose-config.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -ex
+python3 -m pip install pyyaml
+echo
+python3 docker-compose-config.py