Switch win probability to calculate against control

posthog/hogql_queries/experiments/test/test_trends_statistics_count.py

@@ -274,6 +274,51 @@ def run_test(stats_version, calculate_probabilities, are_results_significant, ca
 
         self.run_test_for_both_implementations(run_test)
 
+    @flaky(max_runs=5, min_passes=1)
+    def test_many_variants_win_probabilty_compared_to_control(self):
+        """Test with multiple variants, win probability compared to control"""
+
+        def run_test(stats_version, calculate_probabilities, are_results_significant, calculate_credible_intervals):
+            control_absolute_exposure = 1000
+            control = create_variant("control", count=100, exposure=1, absolute_exposure=control_absolute_exposure)
+            test_a_absolute_exposure = 1000
+            test_a = create_variant(
+                "test_a",
+                count=85,
+                exposure=test_a_absolute_exposure / control_absolute_exposure,
+                absolute_exposure=test_a_absolute_exposure,
+            )
+            test_b_absolute_exposure = 1000
+            test_b = create_variant(
+                "test_b",
+                count=150,
+                exposure=test_b_absolute_exposure / control_absolute_exposure,
+                absolute_exposure=test_b_absolute_exposure,
+            )
+            test_c_absolute_exposure = 1000
+            test_c = create_variant(
+                "test_c",
+                count=110,
+                exposure=test_c_absolute_exposure / control_absolute_exposure,
+                absolute_exposure=test_c_absolute_exposure,
+            )
+
+            probabilities = calculate_probabilities(control, [test_a, test_b, test_c])
+
+            self.assertEqual(len(probabilities), 4)
+            if stats_version == 2:
+                self.assertAlmostEqual(probabilities[0], 0, delta=0.05)
+                self.assertAlmostEqual(probabilities[1], 0.13, delta=0.05)
+                self.assertAlmostEqual(probabilities[2], 0.99, delta=0.05)
+                self.assertAlmostEqual(probabilities[3], 0.75, delta=0.05)
+            else:
+                self.assertAlmostEqual(probabilities[0], 0, delta=0.05)
+                self.assertAlmostEqual(probabilities[1], 0, delta=0.05)
+                self.assertAlmostEqual(probabilities[2], 0.99, delta=0.05)
+                self.assertAlmostEqual(probabilities[3], 0.0, delta=0.05)
+
+        self.run_test_for_both_implementations(run_test)
+
     @flaky(max_runs=5, min_passes=1)
     def test_real_world_data_1(self):
         """Test with multiple variants, one clear winner"""

posthog/hogql_queries/experiments/trends_statistics_v2_count.py

@@ -36,9 +36,9 @@ def calculate_probabilities_v2_count(
     Returns:
     --------
     list[float]
-        A list of probabilities that sum to 1, where:
-        - The first element is the probability that the control variant is the best
-        - Subsequent elements are the probabilities that each test variant is the best
+        A list of probabilities where each element represents:
+        - index 0: probability control variant beats all test variants
+        - index i>0: probability test variant i-1 beats control
 
     Notes:
     ------
@@ -82,12 +82,9 @@ def calculate_probabilities_v2_count(
     control_wins = np.all([samples_control > test_sample for test_sample in test_samples], axis=0)
     probabilities.append(float(np.mean(control_wins)))
 
-    # Probability each test variant wins (beats control and all other test variants)
-    for i, test_sample in enumerate(test_samples):
-        other_test_samples = test_samples[:i] + test_samples[i + 1 :]
-        variant_wins = np.all(
-            [test_sample > samples_control] + [test_sample > other for other in other_test_samples], axis=0
-        )
+    # Probability each test variant wins (beats control only)
+    for test_sample in test_samples:
+        variant_wins = test_sample > samples_control
         probabilities.append(float(np.mean(variant_wins)))
 
     return probabilities