From 635ae4fe3575ed5303f7e3ea17e5cc237a66434f Mon Sep 17 00:00:00 2001 From: Eduardo Filho Date: Fri, 17 Nov 2023 16:57:09 -0500 Subject: [PATCH 1/4] Migrate windows + release sample counts to reflect sampling (10x) --- .../0026_fix_windows_release_sample_count.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 glam/api/migrations/0026_fix_windows_release_sample_count.py diff --git a/glam/api/migrations/0026_fix_windows_release_sample_count.py b/glam/api/migrations/0026_fix_windows_release_sample_count.py new file mode 100644 index 000000000..caf39eb9f --- /dev/null +++ b/glam/api/migrations/0026_fix_windows_release_sample_count.py @@ -0,0 +1,51 @@ +from django.db import migrations + +configs = [ + { + "model": "desktopreleaseaggregation", + "pretty_name": "Desktop", + "filter": {"os": "Windows"} + }, + # TODO: Leaving this here in case we need to migrate fog in the future + #{ + # "model": "fogaggregation", + # "pretty_name": "FOG", + # "filter": {"os": "Windows", "app_id": "release"} + #} +] + +def do_migration(apps, fwd = True): + for config in configs: + model = config['model'] + print(f"\nMigrating {config['pretty_name']}...") + table = apps.get_model('api', model).objects.filter(**config['filter']) + total = table.count() + table_iter = table.iterator(100000) + for i, instance in enumerate(table_iter): + if fwd: + instance.total_sample *= 10 + else: + instance.total_sample /= 10 + instance.save() + if i % 10000 == 0 or i + 1 == total: + print(f"{i} out of {total} rows migrated ({round((i+1)/total*100, 1)}%)", end='\r') + +def multiply_sample_count_by_10(apps, schema_editor): + do_migration(apps) + +def divide_sample_count_by_10(apps, schema_editor): + do_migration(apps, False) + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0025_non_normalized_aggregations"), + ] + + operations = [ + migrations.RunPython(code=multiply_sample_count_by_10, reverse_code=migrations.RunPython.noop), + migrations.RunSQL("REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_desktop_release_aggregation", + "REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_desktop_release_aggregation"), + migrations.RunPython(code=migrations.RunPython.noop, reverse_code=divide_sample_count_by_10), + + ] From 88b7638dd2a427f2448fa8ec8159b6dbf4517271 Mon Sep 17 00:00:00 2001 From: Eduardo Filho Date: Mon, 20 Nov 2023 10:06:40 -0500 Subject: [PATCH 2/4] Fix formatting --- .../0026_fix_windows_release_sample_count.py | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/glam/api/migrations/0026_fix_windows_release_sample_count.py b/glam/api/migrations/0026_fix_windows_release_sample_count.py index caf39eb9f..9d8b48ce7 100644 --- a/glam/api/migrations/0026_fix_windows_release_sample_count.py +++ b/glam/api/migrations/0026_fix_windows_release_sample_count.py @@ -4,21 +4,22 @@ { "model": "desktopreleaseaggregation", "pretty_name": "Desktop", - "filter": {"os": "Windows"} + "filter": {"os": "Windows"}, }, # TODO: Leaving this here in case we need to migrate fog in the future - #{ + # { # "model": "fogaggregation", # "pretty_name": "FOG", # "filter": {"os": "Windows", "app_id": "release"} - #} + # } ] -def do_migration(apps, fwd = True): + +def do_migration(apps, fwd=True): for config in configs: - model = config['model'] + model = config["model"] print(f"\nMigrating {config['pretty_name']}...") - table = apps.get_model('api', model).objects.filter(**config['filter']) + table = apps.get_model("api", model).objects.filter(**config["filter"]) total = table.count() table_iter = table.iterator(100000) for i, instance in enumerate(table_iter): @@ -28,11 +29,16 @@ def do_migration(apps, fwd = True): instance.total_sample /= 10 instance.save() if i % 10000 == 0 or i + 1 == total: - print(f"{i} out of {total} rows migrated ({round((i+1)/total*100, 1)}%)", end='\r') + print( + f"{i} out of {total} rows migrated ({round((i+1)/total*100, 1)}%)", + end="\r", + ) + def multiply_sample_count_by_10(apps, schema_editor): do_migration(apps) + def divide_sample_count_by_10(apps, schema_editor): do_migration(apps, False) @@ -43,9 +49,14 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(code=multiply_sample_count_by_10, reverse_code=migrations.RunPython.noop), - migrations.RunSQL("REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_desktop_release_aggregation", - "REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_desktop_release_aggregation"), - migrations.RunPython(code=migrations.RunPython.noop, reverse_code=divide_sample_count_by_10), - + migrations.RunPython( + code=multiply_sample_count_by_10, reverse_code=migrations.RunPython.noop + ), + migrations.RunSQL( + "REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_desktop_release_aggregation", + "REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_desktop_release_aggregation", + ), + migrations.RunPython( + code=migrations.RunPython.noop, reverse_code=divide_sample_count_by_10 + ), ] From e80add3f1783c76bc7bb95027b6b795324db1064 Mon Sep 17 00:00:00 2001 From: Eduardo Filho Date: Tue, 21 Nov 2023 20:23:42 -0500 Subject: [PATCH 3/4] Migrate FOG user counts, sample counts starting v119 --- .../0026_fix_windows_release_sample_count.py | 18 +++--- .../0027_fix_fog_user_count_after_sampling.py | 57 +++++++++++++++++++ 2 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 glam/api/migrations/0027_fix_fog_user_count_after_sampling.py diff --git a/glam/api/migrations/0026_fix_windows_release_sample_count.py b/glam/api/migrations/0026_fix_windows_release_sample_count.py index 9d8b48ce7..f2dbd2354 100644 --- a/glam/api/migrations/0026_fix_windows_release_sample_count.py +++ b/glam/api/migrations/0026_fix_windows_release_sample_count.py @@ -6,12 +6,11 @@ "pretty_name": "Desktop", "filter": {"os": "Windows"}, }, - # TODO: Leaving this here in case we need to migrate fog in the future - # { - # "model": "fogaggregation", - # "pretty_name": "FOG", - # "filter": {"os": "Windows", "app_id": "release"} - # } + { + "model": "fogaggregation", + "pretty_name": "FOG", + "filter": {"os": "Windows", "app_id": "release", "version__gt": 118}, + }, ] @@ -33,6 +32,7 @@ def do_migration(apps, fwd=True): f"{i} out of {total} rows migrated ({round((i+1)/total*100, 1)}%)", end="\r", ) + print("\nRefreshing views...") def multiply_sample_count_by_10(apps, schema_editor): @@ -54,7 +54,11 @@ class Migration(migrations.Migration): ), migrations.RunSQL( "REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_desktop_release_aggregation", - "REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_desktop_release_aggregation", + reverse_sql="REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_desktop_release_aggregation", + ), + migrations.RunSQL( + "REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_fog_aggregation", + reverse_sql="REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_fog_aggregation", ), migrations.RunPython( code=migrations.RunPython.noop, reverse_code=divide_sample_count_by_10 diff --git a/glam/api/migrations/0027_fix_fog_user_count_after_sampling.py b/glam/api/migrations/0027_fix_fog_user_count_after_sampling.py new file mode 100644 index 000000000..3cc3f1c47 --- /dev/null +++ b/glam/api/migrations/0027_fix_fog_user_count_after_sampling.py @@ -0,0 +1,57 @@ +from django.db import migrations + +configs = [ + { + "model": "fogaggregation", + "pretty_name": "FOG", + "filter": {"os": "Windows", "app_id": "release", "version__gt": 118}, + } +] + + +def do_migration(apps, fwd=True): + for config in configs: + model = config["model"] + print(f"\nMigrating {config['pretty_name']}...") + table = apps.get_model("api", model).objects.filter(**config["filter"]) + total = table.count() + table_iter = table.iterator(100000) + for i, instance in enumerate(table_iter): + if fwd: + instance.total_users *= 10 + else: + instance.total_users /= 10 + instance.save() + if i % 10000 == 0 or i + 1 == total: + print( + f"{i} out of {total} rows migrated ({round((i+1)/total*100, 1)}%)", + end="\r", + ) + print("\nRefreshing views...") + + +def multiply_user_count_by_10(apps, schema_editor): + do_migration(apps) + + +def divide_user_count_by_10(apps, schema_editor): + do_migration(apps, False) + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0026_fix_windows_release_sample_count"), + ] + + operations = [ + migrations.RunPython( + code=multiply_user_count_by_10, reverse_code=migrations.RunPython.noop + ), + migrations.RunSQL( + "REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_fog_aggregation", + reverse_sql="REFRESH MATERIALIZED VIEW CONCURRENTLY view_glam_fog_aggregation", + ), + migrations.RunPython( + code=migrations.RunPython.noop, reverse_code=divide_user_count_by_10 + ), + ] From 5a09757b12b648d595fd3c3df14c565e9863acdd Mon Sep 17 00:00:00 2001 From: Eduardo Filho Date: Wed, 22 Nov 2023 11:37:26 -0500 Subject: [PATCH 4/4] Fix sample_count migration --- .../0026_fix_windows_release_sample_count.py | 11 ++++++----- .../0027_fix_fog_user_count_after_sampling.py | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/glam/api/migrations/0026_fix_windows_release_sample_count.py b/glam/api/migrations/0026_fix_windows_release_sample_count.py index f2dbd2354..df31a08d7 100644 --- a/glam/api/migrations/0026_fix_windows_release_sample_count.py +++ b/glam/api/migrations/0026_fix_windows_release_sample_count.py @@ -22,11 +22,12 @@ def do_migration(apps, fwd=True): total = table.count() table_iter = table.iterator(100000) for i, instance in enumerate(table_iter): - if fwd: - instance.total_sample *= 10 - else: - instance.total_sample /= 10 - instance.save() + if instance.total_sample is not None: + if fwd: + instance.total_sample *= 10 + else: + instance.total_sample /= 10 + instance.save() if i % 10000 == 0 or i + 1 == total: print( f"{i} out of {total} rows migrated ({round((i+1)/total*100, 1)}%)", diff --git a/glam/api/migrations/0027_fix_fog_user_count_after_sampling.py b/glam/api/migrations/0027_fix_fog_user_count_after_sampling.py index 3cc3f1c47..014bd2e73 100644 --- a/glam/api/migrations/0027_fix_fog_user_count_after_sampling.py +++ b/glam/api/migrations/0027_fix_fog_user_count_after_sampling.py @@ -17,11 +17,12 @@ def do_migration(apps, fwd=True): total = table.count() table_iter = table.iterator(100000) for i, instance in enumerate(table_iter): - if fwd: - instance.total_users *= 10 - else: - instance.total_users /= 10 - instance.save() + if instance.total_users is not None: + if fwd: + instance.total_users *= 10 + else: + instance.total_users /= 10 + instance.save() if i % 10000 == 0 or i + 1 == total: print( f"{i} out of {total} rows migrated ({round((i+1)/total*100, 1)}%)",