From ccdf7ef9dc2f1ea6759a5f6a5b049b08d88e98ab Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 27 Jan 2025 09:44:20 -0500 Subject: [PATCH 01/12] Simple command to nuke history with no user ID --- .../commands/dedupe_history_entries.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/meshapi/management/commands/dedupe_history_entries.py diff --git a/src/meshapi/management/commands/dedupe_history_entries.py b/src/meshapi/management/commands/dedupe_history_entries.py new file mode 100644 index 00000000..7d730840 --- /dev/null +++ b/src/meshapi/management/commands/dedupe_history_entries.py @@ -0,0 +1,47 @@ +from argparse import ArgumentParser +import logging +from typing import Any + +from django.core.management.base import BaseCommand +from django.db import models, transaction + +from meshapi.models.devices.device import Device +from meshapi.models.link import Link + + +class Command(BaseCommand): + help = "Deletes duplicate history entries on history tables" + + def add_arguments(self, parser: ArgumentParser) -> None: + pass + + def handle(self, *args: Any, **options: Any) -> None: + self.deduplicate_history(Link.objects.all()) + self.deduplicate_history(Device.objects.all()) + + def deduplicate_history(self, model_objects: models.QuerySet): + for m in model_objects: + logging.info(f"{m.id}, {m}") + + # Delete history from each object + # Atomic block makes it go faster + with transaction.atomic(): + try: + history = m.history.all() + # If there's no history, bail + if not history: + continue + first_history = None + for h in history: + # Hacky way to preserve the first object + if not first_history: + first_history = h + continue + if not h.history_user_id: + #logging.info(f"Deleting history record with user_id = None: {h}") + h.delete() + + except Exception as e: + logging.exception(f"Could not get history for this link: {e}") + raise e + From 67af1a120cebdaccf878cb74aa5b45c774f067b2 Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 27 Jan 2025 19:15:20 -0500 Subject: [PATCH 02/12] checkpoint --- .../commands/dedupe_history_entries.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/meshapi/management/commands/dedupe_history_entries.py b/src/meshapi/management/commands/dedupe_history_entries.py index 7d730840..96cb2382 100644 --- a/src/meshapi/management/commands/dedupe_history_entries.py +++ b/src/meshapi/management/commands/dedupe_history_entries.py @@ -16,30 +16,35 @@ def add_arguments(self, parser: ArgumentParser) -> None: pass def handle(self, *args: Any, **options: Any) -> None: - self.deduplicate_history(Link.objects.all()) + # self.deduplicate_history(Link.objects.all()) self.deduplicate_history(Device.objects.all()) def deduplicate_history(self, model_objects: models.QuerySet): for m in model_objects: logging.info(f"{m.id}, {m}") - - # Delete history from each object - # Atomic block makes it go faster + # Delete history from each object. Atomic block makes it go faster with transaction.atomic(): try: history = m.history.all() # If there's no history, bail if not history: continue - first_history = None + # This is the history object that last changed something + last_meaningful_history = None for h in history: - # Hacky way to preserve the first object - if not first_history: - first_history = h + if not last_meaningful_history: + last_meaningful_history = h continue - if not h.history_user_id: - #logging.info(f"Deleting history record with user_id = None: {h}") - h.delete() + delta = last_meaningful_history.diff_against( + h, + # foreign_keys_are_objs=foreign_keys_are_objs + ) + print(delta) + return + #import pdb; pdb.set_trace() + #if not h.history_user_id: + # #logging.info(f"Deleting history record with user_id = None: {h}") + # h.delete() except Exception as e: logging.exception(f"Could not get history for this link: {e}") From 569084cb2372f35f9afd2b0c9c42dd37ba35173a Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 27 Jan 2025 20:19:38 -0500 Subject: [PATCH 03/12] Got the deltas working --- .../commands/dedupe_history_entries.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/meshapi/management/commands/dedupe_history_entries.py b/src/meshapi/management/commands/dedupe_history_entries.py index 96cb2382..d9d1c38d 100644 --- a/src/meshapi/management/commands/dedupe_history_entries.py +++ b/src/meshapi/management/commands/dedupe_history_entries.py @@ -39,8 +39,18 @@ def deduplicate_history(self, model_objects: models.QuerySet): h, # foreign_keys_are_objs=foreign_keys_are_objs ) - print(delta) - return + # If there were fields that changed meaningfully, then + # track that by updating last_meaningful_history and + # keep going + if delta.changes or delta.changed_fields: + logging.info(f"Preserving Change: {delta}") + last_meaningful_history = h + continue + + # Otherwise, delete the history object + #logging.info("Deleting history") + h.delete() + #import pdb; pdb.set_trace() #if not h.history_user_id: # #logging.info(f"Deleting history record with user_id = None: {h}") @@ -49,4 +59,4 @@ def deduplicate_history(self, model_objects: models.QuerySet): except Exception as e: logging.exception(f"Could not get history for this link: {e}") raise e - + input("Press any key.") From 6bec51dfd5d4a54fbc67399acee3a3fe8f17cdf0 Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 27 Jan 2025 21:23:26 -0500 Subject: [PATCH 04/12] codecov will probably yell at me --- .../management/commands/dedupe_history_entries.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/meshapi/management/commands/dedupe_history_entries.py b/src/meshapi/management/commands/dedupe_history_entries.py index d9d1c38d..522e6c52 100644 --- a/src/meshapi/management/commands/dedupe_history_entries.py +++ b/src/meshapi/management/commands/dedupe_history_entries.py @@ -7,6 +7,7 @@ from meshapi.models.devices.device import Device from meshapi.models.link import Link +from meshapi.models.los import LOS class Command(BaseCommand): @@ -16,8 +17,9 @@ def add_arguments(self, parser: ArgumentParser) -> None: pass def handle(self, *args: Any, **options: Any) -> None: - # self.deduplicate_history(Link.objects.all()) + self.deduplicate_history(Link.objects.all()) self.deduplicate_history(Device.objects.all()) + self.deduplicate_history(LOS.objects.all()) def deduplicate_history(self, model_objects: models.QuerySet): for m in model_objects: @@ -32,12 +34,13 @@ def deduplicate_history(self, model_objects: models.QuerySet): # This is the history object that last changed something last_meaningful_history = None for h in history: + # Grab the first history object we come across if not last_meaningful_history: last_meaningful_history = h continue delta = last_meaningful_history.diff_against( h, - # foreign_keys_are_objs=foreign_keys_are_objs + foreign_keys_are_objs=False # This makes foreign keys show up as UUIDs ) # If there were fields that changed meaningfully, then # track that by updating last_meaningful_history and @@ -48,15 +51,9 @@ def deduplicate_history(self, model_objects: models.QuerySet): continue # Otherwise, delete the history object - #logging.info("Deleting history") h.delete() - #import pdb; pdb.set_trace() - #if not h.history_user_id: - # #logging.info(f"Deleting history record with user_id = None: {h}") - # h.delete() - except Exception as e: logging.exception(f"Could not get history for this link: {e}") raise e - input("Press any key.") + #input("Press any key.") From de84cf586f986a4a090a1ab0af9ede050d4713fc Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 27 Jan 2025 21:25:42 -0500 Subject: [PATCH 05/12] lint --- src/meshapi/management/commands/dedupe_history_entries.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/meshapi/management/commands/dedupe_history_entries.py b/src/meshapi/management/commands/dedupe_history_entries.py index 522e6c52..4ebe6800 100644 --- a/src/meshapi/management/commands/dedupe_history_entries.py +++ b/src/meshapi/management/commands/dedupe_history_entries.py @@ -1,5 +1,5 @@ -from argparse import ArgumentParser import logging +from argparse import ArgumentParser from typing import Any from django.core.management.base import BaseCommand @@ -21,7 +21,7 @@ def handle(self, *args: Any, **options: Any) -> None: self.deduplicate_history(Device.objects.all()) self.deduplicate_history(LOS.objects.all()) - def deduplicate_history(self, model_objects: models.QuerySet): + def deduplicate_history(self, model_objects: models.QuerySet) -> None: for m in model_objects: logging.info(f"{m.id}, {m}") # Delete history from each object. Atomic block makes it go faster @@ -39,8 +39,7 @@ def deduplicate_history(self, model_objects: models.QuerySet): last_meaningful_history = h continue delta = last_meaningful_history.diff_against( - h, - foreign_keys_are_objs=False # This makes foreign keys show up as UUIDs + h, foreign_keys_are_objs=False # This makes foreign keys show up as UUIDs ) # If there were fields that changed meaningfully, then # track that by updating last_meaningful_history and @@ -56,4 +55,3 @@ def deduplicate_history(self, model_objects: models.QuerySet): except Exception as e: logging.exception(f"Could not get history for this link: {e}") raise e - #input("Press any key.") From dd6b9bd42004ba395c149da120e27177fc57482d Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 27 Jan 2025 22:34:45 -0500 Subject: [PATCH 06/12] test --- .../tests/test_dedupe_history_entries.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/meshapi/tests/test_dedupe_history_entries.py diff --git a/src/meshapi/tests/test_dedupe_history_entries.py b/src/meshapi/tests/test_dedupe_history_entries.py new file mode 100644 index 00000000..312123d7 --- /dev/null +++ b/src/meshapi/tests/test_dedupe_history_entries.py @@ -0,0 +1,95 @@ +from datetime import datetime, date, timezone +from django.core import management +from uuid import UUID +from django.test import TestCase +from simple_history.models import HistoricalRecords + +from meshapi.models.devices.device import Device +from meshapi.models.link import Link +from meshapi.models.node import Node + + +class TestDedupeHistoryEntries(TestCase): + def setUp(self) -> None: + self.node1 = Node( + network_number=1234, + status=Node.NodeStatus.ACTIVE, + type=Node.NodeType.STANDARD, + latitude=0, + longitude=0, + ) + self.node1.save() + + self.node2 = Node( + network_number=5678, + status=Node.NodeStatus.ACTIVE, + type=Node.NodeType.STANDARD, + latitude=0, + longitude=0, + ) + self.node2.save() + + self.node3 = Node( + network_number=9012, + status=Node.NodeStatus.ACTIVE, + type=Node.NodeType.STANDARD, + latitude=0, + longitude=0, + ) + self.node3.save() + + self.device1 = Device( + node=self.node1, + status=Device.DeviceStatus.ACTIVE, + name="nycmesh-1234-dev1", + ) + self.device1.save() + + self.device2 = Device( + node=self.node2, + status=Device.DeviceStatus.ACTIVE, + name="nycmesh-5678-dev2", + ) + self.device2.save() + + self.device3 = Device( + node=self.node3, + status=Device.DeviceStatus.ACTIVE, + name="nycmesh-9012-dev3", + ) + # Save the device multiple times to see if we can get some bad history + self.device3.save() + self.device3.save() + self.device3.save() + + self.link = Link( + from_device=self.device1, + to_device=self.device2, + status=Link.LinkStatus.ACTIVE, + type=Link.LinkType.FIVE_GHZ, + uisp_id="fake-uisp-uuid", + ) + + # Save the link multiple times to see if we can get some bad history + self.link.save() + self.link.save() + self.link.save() + self.link.save() + + + def test_deduplicate_history(self): + # Ensure link has 4 entries + self.assertEqual(4, len(self.link.history.all())) + + # Ensure device3 has 3 + self.assertEqual(3, len(self.device3.history.all())) + + # etc + self.assertEqual(1, len(self.device2.history.all())) + + management.call_command("dedupe_history_entries") + self.assertEqual(1, len(self.link.history.all())) + self.assertEqual(1, len(self.device3.history.all())) + self.assertEqual(1, len(self.device2.history.all())) + + From 1d10b0c545a303382a3202e128e861909d7b38b6 Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 27 Jan 2025 22:41:49 -0500 Subject: [PATCH 07/12] test 2 --- src/meshapi/tests/test_dedupe_history_entries.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/meshapi/tests/test_dedupe_history_entries.py b/src/meshapi/tests/test_dedupe_history_entries.py index 312123d7..d303381d 100644 --- a/src/meshapi/tests/test_dedupe_history_entries.py +++ b/src/meshapi/tests/test_dedupe_history_entries.py @@ -1,8 +1,5 @@ -from datetime import datetime, date, timezone from django.core import management -from uuid import UUID from django.test import TestCase -from simple_history.models import HistoricalRecords from meshapi.models.devices.device import Device from meshapi.models.link import Link @@ -52,6 +49,10 @@ def setUp(self) -> None: ) self.device2.save() + # This is a genuine change + self.device2.name="hi mom" + self.device2.save() + self.device3 = Device( node=self.node3, status=Device.DeviceStatus.ACTIVE, @@ -76,7 +77,6 @@ def setUp(self) -> None: self.link.save() self.link.save() - def test_deduplicate_history(self): # Ensure link has 4 entries self.assertEqual(4, len(self.link.history.all())) @@ -85,11 +85,11 @@ def test_deduplicate_history(self): self.assertEqual(3, len(self.device3.history.all())) # etc - self.assertEqual(1, len(self.device2.history.all())) + self.assertEqual(2, len(self.device2.history.all())) + self.assertEqual(1, len(self.device1.history.all())) management.call_command("dedupe_history_entries") self.assertEqual(1, len(self.link.history.all())) self.assertEqual(1, len(self.device3.history.all())) - self.assertEqual(1, len(self.device2.history.all())) - - + self.assertEqual(2, len(self.device2.history.all())) + self.assertEqual(1, len(self.device1.history.all())) From d8bf29d05002345adfa5cf3a99b6b7e6e38f6561 Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 27 Jan 2025 22:45:29 -0500 Subject: [PATCH 08/12] datadog computers we fix distributed system --- src/meshapi/tests/test_dedupe_history_entries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshapi/tests/test_dedupe_history_entries.py b/src/meshapi/tests/test_dedupe_history_entries.py index d303381d..3eeb43d7 100644 --- a/src/meshapi/tests/test_dedupe_history_entries.py +++ b/src/meshapi/tests/test_dedupe_history_entries.py @@ -50,7 +50,7 @@ def setUp(self) -> None: self.device2.save() # This is a genuine change - self.device2.name="hi mom" + self.device2.name = "hi mom" self.device2.save() self.device3 = Device( From c15b2359b4804e83f77e7b7f92fbb83552e42149 Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Wed, 29 Jan 2025 00:52:38 -0500 Subject: [PATCH 09/12] add a counter --- src/meshapi/management/commands/dedupe_history_entries.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/meshapi/management/commands/dedupe_history_entries.py b/src/meshapi/management/commands/dedupe_history_entries.py index 4ebe6800..f96fa54e 100644 --- a/src/meshapi/management/commands/dedupe_history_entries.py +++ b/src/meshapi/management/commands/dedupe_history_entries.py @@ -26,6 +26,7 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: logging.info(f"{m.id}, {m}") # Delete history from each object. Atomic block makes it go faster with transaction.atomic(): + deleted_records = 0 try: history = m.history.all() # If there's no history, bail @@ -51,7 +52,10 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: # Otherwise, delete the history object h.delete() - + deleted_records += 1 except Exception as e: logging.exception(f"Could not get history for this link: {e}") raise e + + logging.info(f"Deleted {deleted_records} records.") + From facdb5f2142ce9727a0173599a27179946063aa5 Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Wed, 29 Jan 2025 01:57:40 -0500 Subject: [PATCH 10/12] Speed up deletes with raw query --- .../commands/dedupe_history_entries.py | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/meshapi/management/commands/dedupe_history_entries.py b/src/meshapi/management/commands/dedupe_history_entries.py index f96fa54e..a59e44bd 100644 --- a/src/meshapi/management/commands/dedupe_history_entries.py +++ b/src/meshapi/management/commands/dedupe_history_entries.py @@ -3,7 +3,7 @@ from typing import Any from django.core.management.base import BaseCommand -from django.db import models, transaction +from django.db import connection, models, transaction from meshapi.models.devices.device import Device from meshapi.models.link import Link @@ -23,6 +23,9 @@ def handle(self, *args: Any, **options: Any) -> None: def deduplicate_history(self, model_objects: models.QuerySet) -> None: for m in model_objects: + history_model = m.history.model + table_name = history_model._meta.db_table + logging.info(f"{m.id}, {m}") # Delete history from each object. Atomic block makes it go faster with transaction.atomic(): @@ -33,13 +36,15 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: if not history: continue # This is the history object that last changed something - last_meaningful_history = None + meaningful_history: list[Any] = ( + [] + ) # I don't think I have a type I can put in here without something yelling at me for h in history: # Grab the first history object we come across - if not last_meaningful_history: - last_meaningful_history = h + if not meaningful_history: + meaningful_history.append(h) continue - delta = last_meaningful_history.diff_against( + delta = meaningful_history[-1].diff_against( h, foreign_keys_are_objs=False # This makes foreign keys show up as UUIDs ) # If there were fields that changed meaningfully, then @@ -47,15 +52,25 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: # keep going if delta.changes or delta.changed_fields: logging.info(f"Preserving Change: {delta}") - last_meaningful_history = h + meaningful_history.append(h) continue - # Otherwise, delete the history object - h.delete() + # Otherwise, delete the history object (just don't preserve it) + # h.delete() deleted_records += 1 + + # Nuke history for this object + with connection.cursor() as c: + query = f"DELETE FROM {table_name} WHERE id = '{m.id}'" + logging.info(query) + c.execute(query) + + # Replace the history with meaningful history + for mh in meaningful_history: + mh.pk = None + mh.save() + except Exception as e: logging.exception(f"Could not get history for this link: {e}") - raise e logging.info(f"Deleted {deleted_records} records.") - From 8c99795cbfa2ca6a51a3fc0a85cd77ba9437821e Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Wed, 29 Jan 2025 03:08:35 -0500 Subject: [PATCH 11/12] delete unneeded prints and take a shortcut --- .../commands/dedupe_history_entries.py | 22 +++--- .../management/commands/testing_queries.py | 78 +++++++++++++++++++ 2 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 src/meshapi/management/commands/testing_queries.py diff --git a/src/meshapi/management/commands/dedupe_history_entries.py b/src/meshapi/management/commands/dedupe_history_entries.py index a59e44bd..2cb4ff3a 100644 --- a/src/meshapi/management/commands/dedupe_history_entries.py +++ b/src/meshapi/management/commands/dedupe_history_entries.py @@ -26,7 +26,6 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: history_model = m.history.model table_name = history_model._meta.db_table - logging.info(f"{m.id}, {m}") # Delete history from each object. Atomic block makes it go faster with transaction.atomic(): deleted_records = 0 @@ -36,9 +35,9 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: if not history: continue # This is the history object that last changed something - meaningful_history: list[Any] = ( - [] - ) # I don't think I have a type I can put in here without something yelling at me + # XXX (wdn): I don't think I have a type I can put in here + # without some robot yelling at me + meaningful_history: list[Any] = [] for h in history: # Grab the first history object we come across if not meaningful_history: @@ -51,7 +50,7 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: # track that by updating last_meaningful_history and # keep going if delta.changes or delta.changed_fields: - logging.info(f"Preserving Change: {delta}") + #logging.info(f"Preserving Change: {delta}") meaningful_history.append(h) continue @@ -59,11 +58,16 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: # h.delete() deleted_records += 1 + if not deleted_records: + continue + + logging.info(f"Deleting {deleted_records} from {m.id}, {m}") + # Nuke history for this object - with connection.cursor() as c: + with connection.cursor() as cursor: query = f"DELETE FROM {table_name} WHERE id = '{m.id}'" - logging.info(query) - c.execute(query) + #logging.info(query) + cursor.execute(query) # Replace the history with meaningful history for mh in meaningful_history: @@ -72,5 +76,3 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: except Exception as e: logging.exception(f"Could not get history for this link: {e}") - - logging.info(f"Deleted {deleted_records} records.") diff --git a/src/meshapi/management/commands/testing_queries.py b/src/meshapi/management/commands/testing_queries.py new file mode 100644 index 00000000..8bb909dc --- /dev/null +++ b/src/meshapi/management/commands/testing_queries.py @@ -0,0 +1,78 @@ +import logging +from argparse import ArgumentParser +from typing import Any + +from django.core.management.base import BaseCommand +from django.db import connection, models, transaction + +from meshapi.models.devices.device import Device +from meshapi.models.link import Link +from meshapi.models.los import LOS + + +class Command(BaseCommand): + help = "Deletes duplicate history entries on history tables" + + def add_arguments(self, parser: ArgumentParser) -> None: + pass + + def handle(self, *args: Any, **options: Any) -> None: + + field_query = f"({', '.join(field.name for field in Link._meta.fields)})" + print(field_query) + return + + for m in Link.objects.all(): + history_model = m.history.model + table_name = history_model._meta.db_table + + # Delete history from each object. Atomic block makes it go faster + with transaction.atomic(): + deleted_records = 0 + try: + history = m.history.all() + # If there's no history, bail + if not history: + continue + # This is the history object that last changed something + # XXX (wdn): I don't think I have a type I can put in here + # without some robot yelling at me + meaningful_history: list[Any] = [] + for h in history: + # Grab the first history object we come across + if not meaningful_history: + meaningful_history.append(h) + continue + delta = meaningful_history[-1].diff_against( + h, foreign_keys_are_objs=False # This makes foreign keys show up as UUIDs + ) + # If there were fields that changed meaningfully, then + # track that by updating last_meaningful_history and + # keep going + if delta.changes or delta.changed_fields: + #logging.info(f"Preserving Change: {delta}") + meaningful_history.append(h) + continue + + # Otherwise, delete the history object (just don't preserve it) + # h.delete() + deleted_records += 1 + + if not deleted_records: + continue + + logging.info(f"Deleting {deleted_records} from {m.id}, {m}") + + # Nuke history for this object + with connection.cursor() as c: + query = f"DELETE FROM {table_name} WHERE id = '{m.id}'" + #logging.info(query) + c.execute(query) + + # Replace the history with meaningful history + for mh in meaningful_history: + mh.pk = None + mh.save() + + except Exception as e: + logging.exception(f"Could not get history for this link: {e}") From 2dc83cc4865c83ac8f791c9dc7311b2dba310bf7 Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Wed, 29 Jan 2025 03:10:20 -0500 Subject: [PATCH 12/12] lint --- .../commands/dedupe_history_entries.py | 4 +- .../management/commands/testing_queries.py | 78 ------------------- 2 files changed, 2 insertions(+), 80 deletions(-) delete mode 100644 src/meshapi/management/commands/testing_queries.py diff --git a/src/meshapi/management/commands/dedupe_history_entries.py b/src/meshapi/management/commands/dedupe_history_entries.py index 2cb4ff3a..9ae11a04 100644 --- a/src/meshapi/management/commands/dedupe_history_entries.py +++ b/src/meshapi/management/commands/dedupe_history_entries.py @@ -50,7 +50,7 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: # track that by updating last_meaningful_history and # keep going if delta.changes or delta.changed_fields: - #logging.info(f"Preserving Change: {delta}") + # logging.info(f"Preserving Change: {delta}") meaningful_history.append(h) continue @@ -66,7 +66,7 @@ def deduplicate_history(self, model_objects: models.QuerySet) -> None: # Nuke history for this object with connection.cursor() as cursor: query = f"DELETE FROM {table_name} WHERE id = '{m.id}'" - #logging.info(query) + # logging.info(query) cursor.execute(query) # Replace the history with meaningful history diff --git a/src/meshapi/management/commands/testing_queries.py b/src/meshapi/management/commands/testing_queries.py deleted file mode 100644 index 8bb909dc..00000000 --- a/src/meshapi/management/commands/testing_queries.py +++ /dev/null @@ -1,78 +0,0 @@ -import logging -from argparse import ArgumentParser -from typing import Any - -from django.core.management.base import BaseCommand -from django.db import connection, models, transaction - -from meshapi.models.devices.device import Device -from meshapi.models.link import Link -from meshapi.models.los import LOS - - -class Command(BaseCommand): - help = "Deletes duplicate history entries on history tables" - - def add_arguments(self, parser: ArgumentParser) -> None: - pass - - def handle(self, *args: Any, **options: Any) -> None: - - field_query = f"({', '.join(field.name for field in Link._meta.fields)})" - print(field_query) - return - - for m in Link.objects.all(): - history_model = m.history.model - table_name = history_model._meta.db_table - - # Delete history from each object. Atomic block makes it go faster - with transaction.atomic(): - deleted_records = 0 - try: - history = m.history.all() - # If there's no history, bail - if not history: - continue - # This is the history object that last changed something - # XXX (wdn): I don't think I have a type I can put in here - # without some robot yelling at me - meaningful_history: list[Any] = [] - for h in history: - # Grab the first history object we come across - if not meaningful_history: - meaningful_history.append(h) - continue - delta = meaningful_history[-1].diff_against( - h, foreign_keys_are_objs=False # This makes foreign keys show up as UUIDs - ) - # If there were fields that changed meaningfully, then - # track that by updating last_meaningful_history and - # keep going - if delta.changes or delta.changed_fields: - #logging.info(f"Preserving Change: {delta}") - meaningful_history.append(h) - continue - - # Otherwise, delete the history object (just don't preserve it) - # h.delete() - deleted_records += 1 - - if not deleted_records: - continue - - logging.info(f"Deleting {deleted_records} from {m.id}, {m}") - - # Nuke history for this object - with connection.cursor() as c: - query = f"DELETE FROM {table_name} WHERE id = '{m.id}'" - #logging.info(query) - c.execute(query) - - # Replace the history with meaningful history - for mh in meaningful_history: - mh.pk = None - mh.save() - - except Exception as e: - logging.exception(f"Could not get history for this link: {e}")