update log time to remove timestamp

src/aap_eda/core/migrations/0056_rulebookprocesslog_log_created_at_and_more.py

@@ -0,0 +1,98 @@
+# Generated by Django 4.2.16 on 2025-01-23 05:05
+
+import logging
+
+from django.db import migrations, models
+
+from aap_eda.core.utils.rulebook_process_logs import (
+    extract_datetime_and_message_from_log_entry,
+)
+
+logger = logging.getLogger(__name__)
+
+CHUNK_SIZE = 1000
+
+
+def up_log_entries(apps, schema_editor):
+    rulebook_process_log_model = apps.get_model("core", "RulebookProcessLog")
+
+    updated_entries = []
+    total_records = 0
+
+    for entry in rulebook_process_log_model.objects.iterator(
+        chunk_size=CHUNK_SIZE
+    ):
+        dt, message = extract_datetime_and_message_from_log_entry(entry.log)
+        entry.log_created_at = dt
+        entry.log = message
+        updated_entries.append(entry)
+
+        # update per CHUNK_SIZE in case of large records in DB
+        if len(updated_entries) >= CHUNK_SIZE:
+            rulebook_process_log_model.objects.bulk_update(
+                updated_entries, ["log", "log_created_at"]
+            )
+            total_records += len(updated_entries)
+            logger.info(f"{total_records} entries are parsed.")
+            updated_entries.clear()
+
+    # update the last trunck
+    if updated_entries:
+        rulebook_process_log_model.objects.bulk_update(
+            updated_entries, ["log", "log_created_at"]
+        )
+        total_records += len(updated_entries)
+
+    logger.info(f"Totally {total_records} entries are parsed.")
+
+
+def down_log_entries(apps, schema_editor):
+    rulebook_process_log_model = apps.get_model("core", "RulebookProcessLog")
+
+    updated_entries = []
+    total_records = 0
+
+    for entry in rulebook_process_log_model.objects.iterator(
+        chunk_size=CHUNK_SIZE
+    ):
+        if entry.log_created_at:
+            timestamp_str = entry.log_created_at.strftime(
+                "%Y-%m-%d %H:%M:%S,%f"
+            ).rstrip("0")
+            message = f"{timestamp_str} {entry.log}"
+            entry.log = message
+
+            updated_entries.append(entry)
+
+        # update per CHUNK_SIZE in case of large records in DB
+        if len(updated_entries) >= CHUNK_SIZE:
+            rulebook_process_log_model.objects.bulk_update(
+                updated_entries, ["log"]
+            )
+            total_records += len(updated_entries)
+            logger.info(f"{total_records} entries are reversed.")
+            updated_entries = []
+
+    # update the last trunck
+    if updated_entries:
+        rulebook_process_log_model.objects.bulk_update(
+            updated_entries, ["log"]
+        )
+        total_records += len(updated_entries)
+
+    logger.info(f"Totally {total_records} entries are reversed.")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0055_activation_created_by_activation_modified_by_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="rulebookprocesslog",
+            name="log_created_at",
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.RunPython(up_log_entries, down_log_entries),
+    ]

src/aap_eda/core/models/rulebook_process.py

@@ -192,6 +192,7 @@ class Meta:
     )
     log = models.TextField()
     log_timestamp = models.BigIntegerField(null=False, default=0)
+    log_created_at = models.DateTimeField(null=True)
 
 
 class RulebookProcessQueue(models.Model):

src/aap_eda/core/utils/rulebook_process_logs.py

@@ -0,0 +1,73 @@
+#  Copyright 2025 Red Hat, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import re
+from datetime import datetime
+from typing import Optional, Tuple
+
+from django.utils.timezone import make_aware
+
+TIMESTAMP_PATTERNS = re.compile(
+    r"^(\*{2} )?"
+    r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})"
+    r"[ ,.](\d{1,6})"
+    r"(?: -)?\s+"  # for '-' or ' '
+)
+
+DATETIME_FORMATS = (
+    "%Y-%m-%d %H:%M:%S,%f",
+    "%Y-%m-%d %H:%M:%S.%f",
+    "%Y-%m-%d %H:%M:%S %f",
+)
+
+
+def extract_datetime_and_message_from_log_entry(
+    log_entry: str,
+) -> Tuple[Optional[datetime], str]:
+    """Extract timestamp and message from a log entry.
+
+    Supported formats:
+    - "2023-01-01 12:34:56,789 message"
+    - "2023-01-01 12:34:56.789 message"
+    - "2023-01-01 12:34:56 789 message"
+    - "** 2023-01-01 12:34:56.789 - message"
+
+    Args:
+        log_entry: Raw log entry string
+
+    Returns:
+        Tuple containing parsed datetime (or None) and cleaned message
+    """
+    match = TIMESTAMP_PATTERNS.match(log_entry)
+    if not match:
+        return None, log_entry.strip()
+
+    prefix, base_time, microseconds = match.groups()
+    timestamp_str = (
+        f"{base_time}{'.' if prefix else ','}{microseconds.ljust(6, '0')}"
+    )
+
+    for fmt in DATETIME_FORMATS:
+        try:
+            dt = datetime.strptime(timestamp_str, fmt)
+            aware_dt = make_aware(dt)
+            break
+        except ValueError:
+            continue
+
+    message = log_entry[match.end() :].strip()
+    if message.startswith("- "):
+        message = message[2:]
+
+    return aware_dt, message

src/aap_eda/services/activation/db_log_handler.py

@@ -20,6 +20,9 @@
 from django.db import IntegrityError
 
 from aap_eda.core import models
+from aap_eda.core.utils.rulebook_process_logs import (
+    extract_datetime_and_message_from_log_entry,
+)
 from aap_eda.services.activation.engine.common import LogHandler
 from aap_eda.services.activation.engine.exceptions import (
     ContainerUpdateLogsError,
@@ -59,15 +62,14 @@ def write(
             lines = [lines]
 
         for line in lines:
-            if timestamp:
-                dt = f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')[:-3]}"
-                line = f"{dt} {line}"
+            dt, message = extract_datetime_and_message_from_log_entry(line)
 
             self.activation_instance_log_buffer.append(
                 models.RulebookProcessLog(
-                    log=line,
+                    log=message,
                     activation_instance_id=self.activation_instance_id,
                     log_timestamp=log_timestamp,
+                    log_created_at=dt,
                 )
             )
             self.line_count += 1

tests/integration/api/test_activation_instance.py

@@ -111,6 +111,7 @@ def test_list_logs_from_activation_instance(
         "id",
         "log",
         "log_timestamp",
+        "log_created_at",
         "activation_instance",
     ]