Merge branch 'main' into bump-python-version

harvester/__init__.py

@@ -1,12 +1,11 @@
 import logging.config
+import os
 
 from dotenv import load_dotenv
 
 from config.logger_config import LOGGING_CONFIG
 from database.interface import HarvesterDBInterface
 
-import os
-
 load_dotenv()
 
 logging.config.dictConfig(LOGGING_CONFIG)
@@ -21,5 +20,5 @@
     "password": os.getenv("HARVEST_SMTP_PASSWORD"),
     "default_sender": os.getenv("HARVEST_SMTP_SENDER"),
     "base_url": os.getenv("REDIRECT_URI").rsplit("/", 1)[0],
-    "recipient": os.getenv("HARVEST_SMTP_RECIPIENT")
+    "recipient": os.getenv("HARVEST_SMTP_RECIPIENT"),
 }

harvester/harvest.py

@@ -16,12 +16,11 @@
 
 sys.path.insert(1, "/".join(os.path.realpath(__file__).split("/")[0:-2]))
 
-from harvester import SMTP_CONFIG
 import smtplib
-from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 
-from harvester import HarvesterDBInterface, db_interface
+from harvester import SMTP_CONFIG, HarvesterDBInterface, db_interface
 from harvester.exceptions import (
     CompareException,
     DCATUSToCKANException,
@@ -73,7 +72,7 @@ class HarvestSource:
             "schema_type",
             "source_type",
             "id",  # db guuid
-            "notification_emails"
+            "notification_emails",
         ],
         repr=False,
     )
@@ -162,7 +161,6 @@ def internal_records_to_id_hash(self, records: list[dict]) -> None:
             )
 
     def get_record_identifier(self, record: dict) -> str:
-
         record_id = "identifier" if self.schema_type.startswith("dcatus") else "url"
 
         if record_id not in record:
@@ -181,7 +179,6 @@ def external_records_to_id_hash(self, records: list[dict]) -> None:
         logger.info("converting harvest records to id: hash")
         for record in records:
             try:
-
                 identifier = self.get_record_identifier(record)
 
                 if self.source_type == "document":
@@ -196,7 +193,7 @@ def external_records_to_id_hash(self, records: list[dict]) -> None:
             except Exception as e:
                 # TODO: do something with 'e'
                 raise ExtractExternalException(
-                    f"{self.title} {self.url} failed to convert to id:hash",
+                    f"{self.name} {self.url} failed to convert to id:hash",
                     self.job_id,
                 )
 
@@ -248,7 +245,7 @@ def compare(self) -> None:
         except Exception as e:
             # TODO: do something with 'e'
             raise CompareException(
-                f"{self.title} {self.url} failed to run compare. exiting.",
+                f"{self.name} {self.url} failed to run compare. exiting.",
                 self.job_id,
             )
 
@@ -435,8 +432,9 @@ def send_notification_emails(self, results: dict) -> None:
 
                 for recipient in all_recipients:
                     msg["To"] = recipient
-                    server.sendmail(SMTP_CONFIG["default_sender"], [recipient],
-                                    msg.as_string())
+                    server.sendmail(
+                        SMTP_CONFIG["default_sender"], [recipient], msg.as_string()
+                    )
                     logger.info(f"Notification email sent to: {recipient}")
         except Exception as e:
             logger.error(f"Failed to send notification email: {e}")
@@ -564,7 +562,6 @@ def status(self, value) -> None:
         self._status = value
 
     def transform(self) -> None:
-
         data = {
             "file": self.metadata["content"],
             "reader": self.reader_map[self.harvest_source.schema_type],
@@ -583,7 +580,11 @@ def transform(self) -> None:
                 self.harvest_source.internal_records_lookup_table[self.identifier],
             )
 
+        record_id = self.harvest_source.internal_records_lookup_table[self.identifier]
         if 200 <= resp.status_code < 300:
+            logger.info(
+                f"successfully transformed record: {self.identifier} db id: {record_id}"
+            )
             self.transformed_data = json.loads(data["writerOutput"])
 
     def validate(self) -> None:

tests/integration/harvest/test_harvest_full_flow.py

@@ -1,5 +1,6 @@
 import json
 from unittest.mock import patch
+
 from harvester.harvest import HarvestSource
 
 
@@ -171,7 +172,7 @@ def test_send_notification_emails(
         interface,
         organization_data,
         source_data_dcatus_single_record,
-        caplog
+        caplog,
     ):
         CKANMock.action.package_create.return_value = {"id": 1234}
         CKANMock.action.package_update = "ok"
@@ -186,8 +187,9 @@ def test_send_notification_emails(
         )
         job_id = harvest_job.id
         harvest_source = HarvestSource(job_id)
-        harvest_source.notification_emails = [source_data_dcatus_single_record[
-                                            "notification_emails"]]
+        harvest_source.notification_emails = [
+            source_data_dcatus_single_record["notification_emails"]
+        ]
 
         results = {"create": 10, "update": 5, "delete": 3, None: 2}
 
@@ -199,4 +201,3 @@ def test_send_notification_emails(
         mock_smtp.side_effect = Exception("SMTP failed")
         harvest_source.send_notification_emails(results)
         assert "Failed to send notification email: SMTP failed" in caplog.text
-

tests/integration/harvest/test_transform.py

@@ -37,6 +37,19 @@ def test_invalid_transform_iso19115_2(
 
         assert test_record.mdt_msgs == expected
 
+        expected_error_msg = (
+            "record failed to transform: structure messages:  \n"
+            "validation messages: WARNING: ISO19115-2 reader: element "
+            "'role' is missing valid nil reason within 'CI_ResponsibleParty'"
+        )
+
+        job_errors = interface.get_harvest_record_errors_by_job(harvest_job.id)
+        assert len(job_errors) == 1
+        assert job_errors[0].message == expected_error_msg
+
+        record = interface.get_harvest_record(job_errors[0].record.id)
+        assert record.status == "error"
+
     def test_valid_transform_iso19115_2(
         self,
         interface,
@@ -50,7 +63,8 @@ def test_valid_transform_iso19115_2(
         harvest_job = interface.add_harvest_job(job_data_waf_iso19115_2)
 
         harvest_source = HarvestSource(harvest_job.id)
-        harvest_source.prepare_external_data()
+        harvest_source.get_record_changes()
+        harvest_source.write_compare_to_db()
 
         name = "http://localhost:80/iso_2_waf/valid_47598.xml"
         test_record = harvest_source.external_records[name]