Merge pull request #338 from swozniewski/master

Catch littered status feedback in csv_parser
MatterMiners · Feb 29, 2024 · fda5a8c · fda5a8c
2 parents 4bc109d + bf6b16f
commit fda5a8c
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 0 deletions.
diff --git a/tardis/utilities/utils.py b/tardis/utilities/utils.py
@@ -51,6 +51,11 @@ def csv_parser(
     if skiptrailingspace:
         input_csv = "\n".join((line.strip() for line in input_csv.splitlines()))
 
+    if len(fieldnames) > 1:
+        input_csv = "\n".join(
+            (line for line in input_csv.splitlines() if delimiter in line)
+        )
+
     replacements = replacements or {}
     with StringIO(input_csv) as csv_input:
         csv_reader = csv.DictReader(

diff --git a/tests/utilities_t/test_utils.py b/tests/utilities_t/test_utils.py
@@ -165,6 +165,43 @@ def test_csv_parser_slurm(self):
             ),
         )
 
+    def test_csv_parser_cleaning(self):
+        littered_input = "\n".join(
+            [
+                "Site-Admins wish you happy holidays!",
+                "5545112||PENDING",
+                "Services will be unavailable.",
+                "5545113||PENDING",
+            ]
+        )
+
+        parsed_rows = csv_parser(
+            input_csv=littered_input,
+            fieldnames=("JobId", "Host", "State"),
+            replacements=dict(undefined=None),
+            delimiter="|",
+            skipinitialspace=True,
+            skiptrailingspace=True,
+        )
+
+        self.assertEqual(
+            next(parsed_rows),
+            dict(
+                JobId="5545112",
+                Host="",
+                State="PENDING",
+            ),
+        )
+
+        self.assertEqual(
+            next(parsed_rows),
+            dict(
+                JobId="5545113",
+                Host="",
+                State="PENDING",
+            ),
+        )
+
 
 class TestDisableLogging(TestCase):
     def test_disable_logging(self):