Handle invalid UTF-8 characters in efibootmgr output

pyanaconda/core/util.py

@@ -170,6 +170,7 @@ def preexec():
 
 
 def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None,
+                 replace_utf_decode_errors=False,
                  log_output=True, binary_output=False, filter_stderr=False,
                  do_preexec=True, env_add=None, user=None):
     """ Run an external program, log the output and return it to the caller
@@ -182,6 +183,7 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None,
         :param stdin: The file object to read stdin from.
         :param stdout: Optional file object to write the output to.
         :param env_prune: environment variable to remove before execution
+        :param replace_utf_decode_errors: whether to substitute � for decoding errors.
         :param log_output: whether to log the output of command
         :param binary_output: whether to treat the output of command as binary data
         :param filter_stderr: whether to exclude the contents of stderr from the returned output
@@ -201,7 +203,10 @@ def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None,
 
         (output_string, err_string) = proc.communicate()
         if not binary_output:
-            output_string = output_string.decode("utf-8")
+            output_string = output_string.decode(
+                "utf-8",
+                errors="strict" if not replace_utf_decode_errors else "replace"
+            )
             if output_string and output_string[-1] != "\n":
                 output_string = output_string + "\n"
 
@@ -261,12 +266,14 @@ def execWithRedirect(command, argv, stdin=None, stdout=None, root='/',
         :return: The return code of the command
     """
     argv = [command] + argv
-    return _run_program(argv, stdin=stdin, stdout=stdout, root=root, env_prune=env_prune,
-                        env_add=env_add, log_output=log_output, binary_output=binary_output,
+    return _run_program(argv, stdin=stdin, stdout=stdout, root=root,
+                        env_prune=env_prune, env_add=env_add,
+                        log_output=log_output, binary_output=binary_output,
                         do_preexec=do_preexec)[0]
 
 
-def execWithCapture(command, argv, stdin=None, root='/', env_prune=None, env_add=None,
+def execWithCapture(command, argv, stdin=None, root='/',
+                    env_prune=None, env_add=None, replace_utf_decode_errors=False,
                     log_output=True, filter_stderr=False, do_preexec=True):
     """ Run an external program and capture standard out and err.
 
@@ -276,15 +283,18 @@ def execWithCapture(command, argv, stdin=None, root='/', env_prune=None, env_add
         :param root: The directory to chroot to before running command.
         :param env_prune: environment variable to remove before execution
         :param env_add: environment variables added for the execution
+        :param replace_utf_decode_errors: whether to ignore decode errors
         :param log_output: Whether to log the output of command
         :param filter_stderr: Whether stderr should be excluded from the returned output
         :param do_preexec: whether to use the preexec function
         :return: The output of the command
     """
     argv = [command] + argv
 
-    return _run_program(argv, stdin=stdin, root=root, log_output=log_output, env_prune=env_prune,
-                        env_add=env_add, filter_stderr=filter_stderr, do_preexec=do_preexec)[1]
+    return _run_program(argv, stdin=stdin, root=root, log_output=log_output,
+                        env_prune=env_prune, env_add=env_add,
+                        replace_utf_decode_errors=replace_utf_decode_errors,
+                        filter_stderr=filter_stderr, do_preexec=do_preexec)[1]
 
 
 def execWithCaptureAsLiveUser(command, argv, stdin=None, root='/', log_output=True,

pyanaconda/modules/storage/bootloader/efi.py

@@ -111,7 +111,9 @@ def add_efi_boot_target(self):
                 self._add_single_efi_boot_target(parent)
 
     def remove_efi_boot_target(self):
-        buf = self.efibootmgr(capture=True)
+        # FIXME: Stop using replace_utf_decode_errors=True once
+        # https://github.com/rhboot/efibootmgr/pull/221/ is merged
+        buf = self.efibootmgr(capture=True, replace_utf_decode_errors=True)
         for line in buf.splitlines():
             try:
                 (slot, _product) = line.split(None, 1)

tests/unit_tests/pyanaconda_tests/core/test_util.py

@@ -60,6 +60,7 @@ def test_run_program_binary(self):
         """Test _run_program with binary output."""
 
         # Echo something that cannot be decoded as utf-8
+        # non utf-8 output should be replaced with U+FFFD
         retcode, output = util._run_program(['echo', '-en', r'\xa0\xa1\xa2'], binary_output=True)
 
         assert retcode == 0
@@ -123,6 +124,21 @@ def test_exec_with_capture_as_live_user(self, mock_get_live_user, mock_start_pro
         assert mock_start_program.call_args.kwargs["env_add"] == {"TEST": "test"}
         assert mock_start_program.call_args.kwargs["env_prune"] == ("TEST_PRUNE",)
 
+    def test_exec_with_capture_non_utf8_handling(self):
+        """Test execWithCapture with non-utf8 output ignored."""
+
+        # Echo something that cannot be decoded as utf-8
+        output = util.execWithCapture(
+            'echo', ['-en', r'Hello world! \xa0\xa1\xa2'],
+            replace_utf_decode_errors=True
+        )
+
+        assert output == 'Hello world! \ufffd\ufffd\ufffd\n'
+
+        # If replace_utf_decode_errors is False (default), the non-utf8 output should raise an Exception
+        with pytest.raises(UnicodeDecodeError):
+            util.execWithCapture('echo', ['-en', r'\xa0\xa1\xa2'])
+
     def test_exec_readlines(self):
         """Test execReadlines."""