Fixed an issue when Python 2 does not keep encoding when converting .…

…INO file // Resolve #3393

HISTORY.rst

@@ -12,6 +12,7 @@ PlatformIO Core 4
 * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 <https://github.com/platformio/platformio-core/issues/3400>`_)
 * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server
 * Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 <https://github.com/platformio/platformio-core/issues/3396>`_)
+* Fixed an issue when Python 2 does not keep encoding when converting .INO file (`issue #3393 <https://github.com/platformio/platformio-core/issues/3393>`_)
 
 4.2.1 (2020-02-17)
 ~~~~~~~~~~~~~~~~~~

platformio/builder/tools/piomisc.py

@@ -15,17 +15,19 @@
 from __future__ import absolute_import
 
 import atexit
+import io
 import re
 import sys
 from os import environ, remove, walk
 from os.path import basename, isdir, isfile, join, realpath, relpath, sep
 from tempfile import mkstemp
 
+import click
 from SCons.Action import Action  # pylint: disable=import-error
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 
 from platformio import fs, util
-from platformio.compat import glob_escape
+from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape
 from platformio.managers.core import get_core_package_dir
 from platformio.proc import exec_command
 
@@ -48,6 +50,40 @@ class InoToCPPConverter(object):
     def __init__(self, env):
         self.env = env
         self._main_ino = None
+        self._safe_encoding = None
+
+    def read_safe_contents(self, path):
+        last_exc = None
+        for encoding in (
+            "utf-8",
+            None,
+            get_filesystem_encoding(),
+            get_locale_encoding(),
+            "latin-1",
+        ):
+            try:
+                with io.open(path, encoding=encoding) as fp:
+                    contents = fp.read()
+                    self._safe_encoding = encoding
+                    return contents
+            except UnicodeDecodeError as e:
+                last_exc = e
+                click.secho(
+                    "Unicode decode error has occurred, please remove invalid "
+                    "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
+                    % path,
+                    fg="yellow",
+                    err=True,
+                )
+        if last_exc:
+            raise last_exc
+        return None
+
+    def write_safe_contents(self, path, contents):
+        with io.open(
+            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
+        ) as fp:
+            return fp.write(contents)
 
     def is_main_node(self, contents):
         return self.DETECTMAIN_RE.search(contents)
@@ -62,7 +98,7 @@ def merge(self, nodes):
         assert nodes
         lines = []
         for node in nodes:
-            contents = fs.get_file_contents(node.get_path())
+            contents = self.read_safe_contents(node.get_path())
             _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
             if self.is_main_node(contents):
                 lines = _lines + lines
@@ -78,16 +114,14 @@ def merge(self, nodes):
     def process(self, contents):
         out_file = self._main_ino + ".cpp"
         assert self._gcc_preprocess(contents, out_file)
-        contents = fs.get_file_contents(out_file)
+        contents = self.read_safe_contents(out_file)
         contents = self._join_multiline_strings(contents)
-        fs.write_file_contents(
-            out_file, self.append_prototypes(contents), errors="backslashreplace"
-        )
+        self.write_safe_contents(out_file, self.append_prototypes(contents))
         return out_file
 
     def _gcc_preprocess(self, contents, out_file):
         tmp_path = mkstemp()[1]
-        fs.write_file_contents(tmp_path, contents, errors="backslashreplace")
+        self.write_safe_contents(tmp_path, contents)
         self.env.Execute(
             self.env.VerboseAction(
                 '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(

platformio/fs.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import io
 import json
 import os
 import re
@@ -49,30 +48,6 @@ def get_source_dir():
     return os.path.dirname(curpath)
 
 
-def get_file_contents(path, encoding=None):
-    try:
-        with io.open(path, encoding=encoding) as fp:
-            return fp.read()
-    except UnicodeDecodeError:
-        click.secho(
-            "Unicode decode error has occurred, please remove invalid "
-            "(non-ASCII or non-UTF8) characters from %s file" % path,
-            fg="yellow",
-            err=True,
-        )
-        with io.open(path, encoding="latin-1") as fp:
-            return fp.read()
-
-
-def write_file_contents(path, contents, errors=None):
-    try:
-        with open(path, "w") as fp:
-            return fp.write(contents)
-    except UnicodeEncodeError:
-        with io.open(path, "w", encoding="latin-1", errors=errors) as fp:
-            return fp.write(contents)
-
-
 def load_json(file_path):
     try:
         with open(file_path, "r") as f:
@@ -102,11 +77,14 @@ def ensure_udev_rules():
     from platformio.util import get_systype  # pylint: disable=import-outside-toplevel
 
     def _rules_to_set(rules_path):
-        return set(
-            l.strip()
-            for l in get_file_contents(rules_path).split("\n")
-            if l.strip() and not l.startswith("#")
-        )
+        result = set()
+        with open(rules_path) as fp:
+            for line in fp.readlines():
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                result.add(line)
+        return result
 
     if "linux" not in get_systype():
         return None