Fixed a "UnicodeDecodeError" when listing built-in libraries on macOS…

… with Python 2.7 // Resolve #3370

HISTORY.rst

@@ -34,6 +34,7 @@ PlatformIO Core 4.0
 * Fixed an issue "Import of non-existent variable 'projenv''" when development platform does not call "env.BuildProgram()" (`issue #3315 <https://github.com/platformio/platformio-core/issues/3315>`_)
 * Fixed an issue when invalid CLI command does not return non-zero exit code
 * Fixed an issue when Project Inspector crashes when flash use > 100% (`issue #3368 <https://github.com/platformio/platformio-core/issues/3368>`_)
+* Fixed a "UnicodeDecodeError" when listing built-in libraries on macOS with Python 2.7 (`issue #3370 <https://github.com/platformio/platformio-core/issues/3370>`_)
 
 4.1.0 (2019-11-07)
 ~~~~~~~~~~~~~~~~~~

platformio/commands/home/rpc/handlers/os.py

@@ -14,7 +14,6 @@
 
 from __future__ import absolute_import
 
-import codecs
 import glob
 import os
 import shutil
@@ -66,10 +65,9 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None):
     def request_content(self, uri, data=None, headers=None, cache_valid=None):
         if uri.startswith("http"):
             return self.fetch_content(uri, data, headers, cache_valid)
-        if not os.path.isfile(uri):
-            return None
-        with codecs.open(uri, encoding="utf-8") as fp:
-            return fp.read()
+        if os.path.isfile(uri):
+            return fs.get_file_contents(uri, encoding="utf8")
+        return None
 
     @staticmethod
     def open_url(url):

platformio/fs.py

@@ -49,9 +49,9 @@ def get_source_dir():
     return os.path.dirname(curpath)
 
 
-def get_file_contents(path):
+def get_file_contents(path, encoding=None):
     try:
-        with open(path) as fp:
+        with io.open(path, encoding=encoding) as fp:
             return fp.read()
     except UnicodeDecodeError:
         click.secho(

platformio/package/manifest/parser.py

@@ -66,7 +66,7 @@ def new_from_file(path, remote_url=False):
         if not type_from_uri:
             raise UnknownManifestError("Unknown manifest file type %s" % path)
         return ManifestParserFactory.new(
-            get_file_contents(path), type_from_uri, remote_url
+            get_file_contents(path, encoding="utf8"), type_from_uri, remote_url
         )
 
     @staticmethod
@@ -76,7 +76,7 @@ def new_from_dir(path, remote_url=None):
         type_from_uri = ManifestFileType.from_uri(remote_url) if remote_url else None
         if type_from_uri and os.path.isfile(os.path.join(path, type_from_uri)):
             return ManifestParserFactory.new(
-                get_file_contents(os.path.join(path, type_from_uri)),
+                get_file_contents(os.path.join(path, type_from_uri), encoding="utf8"),
                 type_from_uri,
                 remote_url=remote_url,
                 package_dir=path,
@@ -88,7 +88,7 @@ def new_from_dir(path, remote_url=None):
                 "Unknown manifest file type in %s directory" % path
             )
         return ManifestParserFactory.new(
-            get_file_contents(os.path.join(path, type_from_dir)),
+            get_file_contents(os.path.join(path, type_from_dir), encoding="utf8"),
             type_from_dir,
             remote_url=remote_url,
             package_dir=path,

platformio/project/config.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import glob
+import io
 import json
 import os
 import re
@@ -448,7 +449,7 @@ def save(self, path=None):
         path = path or self.path
         if path in self._instances:
             del self._instances[path]
-        with open(path or self.path, "w+") as fp:
+        with io.open(path or self.path, mode="w+", encoding="utf8") as fp:
             fp.write(CONFIG_HEADER.strip() + "\n\n")
             self._parser.write(fp)
             fp.seek(0)