From 7bc170a53e20fcd155b55c90c2a3454731fe7bc9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 11 Sep 2020 21:16:18 +0300 Subject: [PATCH 01/51] Fixed an issue with "KeyError: 'versions'" when dependency does not exist in the registry // Resolve #3666 --- HISTORY.rst | 5 +++++ platformio/__init__.py | 2 +- platformio/clients/http.py | 9 ++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c1668e51a3..9c1116ec5c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,11 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +5.0.2 (2020-09-??) +~~~~~~~~~~~~~~~~~~ + +- Fixed an issue with "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) + 5.0.1 (2020-09-10) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__init__.py b/platformio/__init__.py index 145ad0cff4..7159c500ea 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, 1) +VERSION = (5, 0, "2a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 4d59bcaa62..1e22ca975f 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -133,9 +133,7 @@ def send_request(self, method, path, **kwargs): def fetch_json_data(self, method, path, **kwargs): cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None if not cache_valid: - return self.raise_error_from_response( - self.send_request(method, path, **kwargs) - ) + return self._parse_json_response(self.send_request(method, path, **kwargs)) cache_key = ContentCache.key_from_args( method, path, kwargs.get("params"), kwargs.get("data") ) @@ -144,11 +142,12 @@ def fetch_json_data(self, method, path, **kwargs): if result is not None: return json.loads(result) response = self.send_request(method, path, **kwargs) + data = self._parse_json_response(response) cc.set(cache_key, response.text, cache_valid) - return self.raise_error_from_response(response) + return data @staticmethod - def raise_error_from_response(response, expected_codes=(200, 201, 202)): + def _parse_json_response(response, expected_codes=(200, 201, 202)): if response.status_code in expected_codes: try: return response.json() From 687c339f20f4499caac7bf1f920cf78077914b2f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 12 Sep 2020 23:20:46 +0300 Subject: [PATCH 02/51] Fixed a "PermissionError: [WinError 5]" on Windows when external repository is used with `lib_deps` option // Resolve #3664 --- HISTORY.rst | 3 ++- platformio/package/manager/_install.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9c1116ec5c..4503cacbec 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,7 +11,8 @@ PlatformIO Core 5 5.0.2 (2020-09-??) ~~~~~~~~~~~~~~~~~~ -- Fixed an issue with "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) +- Fixed a "PermissionError: [WinError 5]" on Windows when external repository is used with `lib_deps `__ option (`issue #3664 `_) +- Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) 5.0.1 (2020-09-10) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 9d82d6fefd..6cdccf0948 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -152,7 +152,10 @@ def install_from_url(self, url, spec, checksum=None, silent=False): return self._install_tmp_pkg(pkg_item) finally: if os.path.isdir(tmp_dir): - fs.rmtree(tmp_dir) + try: + shutil.rmtree(tmp_dir) + except: # pylint: disable=bare-except + pass def _install_tmp_pkg(self, tmp_pkg): assert isinstance(tmp_pkg, PackageItem) @@ -213,10 +216,10 @@ def _cleanup_dir(path): # move existing into the new place pkg_dir = os.path.join(self.package_dir, target_dirname) _cleanup_dir(pkg_dir) - shutil.move(dst_pkg.path, pkg_dir) + shutil.copytree(dst_pkg.path, pkg_dir, symlinks=True) # move new source to the destination location _cleanup_dir(dst_pkg.path) - shutil.move(tmp_pkg.path, dst_pkg.path) + shutil.copytree(tmp_pkg.path, dst_pkg.path, symlinks=True) return PackageItem(dst_pkg.path) if action == "detach-new": @@ -233,10 +236,10 @@ def _cleanup_dir(path): ) pkg_dir = os.path.join(self.package_dir, target_dirname) _cleanup_dir(pkg_dir) - shutil.move(tmp_pkg.path, pkg_dir) + shutil.copytree(tmp_pkg.path, pkg_dir, symlinks=True) return PackageItem(pkg_dir) # otherwise, overwrite existing _cleanup_dir(dst_pkg.path) - shutil.move(tmp_pkg.path, dst_pkg.path) + shutil.copytree(tmp_pkg.path, dst_pkg.path, symlinks=True) return PackageItem(dst_pkg.path) From eb09af06edcfc0b53e4fb5fd84966170d76e8925 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 12 Sep 2020 23:21:33 +0300 Subject: [PATCH 03/51] Bump version to 5.0.2a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 7159c500ea..89734d6acc 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "2a1") +VERSION = (5, 0, "2a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f656d19ed5dddad2026b23d5ad885b2bc818d088 Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 14 Sep 2020 22:31:11 +0300 Subject: [PATCH 04/51] Docs: Added new section about Arduino STM32L0 core --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 9bbb02295a..f358a74a8f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9bbb02295a7f5ce325f1ed90a8b549fb81ce3857 +Subproject commit f358a74a8fadcc6fac1ae7290ea101dc643c2d4d From bf57b777bf022501b7444b9eac313542e369eeeb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 16 Sep 2020 19:33:55 +0300 Subject: [PATCH 05/51] Docs: Update docs for PlatformIO IDE 2.0 for VSCode --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index f358a74a8f..721e27143b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f358a74a8fadcc6fac1ae7290ea101dc643c2d4d +Subproject commit 721e27143bff46401837e8dfbd5e94173355a598 From 2b5e590819db78917762ded3b81e99b713b4ada6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Sep 2020 19:21:12 +0300 Subject: [PATCH 06/51] Docs: Explain how to install custom Python packages // Resolve #3673 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 721e27143b..24382755ab 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 721e27143bff46401837e8dfbd5e94173355a598 +Subproject commit 24382755ab8a397b3156711e6222def57884838e From 1e0ca8f79c043eb7b5abac5d97e047ce69e3458d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Sep 2020 20:40:11 +0300 Subject: [PATCH 07/51] Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies // Resolve #3669 --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 4 ++-- platformio/compat.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4503cacbec..a57d1ab82a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,7 @@ PlatformIO Core 5 - Fixed a "PermissionError: [WinError 5]" on Windows when external repository is used with `lib_deps `__ option (`issue #3664 `_) - Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) +- Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 `_) 5.0.1 (2020-09-10) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index aac4742621..4e1ca9ca88 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -27,7 +27,7 @@ from SCons.Script import SConscript # pylint: disable=import-error from platformio import __version__, fs -from platformio.compat import string_types +from platformio.compat import MACOS, string_types from platformio.package.version import pepver_to_semver SRC_HEADER_EXT = ["h", "hpp"] @@ -69,7 +69,7 @@ def BuildProgram(env): if ( env.get("LIBS") and env.GetCompilerType() == "gcc" - and env.PioPlatform().is_embedded() + and (env.PioPlatform().is_embedded() or not MACOS) ): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") diff --git a/platformio/compat.py b/platformio/compat.py index 59362d01b3..bf88c6813d 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -26,6 +26,7 @@ PY2 = sys.version_info[0] == 2 CYGWIN = sys.platform.startswith("cygwin") WINDOWS = sys.platform.startswith("win") +MACOS = sys.platform.startswith("darwin") def get_filesystem_encoding(): From a384411a281653be02b9109ea32ddc036d3cd899 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Sep 2020 20:41:10 +0300 Subject: [PATCH 08/51] Bump version to 5.0.2b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 89734d6acc..825fcb6941 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "2a2") +VERSION = (5, 0, "2b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 2370e16f1bb024759babf3b0c09a891024e2dd4f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 19 Sep 2020 19:29:51 +0300 Subject: [PATCH 09/51] Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses // Resolve #3677 --- HISTORY.rst | 1 + platformio/package/manager/base.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a57d1ab82a..9b4bbf31f3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,7 @@ PlatformIO Core 5 - Fixed a "PermissionError: [WinError 5]" on Windows when external repository is used with `lib_deps `__ option (`issue #3664 `_) - Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) - Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 `_) +- Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses (`issue #3677 `_) 5.0.1 (2020-09-10) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index dcfe03f0c6..3664a72e2b 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -51,7 +51,7 @@ class BasePackageManager( # pylint: disable=too-many-public-methods def __init__(self, pkg_type, package_dir): self.pkg_type = pkg_type - self.package_dir = self.ensure_dir_exists(package_dir) + self.package_dir = package_dir self._MEMORY_CACHE = {} self._lockfile = None @@ -62,7 +62,9 @@ def __init__(self, pkg_type, package_dir): def lock(self): if self._lockfile: return + self.ensure_dir_exists(os.path.dirname(self.package_dir)) self._lockfile = LockFile(self.package_dir) + self.ensure_dir_exists(self.package_dir) self._lockfile.acquire() def unlock(self): @@ -91,10 +93,7 @@ def is_system_compatible(value): @staticmethod def ensure_dir_exists(path): if not os.path.isdir(path): - try: - os.makedirs(path) - except: # pylint: disable=bare-except - pass + os.makedirs(path) assert os.path.isdir(path) return path @@ -193,6 +192,9 @@ def build_metadata(self, pkg_dir, spec, vcs_revision=None): return metadata def get_installed(self): + if not os.path.isdir(self.package_dir): + return [] + cache_key = "get_installed" if self.memcache_get(cache_key): return self.memcache_get(cache_key) From 21c12030d57764623b55a5d29748f0f5203b97c5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 19 Sep 2020 19:30:40 +0300 Subject: [PATCH 10/51] Bump version to 5.0.2b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 825fcb6941..4b9abaf31b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "2b1") +VERSION = (5, 0, "2b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 14de3e79c5ab78428b1ab3062d5af30fa31c35c0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 26 Sep 2020 22:52:43 +0300 Subject: [PATCH 11/51] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 24382755ab..b6c44938e2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 24382755ab8a397b3156711e6222def57884838e +Subproject commit b6c44938e24665f9d67bbee6d65bb47ccf2637e0 From 9c20ab81cb68f1ffb7a8cac22ce95c4c797643ec Mon Sep 17 00:00:00 2001 From: Michele Campeotto Date: Fri, 2 Oct 2020 12:28:02 +0200 Subject: [PATCH 12/51] fix quoting of defines in ccls template (#3692) --- platformio/ide/tpls/vim/.ccls.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/ide/tpls/vim/.ccls.tpl b/platformio/ide/tpls/vim/.ccls.tpl index b6d7d55aa4..39e29f8735 100644 --- a/platformio/ide/tpls/vim/.ccls.tpl +++ b/platformio/ide/tpls/vim/.ccls.tpl @@ -18,5 +18,5 @@ clang % end % for define in defines: --D{{ define }} +-D{{ !define }} % end From d835f52a18f91cb722d06aa208d9f57729587dc3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 10 Oct 2020 20:48:56 +0300 Subject: [PATCH 13/51] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b6c44938e2..30129f2035 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b6c44938e24665f9d67bbee6d65bb47ccf2637e0 +Subproject commit 30129f20357e6be989497b0afef3f0e678d153f8 From 1cb7764b0e370098a297573427d73f3249f933a6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 14 Oct 2020 17:53:41 +0300 Subject: [PATCH 14/51] Highlight PlatformIO Labs Technology --- README.rst | 26 ++++++++++---------------- docs | 2 +- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/README.rst b/README.rst index bdd2858b7f..7d4b9c65cd 100644 --- a/README.rst +++ b/README.rst @@ -17,21 +17,19 @@ PlatformIO :target: https://pypi.python.org/pypi/platformio/ :alt: License .. image:: https://img.shields.io/badge/PlatformIO-Community-orange.svg - :alt: Community Forums - :target: https://community.platformio.org?utm_source=github&utm_medium=core + :alt: Community Labs + :target: https://piolabs.com/?utm_source=github&utm_medium=core **Quick Links:** `Web `_ | `PlatformIO IDE `_ | `Project Examples `__ | `Docs `_ | `Donate `_ | -`Contact Us `_ +`Contact Us `_ -**Social:** `Twitter `_ | -`LinkedIn `_ | +**Social:** `LinkedIn `_ | +`Twitter `_ | `Facebook `_ | -`Hackaday `_ | -`Bintray `_ | `Community `_ .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png @@ -51,20 +49,16 @@ Get Started ----------- * `What is PlatformIO? `_ - -Instruments ------------ - * `PlatformIO IDE `_ * `PlatformIO Core (CLI) `_ -* `Library Management `_ * `Project Examples `__ + +Solutions +--------- + +* `Library Management `_ * `Desktop IDEs Integration `_ * `Continuous Integration `_ -* `Advanced Scripting API `_ - -Professional ------------- * `Debugging `_ * `Unit Testing `_ diff --git a/docs b/docs index 30129f2035..9bb1446ce6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 30129f20357e6be989497b0afef3f0e678d153f8 +Subproject commit 9bb1446ce6377add99d8f5ed279ddf47d4726c5d From a24bab0a27c69e7c5009c0c4260a5590ba019c34 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 14 Oct 2020 17:55:45 +0300 Subject: [PATCH 15/51] Fix badge --- README.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 7d4b9c65cd..c137fe436b 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ PlatformIO .. image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg :target: https://pypi.python.org/pypi/platformio/ :alt: License -.. image:: https://img.shields.io/badge/PlatformIO-Community-orange.svg +.. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg :alt: Community Labs :target: https://piolabs.com/?utm_source=github&utm_medium=core @@ -30,7 +30,7 @@ PlatformIO **Social:** `LinkedIn `_ | `Twitter `_ | `Facebook `_ | -`Community `_ +`Community Forums `_ .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png :target: https://platformio.org?utm_source=github&utm_medium=core @@ -60,6 +60,8 @@ Solutions * `Desktop IDEs Integration `_ * `Continuous Integration `_ +**Advanced** + * `Debugging `_ * `Unit Testing `_ * `Static Code Analysis `_ From 5b74c8a942c047a9eb7d72b8cd36d86bd6e93db5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 14 Oct 2020 19:53:30 +0300 Subject: [PATCH 16/51] Minor fixes --- .github/workflows/examples.yml | 6 +++--- docs | 2 +- platformio/__init__.py | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index e3bb201f3a..b0c8b1bd5e 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -26,7 +26,7 @@ jobs: - name: Run on Linux if: startsWith(matrix.os, 'ubuntu') env: - PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,intel_mcs51,aceinna_imu" + PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,intel_mcs51,aceinna_imu" run: | # ChipKIT issue: install 32-bit support for GCC PIC32 sudo apt-get install libc6-i386 @@ -40,7 +40,7 @@ jobs: - name: Run on macOS if: startsWith(matrix.os, 'macos') env: - PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,microchippic32,gd32v,nuclei" + PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,microchippic32,gd32v,nuclei" run: | df -h tox -e testexamples @@ -50,7 +50,7 @@ jobs: env: PLATFORMIO_CORE_DIR: C:/pio PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH - PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap" + PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,riscv_gap" run: | tox -e testexamples diff --git a/docs b/docs index 9bb1446ce6..0dfd73ab92 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9bb1446ce6377add99d8f5ed279ddf47d4726c5d +Subproject commit 0dfd73ab922313911bed62596f9a0cd93a61aa9d diff --git a/platformio/__init__.py b/platformio/__init__.py index 4b9abaf31b..4cb89e406c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -57,8 +57,7 @@ } __check_internet_hosts__ = [ - "140.82.118.3", # Github.com - "35.231.145.151", # Gitlab.com + "185.199.110.153", # Github.com "88.198.170.159", # platformio.org "github.com", "platformio.org", From 411bf1107dd37bc3fd1e86a69399ce27fa823706 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 14 Oct 2020 22:33:04 +0300 Subject: [PATCH 17/51] Disable "lattice_ice40" examples for macOS --- .github/workflows/examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index b0c8b1bd5e..a2239d5bbe 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -40,7 +40,7 @@ jobs: - name: Run on macOS if: startsWith(matrix.os, 'macos') env: - PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,microchippic32,gd32v,nuclei" + PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,microchippic32,gd32v,nuclei,lattice_ice40" run: | df -h tox -e testexamples From a8709812664153b851b6a29ffe88536d01b21d54 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 14 Oct 2020 23:21:59 +0300 Subject: [PATCH 18/51] Docs: Fix custom "platform_packages" for ESP8266/32 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 0dfd73ab92..13aba90ab0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0dfd73ab922313911bed62596f9a0cd93a61aa9d +Subproject commit 13aba90ab0425f2d67b2ac7fd609fce4c3634839 From ee847e03a633a2b59cbf003a71d9fda3e1c34b52 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 16 Oct 2020 14:23:10 +0300 Subject: [PATCH 19/51] Fix an issue with "'NoneType' object has no attribute 'status_code'" --- platformio/clients/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index f8afdfab1b..795a049545 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -142,6 +142,6 @@ def get_package(self, type_, owner, name, version=None): cache_valid="1h", ) except HTTPClientError as e: - if e.response.status_code == 404: + if e.response and e.response.status_code == 404: return None raise e From 428f46fafe082f181a9264f08c8b2a1613507b2c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 16 Oct 2020 17:05:19 +0300 Subject: [PATCH 20/51] Typo test --- platformio/clients/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 795a049545..b0387ce350 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -142,6 +142,6 @@ def get_package(self, type_, owner, name, version=None): cache_valid="1h", ) except HTTPClientError as e: - if e.response and e.response.status_code == 404: + if e.response is not None and e.response.status_code == 404: return None raise e From 23e95965069b51c64c5aa74b6593c1d268de3e46 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 20 Oct 2020 21:06:53 +0300 Subject: [PATCH 21/51] Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry // Resolve #3700 --- HISTORY.rst | 1 + platformio/package/manager/core.py | 104 ++++++++++++++++++++--------- platformio/util.py | 8 +-- 3 files changed, 77 insertions(+), 36 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9b4bbf31f3..2df285791d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ PlatformIO Core 5 5.0.2 (2020-09-??) ~~~~~~~~~~~~~~~~~~ +- Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry (`issue #3700 `_) - Fixed a "PermissionError: [WinError 5]" on Windows when external repository is used with `lib_deps `__ option (`issue #3664 `_) - Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) - Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 `_) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index fd1cd856b3..a324beb466 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -14,12 +14,14 @@ import json import os +import shutil import subprocess import sys from datetime import date from platformio import __core_packages__, exception, fs, util from platformio.compat import PY2 +from platformio.package.exception import UnknownPackageError from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import get_pythonexe_path @@ -74,9 +76,17 @@ def inject_contrib_pysite(verify_openssl=False): # pylint: disable=import-outside-toplevel from site import addsitedir - contrib_pysite_dir = get_core_package_dir("contrib-pysite") + try: + contrib_pysite_dir = get_core_package_dir("contrib-pysite") + except UnknownPackageError: + pm = ToolPackageManager() + contrib_pysite_dir = build_contrib_pysite_package( + os.path.join(pm.package_dir, "contrib-pysite") + ) + if contrib_pysite_dir in sys.path: return True + addsitedir(contrib_pysite_dir) sys.path.insert(0, contrib_pysite_dir) @@ -87,34 +97,31 @@ def inject_contrib_pysite(verify_openssl=False): # pylint: disable=import-error,unused-import,unused-variable from OpenSSL import SSL except: # pylint: disable=bare-except - build_contrib_pysite_deps(get_core_package_dir("contrib-pysite")) + build_contrib_pysite_package(contrib_pysite_dir) return True -def build_contrib_pysite_deps(target_dir): +def build_contrib_pysite_package(target_dir, with_metadata=True): + systype = util.get_systype() if os.path.isdir(target_dir): fs.rmtree(target_dir) os.makedirs(target_dir) # build dependencies - pythonexe = get_pythonexe_path() + args = [ + get_pythonexe_path(), + "-m", + "pip", + "install", + "--no-compile", + "-t", + target_dir, + ] + if "linux" in systype: + args.extend(["--no-binary", ":all:"]) for dep in get_contrib_pysite_deps(): - subprocess.check_call( - [ - pythonexe, - "-m", - "pip", - "install", - # "--no-cache-dir", - "--no-compile", - "--no-binary", - ":all:", - "-t", - target_dir, - dep, - ] - ) + subprocess.check_call(args + [dep]) # build manifests with open(os.path.join(target_dir, "package.json"), "w") as fp: @@ -127,18 +134,55 @@ def build_contrib_pysite_deps(target_dir): sys.version_info.minor, date.today().strftime("%y%m%d"), ), - system=util.get_systype(), + system=list( + set([systype, "linux_armv6l", "linux_armv7l", "linux_armv8l"]) + ) + if systype.startswith("linux_arm") + else systype, + description="Extra Python package for PlatformIO Core", + keywords=["platformio", "platformio-core"], + homepage="https://docs.platformio.org/page/core/index.html", + repository={ + "type": "git", + "url": "https://github.com/platformio/platformio-core", + }, ), fp, ) - pm = ToolPackageManager() - pkg = PackageItem(target_dir) - pkg.metadata = pm.build_metadata( - target_dir, PackageSpec(owner="platformio", name="contrib-pysite") - ) - pkg.dump_meta() - return True + # generate package metadata + if with_metadata: + pm = ToolPackageManager() + pkg = PackageItem(target_dir) + pkg.metadata = pm.build_metadata( + target_dir, PackageSpec(owner="platformio", name="contrib-pysite") + ) + pkg.dump_meta() + + # remove unused files + shutil.rmtree(os.path.join(target_dir, "autobahn", "xbr", "contracts")) + for root, dirs, files in os.walk(target_dir): + for t in ("_test", "test", "tests"): + if t in dirs: + shutil.rmtree(os.path.join(root, t)) + for name in files: + if name.endswith((".chm", ".pyc")): + os.remove(os.path.join(root, name)) + + # apply patches + with open( + os.path.join(target_dir, "autobahn", "twisted", "__init__.py"), "r+" + ) as fp: + contents = fp.read() + contents = contents.replace( + "from autobahn.twisted.wamp import ApplicationSession", + "# from autobahn.twisted.wamp import ApplicationSession", + ) + fp.seek(0) + fp.truncate() + fp.write(contents) + + return target_dir def get_contrib_pysite_deps(): @@ -148,7 +192,7 @@ def get_contrib_pysite_deps(): twisted_version = "19.10.0" if PY2 else "20.3.0" result = [ "twisted == %s" % twisted_version, - "autobahn == %s" % ("19.11.2" if PY2 else "20.4.3"), + "autobahn == %s" % ("19.11.2" if PY2 else "20.7.1"), "json-rpc == 1.13.0", ] @@ -169,8 +213,8 @@ def get_contrib_pysite_deps(): result.append("pypiwin32 == 223") # workaround for twisted wheels twisted_wheel = ( - "https://download.lfd.uci.edu/pythonlibs/g5apjq5m/Twisted-" - "%s-cp%s-cp%sm-win%s.whl" + "https://download.lfd.uci.edu/pythonlibs/x2tqcw5k/Twisted-" + "%s-cp%s-cp%s-win%s.whl" % ( twisted_version, py_version, diff --git a/platformio/util.py b/platformio/util.py index 6da4708b83..e777d10a02 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -19,7 +19,6 @@ import os import platform import re -import sys import time from functools import wraps from glob import glob @@ -167,12 +166,9 @@ def get_mdns_services(): try: import zeroconf except ImportError: - from site import addsitedir - from platformio.package.manager.core import get_core_package_dir + from platformio.package.manager.core import inject_contrib_pysite - contrib_pysite_dir = get_core_package_dir("contrib-pysite") - addsitedir(contrib_pysite_dir) - sys.path.insert(0, contrib_pysite_dir) + inject_contrib_pysite() import zeroconf # pylint: disable=import-outside-toplevel class mDNSListener(object): From 5d87fb875760a88c0e82e8be9b069d9fc8609530 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 22 Oct 2020 19:07:44 +0300 Subject: [PATCH 22/51] Add "Articles" section to Zephyr RTOS --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 13aba90ab0..7eb9279b2c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 13aba90ab0425f2d67b2ac7fd609fce4c3634839 +Subproject commit 7eb9279b2c0683fa2e0a808788684b2c67bbe309 From afd79f4655534d092531fc2f6447bafc5afb54c1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 22 Oct 2020 19:08:20 +0300 Subject: [PATCH 23/51] Improve initiating manifest parser from a package archive --- platformio/package/manifest/parser.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index b492bb7957..4dd4476de0 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -119,12 +119,13 @@ def new_from_archive(path): assert path.endswith("tar.gz") with tarfile.open(path, mode="r:gz") as tf: for t in sorted(ManifestFileType.items().values()): - try: - return ManifestParserFactory.new( - tf.extractfile(t).read().decode(), t - ) - except KeyError: - pass + for member in (t, "./" + t): + try: + return ManifestParserFactory.new( + tf.extractfile(member).read().decode(), t + ) + except KeyError: + pass raise UnknownManifestError("Unknown manifest file type in %s archive" % path) @staticmethod From 7230556d1b8d3f68c48d7a5e90b7423a72ec0a27 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Oct 2020 18:23:28 +0200 Subject: [PATCH 24/51] Move extra IDE data to "extra" section --- platformio/builder/main.py | 1 + platformio/builder/tools/pioide.py | 11 +++++++---- platformio/commands/debug/helpers.py | 13 ++++++++----- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 1547cf99aa..b1e8ffbd64 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -78,6 +78,7 @@ PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), PYTHONEXE=get_pythonexe_path(), + IDE_EXTRA_DATA={}, ) if not int(ARGUMENTS.get("PIOVERBOSE", 0)): diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index c21b150022..b9fab37546 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -164,14 +164,17 @@ def DumpIDEData(env, globalenv): "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")), "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), "prog_path": env.subst("$PROG_PATH"), - "flash_extra_images": [ - {"offset": item[0], "path": env.subst(item[1])} - for item in env.get("FLASH_EXTRA_IMAGES", []) - ], "svd_path": _get_svd_path(env), "compiler_type": env.GetCompilerType(), "targets": globalenv.DumpTargets(), + "extra": dict( + flash_images=[ + {"offset": item[0], "path": env.subst(item[1])} + for item in env.get("FLASH_EXTRA_IMAGES", []) + ] + ), } + data["extra"].update(env.get("IDE_EXTRA_DATA", {})) env_ = env.Clone() # https://github.com/platformio/platformio-atom-ide/issues/34 diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 657e8c48ea..445fe57d1b 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -195,13 +195,16 @@ def _cleanup_cmds(items): def configure_esp32_load_cmds(debug_options, configuration): + """ + DEPRECATED: Moved to ESP32 dev-platform + See platform.py::configure_debug_options + """ + flash_images = configuration.get("extra", {}).get("flash_images") ignore_conds = [ debug_options["load_cmds"] != ["load"], "xtensa-esp32" not in configuration.get("cc_path", ""), - not configuration.get("flash_extra_images"), - not all( - [isfile(item["path"]) for item in configuration.get("flash_extra_images")] - ), + not flash_images, + not all([isfile(item["path"]) for item in flash_images]), ] if any(ignore_conds): return debug_options["load_cmds"] @@ -210,7 +213,7 @@ def configure_esp32_load_cmds(debug_options, configuration): 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( path=fs.to_unix_path(item["path"]), offset=item["offset"] ) - for item in configuration.get("flash_extra_images") + for item in flash_images ] mon_cmds.append( 'monitor program_esp32 "{%s.bin}" 0x10000 verify' From b8cc867ba46395bf144eb5e313ce6f1305706ab4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Oct 2020 18:24:46 +0200 Subject: [PATCH 25/51] Allow dev-platform to provide extra debug configuration using BsePlatform::configure_debug_options API --- platformio/commands/debug/command.py | 50 ++++++++++++++++++---------- platformio/commands/debug/helpers.py | 15 +-------- platformio/platform/base.py | 3 ++ 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index fc83405c0e..2ff969329e 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -24,7 +24,10 @@ from platformio import app, exception, fs, proc from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError +from platformio.commands.platform import platform_install as cmd_platform_install from platformio.package.manager.core import inject_contrib_pysite +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project, load_project_ide_data @@ -73,18 +76,29 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro env_options = config.items(env=env_name, as_dict=True) if not set(env_options.keys()) >= set(["platform", "board"]): raise ProjectEnvsNotAvailableError() - debug_options = helpers.validate_debug_options(ctx, env_options) + + try: + platform = PlatformFactory.new(env_options["platform"]) + except UnknownPlatform: + ctx.invoke( + cmd_platform_install, + platforms=[env_options["platform"]], + skip_default_package=True, + ) + platform = PlatformFactory.new(env_options["platform"]) + + debug_options = helpers.configure_initial_debug_options(platform, env_options) assert debug_options if not interface: return helpers.predebug_project(ctx, project_dir, env_name, False, verbose) - configuration = load_project_ide_data(project_dir, env_name) - if not configuration: - raise DebugInvalidOptionsError("Could not load debug configuration") + ide_data = load_project_ide_data(project_dir, env_name) + if not ide_data: + raise DebugInvalidOptionsError("Could not load a build configuration") if "--version" in __unprocessed: - result = proc.exec_command([configuration["gdb_path"], "--version"]) + result = proc.exec_command([ide_data["gdb_path"], "--version"]) if result["returncode"] == 0: return click.echo(result["out"]) raise exception.PlatformioException("\n".join([result["out"], result["err"]])) @@ -99,23 +113,25 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro nl=False, ) - debug_options["load_cmds"] = helpers.configure_esp32_load_cmds( - debug_options, configuration - ) + try: + debug_options = platform.configure_debug_options(debug_options, ide_data) + except NotImplementedError: + # legacy for ESP32 dev-platform <=2.0.0 + debug_options["load_cmds"] = helpers.configure_esp32_load_cmds( + debug_options, ide_data + ) rebuild_prog = False preload = debug_options["load_cmds"] == ["preload"] load_mode = debug_options["load_mode"] if load_mode == "always": - rebuild_prog = preload or not helpers.has_debug_symbols( - configuration["prog_path"] - ) + rebuild_prog = preload or not helpers.has_debug_symbols(ide_data["prog_path"]) elif load_mode == "modified": rebuild_prog = helpers.is_prog_obsolete( - configuration["prog_path"] - ) or not helpers.has_debug_symbols(configuration["prog_path"]) + ide_data["prog_path"] + ) or not helpers.has_debug_symbols(ide_data["prog_path"]) else: - rebuild_prog = not isfile(configuration["prog_path"]) + rebuild_prog = not isfile(ide_data["prog_path"]) if preload or (not rebuild_prog and load_mode != "always"): # don't load firmware through debug server @@ -139,9 +155,9 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro # save SHA sum of newly created prog if load_mode == "modified": - helpers.is_prog_obsolete(configuration["prog_path"]) + helpers.is_prog_obsolete(ide_data["prog_path"]) - if not isfile(configuration["prog_path"]): + if not isfile(ide_data["prog_path"]): raise DebugInvalidOptionsError("Program/firmware is missed") # run debugging client @@ -151,7 +167,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro from platformio.commands.debug.process.client import GDBClient, reactor client = GDBClient(project_dir, __unprocessed, debug_options, env_options) - client.spawn(configuration["gdb_path"], configuration["prog_path"]) + client.spawn(ide_data["gdb_path"], ide_data["prog_path"]) signal.signal(signal.SIGINT, lambda *args, **kwargs: None) reactor.run() diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 445fe57d1b..62ce4f7394 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -23,11 +23,8 @@ from platformio import fs, util from platformio.commands import PlatformioCLI from platformio.commands.debug.exception import DebugInvalidOptionsError -from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.run.command import cli as cmd_run from platformio.compat import is_bytes -from platformio.platform.exception import UnknownPlatform -from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.options import ProjectOptions @@ -89,21 +86,11 @@ def predebug_project(ctx, project_dir, env_name, preload, verbose): time.sleep(5) -def validate_debug_options(cmd_ctx, env_options): +def configure_initial_debug_options(platform, env_options): def _cleanup_cmds(items): items = ProjectConfig.parse_multi_values(items) return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] - try: - platform = PlatformFactory.new(env_options["platform"]) - except UnknownPlatform: - cmd_ctx.invoke( - cmd_platform_install, - platforms=[env_options["platform"]], - skip_default_package=True, - ) - platform = PlatformFactory.new(env_options["platform"]) - board_config = platform.board_config(env_options["board"]) tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) tool_settings = board_config.get("debug", {}).get("tools", {}).get(tool_name, {}) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index b214da5f54..81d9b085ef 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -203,6 +203,9 @@ def configure_default_packages(self, options, targets): elif "nobuild" in targets and opts.get("type") != "framework": self.packages[name]["optional"] = True + def configure_debug_options(self, initial_debug_options, ide_data): + raise NotImplementedError + def get_lib_storages(self): storages = {} for opts in (self.frameworks or {}).values(): From 0ecfe8105f4f96350228296d94c7f769080c96fb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Oct 2020 22:24:05 +0200 Subject: [PATCH 26/51] Docs: Unify CLI commands to use "pio" short version command --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7eb9279b2c..35700d7432 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7eb9279b2c0683fa2e0a808788684b2c67bbe309 +Subproject commit 35700d7432848861094094245f6b4ddf5d28e20d From 779eaee310b13255bf7a1df8a1399242b90bbd58 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Oct 2020 22:25:47 +0200 Subject: [PATCH 27/51] Bump version to 5.0.2b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 4cb89e406c..f9ebb52c75 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "2b2") +VERSION = (5, 0, "2b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 63a6fe9133a4752a56d00c1a1938669442267c72 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 27 Oct 2020 21:07:02 +0200 Subject: [PATCH 28/51] Improved "core.call" RPC for PlatformIO Home // Resolve #3671 --- HISTORY.rst | 1 + platformio/__init__.py | 2 +- platformio/commands/home/rpc/handlers/piocore.py | 5 +++-- platformio/commands/home/rpc/handlers/project.py | 12 +++++++++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2df285791d..d42543d6e4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ PlatformIO Core 5 ~~~~~~~~~~~~~~~~~~ - Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry (`issue #3700 `_) +- Improved "core.call" RPC for PlatformIO Home (`issue #3671 `_) - Fixed a "PermissionError: [WinError 5]" on Windows when external repository is used with `lib_deps `__ option (`issue #3664 `_) - Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) - Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 `_) diff --git a/platformio/__init__.py b/platformio/__init__.py index f9ebb52c75..bcf6495156 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -47,7 +47,7 @@ __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.3.0", + "contrib-piohome": "~3.3.1", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 00adf5b6b1..81b773c01a 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -95,10 +95,11 @@ def _call_generator(args, options=None): else: args[i] = str(arg) + options = options or {} to_json = "--json-output" in args try: - if args and args[0] == "remote": + if options.get("force_subprocess"): result = yield PIOCoreRPC._call_subprocess(args, options) defer.returnValue(PIOCoreRPC._process_result(result, to_json)) else: @@ -117,7 +118,7 @@ def _call_generator(args, options=None): @staticmethod def _call_inline(args, options): PIOCoreRPC.setup_multithreading_std_streams() - cwd = (options or {}).get("cwd") or os.getcwd() + cwd = options.get("cwd") or os.getcwd() def _thread_task(): with fs.cd(cwd): diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 2db966b699..eb9cd23721 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -198,7 +198,9 @@ def init(self, board, framework, project_dir): and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call(args, options={"cwd": project_dir}) + d = PIOCoreRPC.call( + args, options={"cwd": project_dir, "force_subprocess": True} + ) d.addCallback(self._generate_project_main, project_dir, framework) return d @@ -291,7 +293,9 @@ def import_arduino(self, board, use_arduino_libs, arduino_project_dir): and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call(args, options={"cwd": project_dir}) + d = PIOCoreRPC.call( + args, options={"cwd": project_dir, "force_subprocess": True} + ) d.addCallback(self._finalize_arduino_import, project_dir, arduino_project_dir) return d @@ -324,6 +328,8 @@ def import_pio(project_dir): and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call(args, options={"cwd": new_project_dir}) + d = PIOCoreRPC.call( + args, options={"cwd": new_project_dir, "force_subprocess": True} + ) d.addCallback(lambda _: new_project_dir) return d From e158e54a26cbbfc716dc1df6f89d8f414dc9f958 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 27 Oct 2020 22:57:51 +0200 Subject: [PATCH 29/51] Fix issue with data decoding when calling PIO Core via PIO Home --- platformio/commands/home/rpc/handlers/piocore.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 81b773c01a..7a16f9c64a 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -27,7 +27,13 @@ from platformio import __main__, __version__, fs from platformio.commands.home import helpers -from platformio.compat import PY2, get_filesystem_encoding, is_bytes, string_types +from platformio.compat import ( + PY2, + get_filesystem_encoding, + get_locale_encoding, + is_bytes, + string_types, +) try: from thread import get_ident as thread_get_ident @@ -144,13 +150,15 @@ def _call_subprocess(args, options): @staticmethod def _process_result(result, to_json=False): out, err, code = result + if out and is_bytes(out): + out = out.decode(get_locale_encoding()) + if err and is_bytes(err): + err = err.decode(get_locale_encoding()) text = ("%s\n\n%s" % (out, err)).strip() if code != 0: raise Exception(text) if not to_json: return text - if is_bytes(out): - out = out.decode() try: return json.loads(out) except ValueError as e: From 2cd19b0273d49c0aea4c529c8ba2ba890dff8989 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 27 Oct 2020 23:00:33 +0200 Subject: [PATCH 30/51] Bump version to 5.0.2b4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index bcf6495156..91e02737c2 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "2b3") +VERSION = (5, 0, "2b4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 16f5f3ef46ef49c254957fc6599bb03885cf167d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Oct 2020 14:18:09 +0200 Subject: [PATCH 31/51] Do not pack binary files and docs to the package source archive --- platformio/package/pack.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index d88957fc5a..bf92e07002 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -29,15 +29,34 @@ class PackagePacker(object): EXCLUDE_DEFAULT = [ + # PlatformIO internal files + PackageItem.METAFILE_NAME, + ".pio/", + "**/.pio/", + # Hidden files "._*", "__*", ".DS_Store", + ".vscode", + ".cache", + "**/.cache", + # VCS ".git/", ".hg/", ".svn/", - ".pio/", - "**/.pio/", - PackageItem.METAFILE_NAME, + # Docs + "docs", + "mkdocs", + "**/*.[pP][dD][fF]", + "**/*.[dD][oO][cC]?", + "**/*.[pP][pP][tT]?", + # Binary files + "**/*.[jJ][pP][gG]", + "**/*.[jJ][pP][eE][gG]", + "**/*.[pP][nN][gG]", + "**/*.[gG][iI][fF]", + "**/*.[zZ][iI][pP]", + "**/*.[gG][zZ]", ] INCLUDE_DEFAULT = ManifestFileType.items().values() From 9ae981614ffa75ae37a4bb8272ae9203ebc5e934 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Oct 2020 20:56:53 +0200 Subject: [PATCH 32/51] Add pack/sdist target --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 57aa76c40b..3ddd1e27da 100644 --- a/Makefile +++ b/Makefile @@ -31,5 +31,8 @@ profile: python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS} snakeviz .tox/.tmp/cprofile.prof +pack: + python setup.py sdist + publish: python setup.py sdist upload From 175be346a8c224c833daaed261e13f0d3974397a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Oct 2020 20:57:26 +0200 Subject: [PATCH 33/51] Extend package filters --- platformio/package/pack.py | 50 +++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index bf92e07002..4aad832b92 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -28,6 +28,7 @@ class PackagePacker(object): + INCLUDE_DEFAULT = ManifestFileType.items().values() EXCLUDE_DEFAULT = [ # PlatformIO internal files PackageItem.METAFILE_NAME, @@ -44,12 +45,20 @@ class PackagePacker(object): ".git/", ".hg/", ".svn/", + # Tests + "tests?", # Docs + "doc", "docs", "mkdocs", "**/*.[pP][dD][fF]", "**/*.[dD][oO][cC]?", "**/*.[pP][pP][tT]?", + "**/*.[dD][oO][xX]", + "**/*.[hH][tT][mM]?", + "**/*.[tT][eE][xX]", + "**/*.[jJ][sS]", + "**/*.[cC][sS][sS]", # Binary files "**/*.[jJ][pP][gG]", "**/*.[jJ][pP][eE][gG]", @@ -57,8 +66,29 @@ class PackagePacker(object): "**/*.[gG][iI][fF]", "**/*.[zZ][iI][pP]", "**/*.[gG][zZ]", + "**/*.3[gG][pP]", + "**/*.[mM][oO][vV]", + "**/*.[mM][pP][34]", + "**/*.[pP][sS][dD]", + "**/*.[wW][aA][wW]", + ] + EXCLUDE_LIBRARY_EXTRA = [ + "assets", + "extra", + "resources", + "html", + "media", + "doxygen", + "**/build/", + "**/*.flat", + "**/*.[jJ][aA][rR]", + "**/*.[eE][xX][eE]", + "**/*.[bB][iI][nN]", + "**/*.[hH][eE][xX]", + "**/*.[dD][bB]", + "**/*.[dD][aA][tT]", + "**/*.[dD][lL][lL]", ] - INCLUDE_DEFAULT = ManifestFileType.items().values() def __init__(self, package, manifest_uri=None): self.package = package @@ -149,16 +179,28 @@ def _create_tarball(self, src, dst, manifest): json.dump(manifest_updated, fp, indent=2, ensure_ascii=False) include = None - src_filters = self.compute_src_filters(include, exclude) + src_filters = self.compute_src_filters(src, include, exclude) with tarfile.open(dst, "w:gz") as tar: for f in fs.match_src_files(src, src_filters, followlinks=False): tar.add(os.path.join(src, f), f) return dst - def compute_src_filters(self, include, exclude): + def compute_src_filters(self, src, include, exclude): + exclude_default = self.EXCLUDE_DEFAULT[:] + # extend with library extra filters + if any( + os.path.isfile(os.path.join(src, name)) + for name in ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ) + ): + exclude_default.extend(self.EXCLUDE_LIBRARY_EXTRA) + result = ["+<%s>" % p for p in include or ["*", ".*"]] result += ["-<%s>" % p for p in exclude or []] - result += ["-<%s>" % p for p in self.EXCLUDE_DEFAULT] + result += ["-<%s>" % p for p in exclude_default] # automatically include manifests result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT] return result From 891f78be374d7f1ab2aaef9728a6faa1b21e46b5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Oct 2020 22:32:27 +0200 Subject: [PATCH 34/51] Use "ensure_python3" util --- platformio/commands/remote/command.py | 12 +++--------- platformio/compat.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index cafbbd1f30..1a2c8ee2f4 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -23,12 +23,12 @@ import click -from platformio import exception, fs, proc +from platformio import fs, proc from platformio.commands.device import helpers as device_helpers from platformio.commands.device.command import device_monitor as cmd_device_monitor from platformio.commands.run.command import cli as cmd_run from platformio.commands.test.command import cli as cmd_test -from platformio.compat import PY2 +from platformio.compat import ensure_python3 from platformio.package.manager.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError @@ -37,13 +37,7 @@ @click.option("-a", "--agent", multiple=True) @click.pass_context def cli(ctx, agent): - if PY2: - raise exception.UserSideException( - "PlatformIO Remote Development requires Python 3.5 or above. \n" - "Please install the latest Python 3 and reinstall PlatformIO Core using " - "installation script:\n" - "https://docs.platformio.org/page/core/installation.html" - ) + assert ensure_python3() ctx.obj = agent inject_contrib_pysite(verify_openssl=True) diff --git a/platformio/compat.py b/platformio/compat.py index bf88c6813d..0fde6b4633 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -23,6 +23,8 @@ import re import sys +from platformio.exception import UserSideException + PY2 = sys.version_info[0] == 2 CYGWIN = sys.platform.startswith("cygwin") WINDOWS = sys.platform.startswith("win") @@ -59,6 +61,17 @@ def ci_strings_are_equal(a, b): return a.strip().lower() == b.strip().lower() +def ensure_python3(raise_exception=True): + if not raise_exception or not PY2: + return not PY2 + raise UserSideException( + "Python 3.5 or later is required for this operation. \n" + "Please install the latest Python 3 and reinstall PlatformIO Core using " + "installation script:\n" + "https://docs.platformio.org/page/core/installation.html" + ) + + if PY2: import imp From edbe21341073088db378eb31670eed73d8ba14e8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Oct 2020 22:32:48 +0200 Subject: [PATCH 35/51] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 35700d7432..a33e885d68 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 35700d7432848861094094245f6b4ddf5d28e20d +Subproject commit a33e885d68476a2c528c324d7f1127163886fd43 From cacddb9abbecaa2d5523e7f47267043f190139aa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Oct 2020 22:33:24 +0200 Subject: [PATCH 36/51] Support package packing on the Python 3+ only --- platformio/package/pack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 4aad832b92..ebdca496a4 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -20,6 +20,7 @@ import tempfile from platformio import fs +from platformio.compat import ensure_python3 from platformio.package.exception import PackageException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema @@ -91,6 +92,7 @@ class PackagePacker(object): ] def __init__(self, package, manifest_uri=None): + assert ensure_python3() self.package = package self.manifest_uri = manifest_uri From adf9ba29df7ccc1da1ec6c434e6a187af239b937 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Oct 2020 22:52:48 +0200 Subject: [PATCH 37/51] Fixed an issue when "pio package publish" command removes original archive after submitting to the registry // Resolve #3716 --- HISTORY.rst | 1 + platformio/commands/package.py | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d42543d6e4..47104ec315 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ PlatformIO Core 5 - Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) - Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 `_) - Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses (`issue #3677 `_) +- Fixed an issue when "pio package publish" command removes original archive after submitting to the registry `issue #3716 `_) 5.0.1 (2020-09-10) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/package.py b/platformio/commands/package.py index f62362ff57..bda365706a 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -13,11 +13,14 @@ # limitations under the License. import os +import tempfile from datetime import datetime import click +from platformio import fs from platformio.clients.registry import RegistryClient +from platformio.compat import ensure_python3 from platformio.package.meta import PackageSpec, PackageType from platformio.package.pack import PackagePacker @@ -77,13 +80,16 @@ def package_pack(package, output): help="Notify by email when package is processed", ) def package_publish(package, owner, released_at, private, notify): - p = PackagePacker(package) - archive_path = p.pack() - response = RegistryClient().publish_package( - archive_path, owner, released_at, private, notify - ) - os.remove(archive_path) - click.secho(response.get("message"), fg="green") + assert ensure_python3() + with tempfile.TemporaryDirectory() as tmp_dir: # pylint: disable=no-member + with fs.cd(tmp_dir): + p = PackagePacker(package) + archive_path = p.pack() + response = RegistryClient().publish_package( + archive_path, owner, released_at, private, notify + ) + os.remove(archive_path) + click.secho(response.get("message"), fg="green") @cli.command("unpublish", short_help="Remove a pushed package from the registry") From 9efac669e6e7f34326d6b5797c2c686076d89bad Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 28 Oct 2020 22:53:29 +0200 Subject: [PATCH 38/51] Bump version to 5.0.2b5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 91e02737c2..3c5ec21e81 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "2b4") +VERSION = (5, 0, "2b5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 16f90dd82152c2facc3759f8b5086f6ac1406f89 Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 29 Oct 2020 12:22:26 +0200 Subject: [PATCH 39/51] Ignore possible empty defines when exporting IDE data // Resolve #3690 --- platformio/builder/tools/pioide.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index b9fab37546..8248c06aaa 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -93,7 +93,9 @@ def _dump_defines(env): defines = [] # global symbols for item in processDefines(env.get("CPPDEFINES", [])): - defines.append(env.subst(item).replace("\\", "")) + item = item.strip() + if item: + defines.append(env.subst(item).replace("\\", "")) # special symbol for Atmel AVR MCU if env["PIOPLATFORM"] == "atmelavr": From 175448dedafc8a18920458350c3b22018fadd812 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Oct 2020 14:37:50 +0200 Subject: [PATCH 40/51] Fix tests on PY2 --- tests/package/test_manager.py | 2 ++ tests/package/test_pack.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 8b948a95e4..920aee049f 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -21,6 +21,7 @@ import semantic_version from platformio import fs, util +from platformio.compat import PY2 from platformio.package.exception import ( MissingPackageManifestError, UnknownPackageError, @@ -144,6 +145,7 @@ def test_build_metadata(isolated_pio_core, tmpdir_factory): assert metadata.version.build[1] == vcs_revision +@pytest.mark.skipif(PY2, reason="Requires Python 3.5 or higher") def test_install_from_url(isolated_pio_core, tmpdir_factory): tmp_dir = tmpdir_factory.mktemp("tmp") storage_dir = tmpdir_factory.mktemp("storage") diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py index a964f5cdc0..92db31c4c9 100644 --- a/tests/package/test_pack.py +++ b/tests/package/test_pack.py @@ -19,10 +19,12 @@ import pytest from platformio import fs -from platformio.compat import WINDOWS +from platformio.compat import PY2, WINDOWS from platformio.package.exception import UnknownManifestError from platformio.package.pack import PackagePacker +pytestmark = pytest.mark.skipif(PY2, reason="Requires Python 3.5 or higher") + def test_base(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") From 5ea864da394d8acb6e1c95a1d8feb5535f15121e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Oct 2020 18:08:58 +0200 Subject: [PATCH 41/51] Add py39 env --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 0faae46b5e..bb41a67b04 100644 --- a/tox.ini +++ b/tox.ini @@ -13,13 +13,13 @@ # limitations under the License. [tox] -envlist = py27,py37,py38 +envlist = py27,py37,py38,py39 [testenv] passenv = * usedevelop = True deps = - py36,py37,py38: black + py36,py37,py38,py39: black isort<5 pylint pytest From da6cde5cbda18cf76f1e0ca341fae8bccadef979 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Oct 2020 18:09:08 +0200 Subject: [PATCH 42/51] Sync docs --- docs | 2 +- scripts/docspregen.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs b/docs index a33e885d68..500b826845 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a33e885d68476a2c528c324d7f1127163886fd43 +Subproject commit 500b826845400c7ceff4c787920087ba683239fe diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 3698aa0bc7..881160ed20 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -1066,6 +1066,8 @@ def update_project_examples(): # Frameworks frameworks = [] for framework in API_FRAMEWORKS: + if framework["name"] not in framework_examples_md_lines: + continue readme_dir = join(project_examples_dir, "frameworks", framework["name"]) if not isdir(readme_dir): os.makedirs(readme_dir) From 5a1b0e19b2b534d6faedb827bffc7ddb3f33479d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Oct 2020 22:59:48 +0200 Subject: [PATCH 43/51] Initialize a new project or update existing passing working environment name and its options // Resolve #3686 --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/project.py | 28 +++++++++++++++++++++++++--- platformio/project/config.py | 6 ------ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 47104ec315..7d498c78b3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ PlatformIO Core 5 5.0.2 (2020-09-??) ~~~~~~~~~~~~~~~~~~ +- Initialize a new project or update existing passing working environment name and its options (`issue #3686 `_) - Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry (`issue #3700 `_) - Improved "core.call" RPC for PlatformIO Home (`issue #3671 `_) - Fixed a "PermissionError: [WinError 5]" on Windows when external repository is used with `lib_deps `__ option (`issue #3664 `_) diff --git a/docs b/docs index 500b826845..bc6c9b2e4c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 500b826845400c7ceff4c787920087ba683239fe +Subproject commit bc6c9b2e4cc2dd06da7da942aa9d13494c6e3941 diff --git a/platformio/commands/project.py b/platformio/commands/project.py index f861b5b199..592f3a483c 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -174,8 +174,10 @@ def project_init( if is_new_project: init_base_project(project_dir) - if board: - fill_project_envs( + if environment: + update_project_env(project_dir, environment, project_option) + elif board: + update_board_envs( ctx, project_dir, board, project_option, env_prefix, ide is not None ) @@ -358,7 +360,7 @@ def init_cvs_ignore(project_dir): fp.write(".pio\n") -def fill_project_envs( +def update_board_envs( ctx, project_dir, board_ids, project_option, env_prefix, force_download ): config = ProjectConfig( @@ -417,6 +419,26 @@ def _install_dependent_platforms(ctx, platforms): ) +def update_project_env(project_dir, environment, project_option): + if not project_option: + return + config = ProjectConfig( + os.path.join(project_dir, "platformio.ini"), parse_extra=False + ) + + section = "env:%s" % environment + if not config.has_section(section): + config.add_section(section) + + for item in project_option: + if "=" not in item: + continue + _name, _value = item.split("=", 1) + config.set(section, _name.strip(), _value.strip()) + + config.save() + + def get_best_envname(config, board_ids=None): envname = None default_envs = config.default_envs() diff --git a/platformio/project/config.py b/platformio/project/config.py index 2d841b396e..786f080aab 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -358,12 +358,6 @@ def validate(self, envs=None, silent=False): click.secho("Warning! %s" % warning, fg="yellow") return True - def remove_option(self, section, option): - return self._parser.remove_option(section, option) - - def remove_section(self, section): - return self._parser.remove_section(section) - class ProjectConfigDirsMixin(object): def _get_core_dir(self, exists=False): From 743a43ae17a0657719a6b188c2b5e0aa4af8e5f6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Oct 2020 23:17:47 +0200 Subject: [PATCH 44/51] Fixed an issue when multiple `pio lib install` command with the same local library results in duplicates in ``lib_deps`` // Resolve #3715 --- HISTORY.rst | 7 ++++--- platformio/commands/lib/helpers.py | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7d498c78b3..d239b46ecd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,14 +11,15 @@ PlatformIO Core 5 5.0.2 (2020-09-??) ~~~~~~~~~~~~~~~~~~ -- Initialize a new project or update existing passing working environment name and its options (`issue #3686 `_) +- Initialize a new project or update the existing passing working environment name and its options (`issue #3686 `_) - Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry (`issue #3700 `_) - Improved "core.call" RPC for PlatformIO Home (`issue #3671 `_) -- Fixed a "PermissionError: [WinError 5]" on Windows when external repository is used with `lib_deps `__ option (`issue #3664 `_) +- Fixed a "PermissionError: [WinError 5]" on Windows when an external repository is used with `lib_deps `__ option (`issue #3664 `_) - Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) - Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 `_) - Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses (`issue #3677 `_) -- Fixed an issue when "pio package publish" command removes original archive after submitting to the registry `issue #3716 `_) +- Fixed an issue when `pio package publish `__ command removes original archive after submitting to the registry (`issue #3716 `_) +- Fixed an issue when multiple `pio lib install `__ command with the same local library results in duplicates in ``lib_deps`` (`issue #3715 `_) 5.0.1 (2020-09-10) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py index 732604a3f9..c720b10d59 100644 --- a/platformio/commands/lib/helpers.py +++ b/platformio/commands/lib/helpers.py @@ -81,15 +81,22 @@ def save_project_libdeps(project_dir, specs, environments=None, action="add"): if environments and env not in environments: continue config.expand_interpolations = False - lib_deps = [] + candidates = [] try: - lib_deps = ignore_deps_by_specs(config.get("env:" + env, "lib_deps"), specs) + candidates = ignore_deps_by_specs( + config.get("env:" + env, "lib_deps"), specs + ) except InvalidProjectConfError: pass if action == "add": - lib_deps.extend(spec.as_dependency() for spec in specs) - if lib_deps: - config.set("env:" + env, "lib_deps", lib_deps) + candidates.extend(spec.as_dependency() for spec in specs) + if candidates: + result = [] + for item in candidates: + item = item.strip() + if item and item not in result: + result.append(item) + config.set("env:" + env, "lib_deps", result) elif config.has_option("env:" + env, "lib_deps"): config.remove_option("env:" + env, "lib_deps") config.save() From 454cd8d7847074988bf967f6240d59f25bdf310e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Oct 2020 23:18:39 +0200 Subject: [PATCH 45/51] Bump version to 5.0.2rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 3c5ec21e81..f873bcb961 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "2b5") +VERSION = (5, 0, "2rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From e333bb1cca0ecf783aa04d15ca2069b0ef3cbadb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Oct 2020 23:42:15 +0200 Subject: [PATCH 46/51] Tests: skip dev-plalform without examples --- tests/test_examples.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 994eb8c014..9e2b7cc079 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import random from glob import glob -from os import listdir, walk -from os.path import basename, dirname, getsize, isdir, isfile, join, normpath import pytest @@ -32,24 +31,26 @@ def pytest_generate_tests(metafunc): examples_dirs = [] # repo examples - examples_dirs.append(normpath(join(dirname(__file__), "..", "examples"))) + examples_dirs.append( + os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "examples")) + ) # dev/platforms for pkg in PlatformPackageManager().get_installed(): p = PlatformFactory.new(pkg) - examples_dir = join(p.get_dir(), "examples") - assert isdir(examples_dir) - examples_dirs.append(examples_dir) + examples_dir = os.path.join(p.get_dir(), "examples") + if os.path.isdir(examples_dir): + examples_dirs.append(examples_dir) project_dirs = [] for examples_dir in examples_dirs: candidates = {} - for root, _, files in walk(examples_dir): + for root, _, files in os.walk(examples_dir): if "platformio.ini" not in files or ".skiptest" in files: continue if "zephyr-" in root and PY2: continue - group = basename(root) + group = os.path.basename(root) if "-" in group: group = group.split("-", 1)[0] if group not in candidates: @@ -67,7 +68,7 @@ def test_run(pioproject_dir): with fs.cd(pioproject_dir): config = ProjectConfig() build_dir = config.get_optional_dir("build") - if isdir(build_dir): + if os.path.isdir(build_dir): fs.rmtree(build_dir) env_names = config.envs() @@ -77,18 +78,18 @@ def test_run(pioproject_dir): if result["returncode"] != 0: pytest.fail(str(result)) - assert isdir(build_dir) + assert os.path.isdir(build_dir) # check .elf file - for item in listdir(build_dir): - if not isdir(item): + for item in os.listdir(build_dir): + if not os.path.isdir(item): continue - assert isfile(join(build_dir, item, "firmware.elf")) + assert os.path.isfile(os.path.join(build_dir, item, "firmware.elf")) # check .hex or .bin files firmwares = [] for ext in ("bin", "hex"): - firmwares += glob(join(build_dir, item, "firmware*.%s" % ext)) + firmwares += glob(os.path.join(build_dir, item, "firmware*.%s" % ext)) if not firmwares: pytest.fail("Missed firmware file") for firmware in firmwares: - assert getsize(firmware) > 0 + assert os.path.getsize(firmware) > 0 From 0bbe7f8c73fc24e94a126d2c117d5211e9a0e22a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Oct 2020 23:48:44 +0200 Subject: [PATCH 47/51] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index bc6c9b2e4c..9a9aa8952f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit bc6c9b2e4cc2dd06da7da942aa9d13494c6e3941 +Subproject commit 9a9aa8952f35096987d82b00748957ae19556d98 From 96b1a1c79c4fc64fe2b688c1fed4719bba58d7af Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Oct 2020 14:11:27 +0200 Subject: [PATCH 48/51] Fixed an issue with a "wrong" timestamp in device monitor output using `"time" filter` // Resolve #3712 --- HISTORY.rst | 1 + platformio/commands/device/filters/time.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d239b46ecd..80d2487284 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,7 @@ PlatformIO Core 5 - Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses (`issue #3677 `_) - Fixed an issue when `pio package publish `__ command removes original archive after submitting to the registry (`issue #3716 `_) - Fixed an issue when multiple `pio lib install `__ command with the same local library results in duplicates in ``lib_deps`` (`issue #3715 `_) +- Fixed an issue with a "wrong" timestamp in device monitor output using `"time" filter `__ (`issue #3712 `_) 5.0.1 (2020-09-10) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/device/filters/time.py b/platformio/commands/device/filters/time.py index 203e6aa9fc..0c2d888410 100644 --- a/platformio/commands/device/filters/time.py +++ b/platformio/commands/device/filters/time.py @@ -22,13 +22,16 @@ class Timestamp(DeviceMonitorFilter): def __init__(self, *args, **kwargs): super(Timestamp, self).__init__(*args, **kwargs) - self._first_text_received = False + self._line_started = False def rx(self, text): - if self._first_text_received and "\n" not in text: + if self._line_started and "\n" not in text: return text timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] - if not self._first_text_received: - self._first_text_received = True - return "%s > %s" % (timestamp, text) + if not self._line_started: + self._line_started = True + text = "%s > %s" % (timestamp, text) + if text.endswith("\n"): + self._line_started = False + return text[:-1].replace("\n", "\n%s > " % timestamp) + "\n" return text.replace("\n", "\n%s > " % timestamp) From 5cf73a91657e8cf17e8394d8fde074fd4b8f6c73 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Oct 2020 17:50:43 +0200 Subject: [PATCH 49/51] Remove all hooks when dumping data to JSON and Python 3 is used --- platformio/compat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/compat.py b/platformio/compat.py index 0fde6b4633..1ef2883b22 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -98,7 +98,7 @@ def dump_json_to_unicode(obj): if isinstance(obj, unicode): return obj return json.dumps( - obj, encoding=get_filesystem_encoding(), ensure_ascii=False, sort_keys=True + obj, encoding=get_filesystem_encoding(), ensure_ascii=False ).encode("utf8") _magic_check = re.compile("([*?[])") @@ -146,7 +146,7 @@ def hashlib_encode_data(data): def dump_json_to_unicode(obj): if isinstance(obj, string_types): return obj - return json.dumps(obj, ensure_ascii=False, sort_keys=True) + return json.dumps(obj) def glob_recursive(pathname): return glob.glob(pathname, recursive=True) From 344e94d8a166263ae035e98fba53f395161fa005 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Oct 2020 17:51:02 +0200 Subject: [PATCH 50/51] Bump version to 5.0.2rc2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index f873bcb961..b83f1906b8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "2rc1") +VERSION = (5, 0, "2rc2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From d6df6cbb5d82f3fc49b6bf29fd00ffddf1dae50c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 30 Oct 2020 18:10:47 +0200 Subject: [PATCH 51/51] Bump version to 5.0.2 --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 80d2487284..41b7ab201b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,7 +8,7 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** -5.0.2 (2020-09-??) +5.0.2 (2020-10-30) ~~~~~~~~~~~~~~~~~~ - Initialize a new project or update the existing passing working environment name and its options (`issue #3686 `_) diff --git a/docs b/docs index 9a9aa8952f..deae09a880 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9a9aa8952f35096987d82b00748957ae19556d98 +Subproject commit deae09a880822fb8c0b4973c29f75f2343476d6f diff --git a/platformio/__init__.py b/platformio/__init__.py index b83f1906b8..55813e698c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "2rc2") +VERSION = (5, 0, 2) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"