From 91ae8b4cc7440e00f18df31850d08dc67d2f108d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Mar 2020 15:15:30 +0200 Subject: [PATCH 01/69] Fixed typo in history --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9606fdd925..ad47afc21f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,7 +11,7 @@ PlatformIO Core 4 * Fixed a SyntaxError "'return' with argument inside generator" for PIO Unified Debugger when Python 2.7 is used * Fixed an issue when ``lib_archive = no`` was not honored in `"platformio.ini" `__ -* Fixed an TypeError "super(type, obj): obj must be an instance or subtype of type" when device monitor is used with a custom dev-platform filter (`issue #3431 `_) +* Fixed a TypeError "super(type, obj): obj must be an instance or subtype of type" when device monitor is used with a custom dev-platform filter (`issue #3431 `_) 4.3.0 (2020-03-19) ~~~~~~~~~~~~~~~~~~ From 096c2f6165a3fa5932a18c32606722432f169341 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Mar 2020 17:11:31 +0200 Subject: [PATCH 02/69] Typo fix in docs --- docs | 2 +- scripts/docspregen.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs b/docs index d97117eb2e..a6b9ca4d7c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d97117eb2e40380c309d1ef86ebf3501c673f270 +Subproject commit a6b9ca4d7caed38cd201498b4d0281421c7d6ee9 diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 2ed1b2d50c..eb075ae524 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -429,8 +429,8 @@ def generate_platform(name, rst_dir): .. note:: * You can list pre-configured boards by :ref:`cmd_boards` command or `PlatformIO Boards Explorer `_ - * For more detailed ``board`` information please scroll tables below by - horizontal. + * For more detailed ``board`` information please scroll the tables below by + horizontally. """) for vendor, boards in sorted(vendors.items()): @@ -533,7 +533,7 @@ def generate_framework(type_, data, rst_dir=None): .. note:: * You can list pre-configured boards by :ref:`cmd_boards` command or `PlatformIO Boards Explorer `_ - * For more detailed ``board`` information please scroll tables below by horizontal. + * For more detailed ``board`` information please scroll the tables below by horizontally. """) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) From 32642b7ec8ae6e37f40e114ccf8acc615a50c0c2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Mar 2020 17:16:49 +0200 Subject: [PATCH 03/69] Fix broken link to Renode in history --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index ad47afc21f..d4b00fe6f5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -35,7 +35,7 @@ PlatformIO Core 4 - Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter * New standalone (1-script) `PlatformIO Core Installer `_ -* Initial support for `Renode `__ simulation framework (`issue #3401 `_) +* Initial support for `Renode `__ simulation framework (`issue #3401 `_) * Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) * Improved support for Arduino "library.properties" ``depends`` field * Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server From dbeaaf270c6546f111fc6b0ab57fad802db1e27d Mon Sep 17 00:00:00 2001 From: Richard Coleman Date: Fri, 20 Mar 2020 22:02:50 +0000 Subject: [PATCH 04/69] fix typo in URL (#3432) --- platformio/package/exception.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/exception.py b/platformio/package/exception.py index 550a3628dd..1155529149 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -41,5 +41,5 @@ def __init__(self, messages, data, valid_data): def __str__(self): return ( "Invalid manifest fields: %s. \nPlease check specification -> " - "htts://docs.platformio.org/page/librarymanager/config.html" % self.messages + "https://docs.platformio.org/page/librarymanager/config.html" % self.messages ) From efd3b244e1459d78161e2a36538b561624a094b9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 21 Mar 2020 13:27:46 +0200 Subject: [PATCH 05/69] Force PIPE reader to UTF-8 on Windows // Issue #3417 --- platformio/proc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platformio/proc.py b/platformio/proc.py index 80e50201a0..1937b257a8 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -20,6 +20,7 @@ from platformio import exception from platformio.compat import ( + PY2, WINDOWS, get_filesystem_encoding, get_locale_encoding, @@ -30,7 +31,10 @@ class AsyncPipeBase(object): def __init__(self): self._fd_read, self._fd_write = os.pipe() - self._pipe_reader = os.fdopen(self._fd_read) + if PY2: + self._pipe_reader = os.fdopen(self._fd_read) + else: + self._pipe_reader = os.fdopen(self._fd_read, encoding="utf-8") self._buffer = "" self._thread = Thread(target=self.run) self._thread.start() From fd137fe05473481a9e8df87d17c4f987661d1cd0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 21 Mar 2020 13:28:31 +0200 Subject: [PATCH 06/69] Bump version to 4.3.2a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index afa9e24dee..6096e53af6 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, 1) +VERSION = (4, 3, "2a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 81a4d289186e9a788346b1dbf23989e0d1b1aab2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 21 Mar 2020 16:39:17 +0200 Subject: [PATCH 07/69] Docs: Remove duplicate demo image of PlatformIO for CLion --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index a6b9ca4d7c..51a6607609 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a6b9ca4d7caed38cd201498b4d0281421c7d6ee9 +Subproject commit 51a6607609795182914774116f38c2dffea87f2c From f95230b86e4266ad7c3af0030ca1388c3c51ecd9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 21 Mar 2020 21:53:42 +0200 Subject: [PATCH 08/69] Fixed UnicodeDecodeError on Windows when network drive (NAS) is used // Resolve #3417 --- HISTORY.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index d4b00fe6f5..f78c22f389 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,11 @@ Release Notes PlatformIO Core 4 ----------------- +4.3.2 (2020-??-??) +~~~~~~~~~~~~~~~~~~ + +* Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) + 4.3.1 (2020-03-20) ~~~~~~~~~~~~~~~~~~ From 79b3a232fc66c467224704451725136b1f2b7d76 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 21 Mar 2020 22:00:14 +0200 Subject: [PATCH 09/69] Move debug client and server implementations to "process" folder --- platformio/commands/debug/command.py | 2 +- platformio/commands/debug/process/__init__.py | 13 +++++++++++++ .../commands/debug/{process.py => process/base.py} | 0 platformio/commands/debug/{ => process}/client.py | 4 ++-- platformio/commands/debug/{ => process}/server.py | 2 +- platformio/package/exception.py | 3 ++- 6 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 platformio/commands/debug/process/__init__.py rename platformio/commands/debug/{process.py => process/base.py} (100%) rename platformio/commands/debug/{ => process}/client.py (98%) rename platformio/commands/debug/{ => process}/server.py (98%) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index c3c29dd166..2528611161 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -148,7 +148,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro inject_contrib_pysite() # pylint: disable=import-outside-toplevel - from platformio.commands.debug.client import GDBClient, reactor + from platformio.commands.debug.process.client import GDBClient, reactor client = GDBClient(project_dir, __unprocessed, debug_options, env_options) client.spawn(configuration["gdb_path"], configuration["prog_path"]) diff --git a/platformio/commands/debug/process/__init__.py b/platformio/commands/debug/process/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/commands/debug/process/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process/base.py similarity index 100% rename from platformio/commands/debug/process.py rename to platformio/commands/debug/process/base.py diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/process/client.py similarity index 98% rename from platformio/commands/debug/client.py rename to platformio/commands/debug/process/client.py index 4e2298f9c5..7ae5363e30 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/process/client.py @@ -30,8 +30,8 @@ from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.debug.initcfgs import get_gdb_init_config -from platformio.commands.debug.process import BaseProcess -from platformio.commands.debug.server import DebugServer +from platformio.commands.debug.process.base import BaseProcess +from platformio.commands.debug.process.server import DebugServer from platformio.compat import hashlib_encode_data, is_bytes from platformio.project.helpers import get_project_cache_dir diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/process/server.py similarity index 98% rename from platformio/commands/debug/server.py rename to platformio/commands/debug/process/server.py index ef06b58c93..7bd5d4850b 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/process/server.py @@ -22,7 +22,7 @@ from platformio import fs, util from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode -from platformio.commands.debug.process import BaseProcess +from platformio.commands.debug.process.base import BaseProcess from platformio.proc import where_is_program diff --git a/platformio/package/exception.py b/platformio/package/exception.py index 1155529149..adadc0889f 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -41,5 +41,6 @@ def __init__(self, messages, data, valid_data): def __str__(self): return ( "Invalid manifest fields: %s. \nPlease check specification -> " - "https://docs.platformio.org/page/librarymanager/config.html" % self.messages + "https://docs.platformio.org/page/librarymanager/config.html" + % self.messages ) From 589d6f9e12e896af2779f5e2793d1897e71244d4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Mar 2020 19:35:18 +0200 Subject: [PATCH 10/69] Docs: Sync Espressif 32 dev-platform --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 51a6607609..ec3978d15c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 51a6607609795182914774116f38c2dffea87f2c +Subproject commit ec3978d15cfca9b509a37bdbd72fe4a9c9db9e66 diff --git a/examples b/examples index 370c2c41a1..90932ce44f 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 370c2c41a10fe02aafe0f4910f827a00b0f24ad8 +Subproject commit 90932ce44f40d7b08311f8453d2a7266b72bb6e7 From 11c946bfe4a9e08214edc53ea21d250d20e36781 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Mar 2020 19:52:57 +0200 Subject: [PATCH 11/69] Sync Espressif 32 dev-platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index ec3978d15c..b2db3105b0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ec3978d15cfca9b509a37bdbd72fe4a9c9db9e66 +Subproject commit b2db3105b07f1fcadbe4b426ea8965e046ca8f0d From d42481d1964dd371bef3c461139454ce3ff75898 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 24 Mar 2020 18:03:23 +0200 Subject: [PATCH 12/69] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b2db3105b0..fac379a466 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b2db3105b07f1fcadbe4b426ea8965e046ca8f0d +Subproject commit fac379a466d78b998fc78d5ca4be2ba33721dd6c From 45e523a46825957ff7ade5f673138ea3e341c675 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 25 Mar 2020 17:01:12 +0200 Subject: [PATCH 13/69] Docs: Sync with Atmel SAM dev-platform --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index fac379a466..7855a18775 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit fac379a466d78b998fc78d5ca4be2ba33721dd6c +Subproject commit 7855a18775bcf5e7a0357252dd59b2bfc556a8af diff --git a/examples b/examples index 90932ce44f..9bc28310ca 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 90932ce44f40d7b08311f8453d2a7266b72bb6e7 +Subproject commit 9bc28310ca41d4d84100bca7bb8142d92090d7c7 From 1b0810ec8745f270d2213e99b84771071744aed6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 26 Mar 2020 22:31:15 +0200 Subject: [PATCH 14/69] Docs: Fix broken link for creating dev-platform // Resolve #123 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7855a18775..a9522c3c16 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7855a18775bcf5e7a0357252dd59b2bfc556a8af +Subproject commit a9522c3c16c9d36b9354a72b5d42755086af4c4e From e92b498b6857dcf8a4591db3066c8572342abd4d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Mar 2020 13:34:14 +0200 Subject: [PATCH 15/69] Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" // Resolve #3442 --- HISTORY.rst | 1 + platformio/commands/lib.py | 6 +++++- tests/test_projectconf.py | 8 ++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index f78c22f389..4035d7af94 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ PlatformIO Core 4 ~~~~~~~~~~~~~~~~~~ * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) +* Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 `_) 4.3.1 (2020-03-20) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index e3b415d58a..d08529fdce 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -29,6 +29,7 @@ from platformio.package.manifest.schema import ManifestSchema from platformio.proc import is_ci from platformio.project.config import ProjectConfig +from platformio.project.exception import InvalidProjectConfError from platformio.project.helpers import get_project_dir, is_platformio_project try: @@ -180,7 +181,10 @@ def lib_install( # pylint: disable=too-many-arguments if project_environments and env not in project_environments: continue config.expand_interpolations = False - lib_deps = config.get("env:" + env, "lib_deps", []) + try: + lib_deps = config.get("env:" + env, "lib_deps") + except InvalidProjectConfError: + lib_deps = [] for library in libraries: if library in lib_deps: continue diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 0e68f2d33a..4d832c9754 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -121,11 +121,19 @@ def test_defaults(config): assert config.get_optional_dir("core") == os.path.join( os.path.expanduser("~"), ".platformio" ) + assert config.get("strict_ldf", "lib_deps", ["Empty"]) == ["Empty"] assert config.get("env:extra_2", "lib_compat_mode") == "soft" assert config.get("env:extra_2", "build_type") == "release" assert config.get("env:extra_2", "build_type", None) is None assert config.get("env:extra_2", "lib_archive", "no") is False + config.expand_interpolations = False + with pytest.raises( + InvalidProjectConfError, match="No option 'lib_deps' in section: 'strict_ldf'" + ): + assert config.get("strict_ldf", "lib_deps", ["Empty"]) == ["Empty"] + config.expand_interpolations = True + def test_sections(config): with pytest.raises(ConfigParser.NoSectionError): From f976cf7ae5068c301655aa2a561f600059435722 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Mar 2020 17:15:18 +0300 Subject: [PATCH 16/69] Docs: Extend tutorials list --- docs | 2 +- scripts/docspregen.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index a9522c3c16..bbbcef7e94 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a9522c3c16c9d36b9354a72b5d42755086af4c4e +Subproject commit bbbcef7e94e97520b2723ea79d3eab2883fb1c45 diff --git a/scripts/docspregen.py b/scripts/docspregen.py index eb075ae524..d388842fbf 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -53,7 +53,7 @@ def is_compat_platform_and_framework(platform, framework): return framework in (p.frameworks or {}).keys() -def campaign_url(url, source="platformio", medium="docs"): +def campaign_url(url, source="platformio.org", medium="docs"): data = urlparse(url) query = data.query if query: From 7412cf586ba852947b9cdb60f0f544d72873907a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Apr 2020 00:59:35 +0300 Subject: [PATCH 17/69] Update docs for Zephyr RTOS 2.2 --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index bbbcef7e94..9828594e7b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit bbbcef7e94e97520b2723ea79d3eab2883fb1c45 +Subproject commit 9828594e7b8dcea50712af45341850cf1659031a diff --git a/examples b/examples index 9bc28310ca..68d29b5dbc 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 9bc28310ca41d4d84100bca7bb8142d92090d7c7 +Subproject commit 68d29b5dbcd6301ba9adc03e2cb0b27fbaf7cd91 From 11cb3a1bf7c59d16bd1adc4cb067b0b3c882cea3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 4 Apr 2020 00:45:45 +0300 Subject: [PATCH 18/69] Docs: Add info about stm32pio tool for STM32Cube framework --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 9828594e7b..4faa94e3c0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9828594e7b8dcea50712af45341850cf1659031a +Subproject commit 4faa94e3c0b942f1e0323fa984b459228c9195f0 From 5ea759bc3e09a4f6079b82eb04bfda1180910b02 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 5 Apr 2020 19:48:50 +0300 Subject: [PATCH 19/69] Document PIO Core: Integration with custom applications (extensions, plugins) --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 4faa94e3c0..2ef24432bf 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4faa94e3c0b942f1e0323fa984b459228c9195f0 +Subproject commit 2ef24432bfecf95562fb2b9be5b311a54e201f16 From 3cd6c618a48ec16f5a065951fd2be0258093a183 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Apr 2020 16:56:05 +0300 Subject: [PATCH 20/69] Docs: Sync STM32 dev-platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 2ef24432bf..41f192fc0b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2ef24432bfecf95562fb2b9be5b311a54e201f16 +Subproject commit 41f192fc0b2f2ce3a867bdabb837abcbd0d85806 From 2e320c01b33124276134976292d850d22ca5a845 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Apr 2020 18:19:34 +0300 Subject: [PATCH 21/69] Fix test --- tests/commands/test_platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index ff30dabbb5..b2db5d83f9 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -63,7 +63,7 @@ def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke( cli_platform.platform_install, [ - "https://github.com/platformio/" "platform-espressif8266.git#feature/stage", + "https://github.com/platformio/" "platform-espressif8266.git", "--skip-default-package", ], ) From baa7aab1d7eb614863ad6652317c702c855dc025 Mon Sep 17 00:00:00 2001 From: valeros Date: Tue, 7 Apr 2020 11:35:17 +0300 Subject: [PATCH 22/69] Specify C++ as the language for .ino files when preprocessing them for PVS-Studio // Resolve #3450 --- platformio/commands/check/tools/pvsstudio.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index 7407da63a4..8c49fdb156 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -140,9 +140,7 @@ def configure_command(self, src_file): # pylint: disable=arguments-differ os.remove(self._tmp_output_file) if not os.path.isfile(self._tmp_preprocessed_file): - click.echo( - "Error: Missing preprocessed file '%s'" % (self._tmp_preprocessed_file) - ) + click.echo("Error: Missing preprocessed file for '%s'" % src_file) return "" cmd = [ @@ -175,6 +173,9 @@ def _generate_tmp_file_path(self): return os.path.join(self._tmp_dir, next(tempfile._get_candidate_names())) def _prepare_preprocessed_file(self, src_file): + if os.path.isfile(self._tmp_preprocessed_file): + os.remove(self._tmp_preprocessed_file) + flags = self.cxx_flags compiler = self.cxx_path if src_file.endswith(".c"): @@ -186,8 +187,12 @@ def _prepare_preprocessed_file(self, src_file): cmd.extend(["-D%s" % d for d in self.cpp_defines]) cmd.append('@"%s"' % self._tmp_cmd_file) + # Explicitly specify C++ as the language used in .ino files + if src_file.endswith(".ino"): + cmd.insert(1, "-xc++") + result = proc.exec_command(" ".join(cmd), shell=True) - if result["returncode"] != 0: + if result["returncode"] != 0 or result["err"]: if self.options.get("verbose"): click.echo(" ".join(cmd)) click.echo(result["err"]) From ad7e3f83aa09e92bbfd090d418029ab7022f16af Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Apr 2020 17:18:59 +0300 Subject: [PATCH 23/69] Fix tests/commands/test_init.py --- docs | 2 +- tests/commands/test_init.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 41f192fc0b..22080ab244 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 41f192fc0b2f2ce3a867bdabb837abcbd0d85806 +Subproject commit 22080ab2442f07216a7c2727723b9f80fd5090c0 diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index fccdee7d92..b874ead763 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -156,5 +156,5 @@ def test_init_custom_framework(clirunner, validate_cliresult): def test_init_incorrect_board(clirunner): result = clirunner.invoke(cmd_init, ["-b", "missed_board"]) assert result.exit_code == 2 - assert 'Error: Invalid value for "-b" / "--board' in result.output + assert "Error: Invalid value for" in result.output assert isinstance(result.exception, SystemExit) From a636a60e004d37f3d4b163eeae6161aee83267ce Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Apr 2020 22:34:04 +0300 Subject: [PATCH 24/69] Sort examples --- examples | 2 +- scripts/docspregen.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples b/examples index 68d29b5dbc..e750a6dfa2 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 68d29b5dbcd6301ba9adc03e2cb0b27fbaf7cd91 +Subproject commit e750a6dfa2f54cbfe9013224a4438e2d35dc8da7 diff --git a/scripts/docspregen.py b/scripts/docspregen.py index d388842fbf..3628f6092d 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -906,7 +906,7 @@ def update_project_examples(): platform_examples_dir = join(p.get_dir(), "examples") examples_md_lines = [] if isdir(platform_examples_dir): - for item in os.listdir(platform_examples_dir): + for item in sorted(os.listdir(platform_examples_dir)): example_dir = join(platform_examples_dir, item) if not isdir(example_dir) or not os.listdir(example_dir): continue From d5ebbb99a7d8740f1f73a21bf94ce17d3c227d08 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 9 Apr 2020 12:02:38 +0300 Subject: [PATCH 25/69] Dynamically choose extension for file with unit test transports (#3454) C file should be used by default as only Arduino and mbed require C++ files. There might be a lot of legacy projects so custom transport is also set to use C++. --- platformio/commands/test/processor.py | 31 +++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 5291b9f7ab..ac48a69206 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -29,6 +29,7 @@ "flush": "Serial.flush()", "begin": "Serial.begin($baudrate)", "end": "Serial.end()", + "language": "cpp", }, "mbed": { "include": "#include ", @@ -37,6 +38,7 @@ "flush": "", "begin": "pc.baud($baudrate)", "end": "", + "language": "cpp", }, "espidf": { "include": "#include ", @@ -46,6 +48,14 @@ "begin": "", "end": "", }, + "zephyr": { + "include": "#include ", + "object": "", + "putchar": "printk(\"%c\", c)", + "flush": "", + "begin": "", + "end": "", + }, "native": { "include": "#include ", "object": "", @@ -61,6 +71,7 @@ "flush": "unittest_uart_flush()", "begin": "unittest_uart_begin()", "end": "unittest_uart_end()", + "language": "cpp", }, } @@ -80,7 +91,7 @@ def __init__(self, cmd_ctx, testname, envname, options): self.env_name = envname self.env_options = options["project_config"].items(env=envname, as_dict=True) self._run_failed = False - self._outputcpp_generated = False + self._output_file_generated = False def get_transport(self): transport = None @@ -105,11 +116,11 @@ def print_progress(self, text): click.secho(text, bold=self.options.get("verbose")) def build_or_upload(self, target): - if not self._outputcpp_generated: - self.generate_outputcpp( + if not self._output_file_generated: + self.generate_output_file( self.options["project_config"].get_optional_dir("test") ) - self._outputcpp_generated = True + self._output_file_generated = True if self.test_name != "*": self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name @@ -147,10 +158,10 @@ def on_run_out(self, line): else: click.echo(line) - def generate_outputcpp(self, test_dir): + def generate_output_file(self, test_dir): assert isdir(test_dir) - cpp_tpl = "\n".join( + file_tpl = "\n".join( [ "$include", "#include ", @@ -194,10 +205,12 @@ def delete_tmptest_file(file_): fg="yellow", ) - tpl = Template(cpp_tpl).substitute(TRANSPORT_OPTIONS[self.get_transport()]) + transport_options = TRANSPORT_OPTIONS[self.get_transport()] + tpl = Template(file_tpl).substitute(transport_options) data = Template(tpl).substitute(baudrate=self.get_baudrate()) - - tmp_file = join(test_dir, "output_export.cpp") + tmp_file = join( + test_dir, "output_export." + transport_options.get("language", "c") + ) with open(tmp_file, "w") as fp: fp.write(data) From aba2ea97465c45d4538eb6752d921865a79442bf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 Apr 2020 12:44:52 +0300 Subject: [PATCH 26/69] Temporary disable "infineonxmc" from CI due to a broken dev-platform --- HISTORY.rst | 1 + tests/test_examples.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4035d7af94..acaa6ef386 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ PlatformIO Core 4 4.3.2 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +* Fixed PIO Unit Testing for Zephyr RTOS * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) * Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 `_) diff --git a/tests/test_examples.py b/tests/test_examples.py index b0d0dfcbf8..ea2714602a 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -38,7 +38,7 @@ def pytest_generate_tests(metafunc): p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) ignore_conds = [ not p.is_embedded(), - p.name == "ststm8", + p.name in ("ststm8", "infineonxmc"), # issue with "version `CXXABI_1.3.9' not found (required by sdcc)" "linux" in util.get_systype() and p.name == "intel_mcs51", ] From ccc43633b760029212a246d1d148ff30132bc673 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 Apr 2020 13:14:10 +0300 Subject: [PATCH 27/69] Support for a new dev-platform NXP i.MX RT --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 22080ab244..c861add2ab 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 22080ab2442f07216a7c2727723b9f80fd5090c0 +Subproject commit c861add2abda3fc8c2e0b4fb4c0413a18e54e621 diff --git a/examples b/examples index e750a6dfa2..8805b1833c 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit e750a6dfa2f54cbfe9013224a4438e2d35dc8da7 +Subproject commit 8805b1833c97fefcc8a5b9100a8f460810ce1a85 From 030ddf4ea102312017793806115d0ce7e554e593 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 Apr 2020 17:08:16 +0300 Subject: [PATCH 28/69] Apply black formatting --- platformio/commands/test/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index ac48a69206..9024ed0eec 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -51,7 +51,7 @@ "zephyr": { "include": "#include ", "object": "", - "putchar": "printk(\"%c\", c)", + "putchar": 'printk("%c", c)', "flush": "", "begin": "", "end": "", From ae57829190c0983b16833a6541ad0f2580b5c26d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 Apr 2020 17:59:58 +0300 Subject: [PATCH 29/69] Generate user agent based on PIO Core environment --- platformio/app.py | 14 +++++++++++++- platformio/commands/upgrade.py | 7 ++++--- platformio/downloader.py | 4 ++-- platformio/managers/package.py | 2 +- platformio/telemetry.py | 23 ++++++++++------------- platformio/util.py | 10 +++------- 6 files changed, 33 insertions(+), 27 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index f53b17d0d8..e32004ab97 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -15,6 +15,7 @@ import codecs import hashlib import os +import platform import uuid from os import environ, getenv, listdir, remove from os.path import dirname, isdir, isfile, join, realpath @@ -22,7 +23,7 @@ import requests -from platformio import exception, fs, lockfile +from platformio import __version__, exception, fs, lockfile from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.proc import is_ci from platformio.project.helpers import ( @@ -414,3 +415,14 @@ def get_cid(): if WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) return cid + + +def get_user_agent(): + data = ["PlatformIO/%s" % __version__, "CI/%d" % int(is_ci())] + if get_session_var("caller_id"): + data.append("Caller/%s" % get_session_var("caller_id")) + if os.getenv("PLATFORMIO_IDE"): + data.append("IDE/%s" % os.getenv("PLATFORMIO_IDE")) + data.append("Python/%s" % platform.python_version()) + data.append("Platform/%s" % platform.platform()) + return " ".join(data) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 947b7a0bdc..6303ea69a5 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -19,7 +19,7 @@ import click import requests -from platformio import VERSION, __version__, app, exception, util +from platformio import VERSION, __version__, app, exception from platformio.compat import WINDOWS from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir @@ -133,7 +133,7 @@ def get_develop_latest_version(): r = requests.get( "https://raw.githubusercontent.com/platformio/platformio" "/develop/platformio/__init__.py", - headers=util.get_request_defheaders(), + headers={"User-Agent": app.get_user_agent()}, ) r.raise_for_status() for line in r.text.split("\n"): @@ -153,7 +153,8 @@ def get_develop_latest_version(): def get_pypi_latest_version(): r = requests.get( - "https://pypi.org/pypi/platformio/json", headers=util.get_request_defheaders() + "https://pypi.org/pypi/platformio/json", + headers={"User-Agent": app.get_user_agent()}, ) r.raise_for_status() return r.json()["info"]["version"] diff --git a/platformio/downloader.py b/platformio/downloader.py index c6ecda807c..21f5477bd8 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -23,7 +23,7 @@ import click import requests -from platformio import util +from platformio import app, util from platformio.exception import ( FDSHASumMismatch, FDSizeMismatch, @@ -38,7 +38,7 @@ def __init__(self, url, dest_dir=None): self._request = requests.get( url, stream=True, - headers=util.get_request_defheaders(), + headers={"User-Agent": app.get_user_agent()}, verify=sys.version_info >= (2, 7, 9), ) if self._request.status_code != 200: diff --git a/platformio/managers/package.py b/platformio/managers/package.py index d6481541b1..adb9dcb560 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -53,7 +53,7 @@ def __next__(self): def load_manifest(url): r = None try: - r = requests.get(url, headers=util.get_request_defheaders()) + r = requests.get(url, headers={"User-Agent": app.get_user_agent()}) r.raise_for_status() return r.json() except: # pylint: disable=bare-except diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 54bd8c4bc5..1e12584d53 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -13,8 +13,8 @@ # limitations under the License. import atexit +import hashlib import os -import platform import re import sys import threading @@ -27,8 +27,9 @@ from platformio import __version__, app, exception, util from platformio.commands import PlatformioCLI -from platformio.compat import string_types +from platformio.compat import hashlib_encode_data, string_types from platformio.proc import is_ci, is_container +from platformio.project.helpers import is_platformio_project try: import queue @@ -93,15 +94,7 @@ def __setitem__(self, name, value): def _prefill_appinfo(self): self["av"] = __version__ - - # gather dependent packages - dpdata = [] - dpdata.append("PlatformIO/%s" % __version__) - if app.get_session_var("caller_id"): - dpdata.append("Caller/%s" % app.get_session_var("caller_id")) - if os.getenv("PLATFORMIO_IDE"): - dpdata.append("IDE/%s" % os.getenv("PLATFORMIO_IDE")) - self["an"] = " ".join(dpdata) + self["an"] = app.get_user_agent() def _prefill_sysargs(self): args = [] @@ -127,7 +120,6 @@ def _filter_args(items): caller_id = str(app.get_session_var("caller_id")) self["cd1"] = util.get_systype() - self["cd2"] = "Python/%s %s" % (platform.python_version(), platform.platform()) self["cd4"] = ( 1 if (not util.is_ci() and (caller_id or not is_container())) else 0 ) @@ -269,7 +261,7 @@ def _send_data(self, data): r = self._http_session.post( "https://ssl.google-analytics.com/collect", data=data, - headers=util.get_request_defheaders(), + headers={"User-Agent": app.get_user_agent()}, timeout=1, ) r.raise_for_status() @@ -331,15 +323,20 @@ def measure_ci(): def encode_run_environment(options): non_sensative_keys = [ "platform", + "platform_packages", "framework", "board", "upload_protocol", "check_tool", "debug_tool", + "monitor_filters", ] safe_options = [ "%s=%s" % (k, v) for k, v in sorted(options.items()) if k in non_sensative_keys ] + if is_platformio_project(os.getcwd()): + phash = hashlib.sha1(hashlib_encode_data(app.get_cid())) + safe_options.append("pid=%s" % phash.hexdigest()) return "&".join(safe_options) diff --git a/platformio/util.py b/platformio/util.py index d35ce6a20a..9cda5ebf56 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -270,11 +270,6 @@ def get_services(self): return items -def get_request_defheaders(): - data = (__version__, int(is_ci()), requests.utils.default_user_agent()) - return {"User-Agent": "PlatformIO/%s CI/%d %s" % data} - - @memoized(expire="60s") def _api_request_session(): return requests.Session() @@ -284,18 +279,19 @@ def _api_request_session(): def _get_api_result( url, params=None, data=None, auth=None # pylint: disable=too-many-branches ): - from platformio.app import get_setting # pylint: disable=import-outside-toplevel + # pylint: disable=import-outside-toplevel + from platformio.app import get_user_agent, get_setting result = {} r = None verify_ssl = sys.version_info >= (2, 7, 9) - headers = get_request_defheaders() if not url.startswith("http"): url = __apiurl__ + url if not get_setting("strict_ssl"): url = url.replace("https://", "http://") + headers = {"User-Agent": get_user_agent()} try: if data: r = _api_request_session().post( From 5ba7753bfa8c2709b220dccd7a7dbbcb22d1f322 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 12 Apr 2020 17:43:27 +0300 Subject: [PATCH 30/69] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index c861add2ab..88d032b71f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c861add2abda3fc8c2e0b4fb4c0413a18e54e621 +Subproject commit 88d032b71fe28eae8b145801aaaea05cafd8a14c From 788351a0cd2559e6b1cd917de7b7b3650408cec2 Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 13 Apr 2020 16:41:03 +0300 Subject: [PATCH 31/69] Fixed an incorrect node path used for pattern matching when processing middleware nodes --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index acaa6ef386..a7664799ee 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ PlatformIO Core 4 * Fixed PIO Unit Testing for Zephyr RTOS * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) * Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 `_) +* Fixed an incorrect node path used for pattern matching when processing middleware nodes 4.3.1 (2020-03-20) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 1132f447e0..c0cc11de71 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -285,7 +285,7 @@ def CollectBuildFiles( for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []): tmp = [] for node in sources: - if pattern and not fnmatch.fnmatch(node.get_path(), pattern): + if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern): tmp.append(node) continue n = callback(node) From 1f4aff7f27291084241255e6612db62d79ac018c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 Apr 2020 16:07:02 +0300 Subject: [PATCH 32/69] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 88d032b71f..ed17733aa3 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 88d032b71fe28eae8b145801aaaea05cafd8a14c +Subproject commit ed17733aa3184561242baf2b892042df76516664 From 445ca937fdefc8938c402f8117df083c56a71e1e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 18 Apr 2020 22:40:27 +0300 Subject: [PATCH 33/69] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index ed17733aa3..2bf2daaa0a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ed17733aa3184561242baf2b892042df76516664 +Subproject commit 2bf2daaa0a9b33d88d809d6d529059304e72f855 From 7780003d01178afa190a9734225b4db34256fd80 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Sun, 19 Apr 2020 19:06:06 +0300 Subject: [PATCH 34/69] New Account Management System (#3443) * add login for PIO account to account cli * Remove PyJWT lib. Fixes. * Add password change for account * Refactoring. Add Account Client. * Fixes. * http -> https. * adding error handling for expired session. * Change broker requests from json to form-data. * Add pio accoint register command. fixes * Fixes. * Fixes. * Add username and password validation * fixes * Add token, forgot commands to pio account * fix domain * add update command for pio account * fixes * refactor profile update output * lint * Update exception text. * Fix logout * Add custom user-agent for pio account * add profile show command. minor fixes. * Fix pio account show output format. * Move account related exceptions * cleaning * minor fix * Remove try except for account command authenticated/non-authenticated errors * fix profile update cli command * rename first name and last name vars to 'firstname' and 'lastname' --- platformio/commands/account.py | 72 ----- platformio/commands/account/__init__.py | 13 + platformio/commands/account/client.py | 217 ++++++++++++++ platformio/commands/account/command.py | 278 ++++++++++++++++++ platformio/commands/account/exception.py | 30 ++ .../commands/home/rpc/handlers/piocore.py | 2 +- 6 files changed, 539 insertions(+), 73 deletions(-) delete mode 100644 platformio/commands/account.py create mode 100644 platformio/commands/account/__init__.py create mode 100644 platformio/commands/account/client.py create mode 100644 platformio/commands/account/command.py create mode 100644 platformio/commands/account/exception.py diff --git a/platformio/commands/account.py b/platformio/commands/account.py deleted file mode 100644 index d728a558dc..0000000000 --- a/platformio/commands/account.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=unused-argument - -import sys - -import click - -from platformio.managers.core import pioplus_call - - -@click.group("account", short_help="Manage PIO Account") -def cli(): - pass - - -@cli.command("register", short_help="Create new PIO Account") -@click.option("-u", "--username") -def account_register(**kwargs): - pioplus_call(sys.argv[1:]) - - -@cli.command("login", short_help="Log in to PIO Account") -@click.option("-u", "--username") -@click.option("-p", "--password") -def account_login(**kwargs): - pioplus_call(sys.argv[1:]) - - -@cli.command("logout", short_help="Log out of PIO Account") -def account_logout(): - pioplus_call(sys.argv[1:]) - - -@cli.command("password", short_help="Change password") -@click.option("--old-password") -@click.option("--new-password") -def account_password(**kwargs): - pioplus_call(sys.argv[1:]) - - -@cli.command("token", short_help="Get or regenerate Authentication Token") -@click.option("-p", "--password") -@click.option("--regenerate", is_flag=True) -@click.option("--json-output", is_flag=True) -def account_token(**kwargs): - pioplus_call(sys.argv[1:]) - - -@cli.command("forgot", short_help="Forgot password") -@click.option("-u", "--username") -def account_forgot(**kwargs): - pioplus_call(sys.argv[1:]) - - -@cli.command("show", short_help="PIO Account information") -@click.option("--offline", is_flag=True) -@click.option("--json-output", is_flag=True) -def account_show(**kwargs): - pioplus_call(sys.argv[1:]) diff --git a/platformio/commands/account/__init__.py b/platformio/commands/account/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/commands/account/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py new file mode 100644 index 0000000000..4c49922a76 --- /dev/null +++ b/platformio/commands/account/client.py @@ -0,0 +1,217 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import os +import time + +import requests.adapters +from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error + +from platformio import app +from platformio.commands.account import exception + + +class AccountClient(object): + def __init__( + self, api_base_url="https://api.accounts.platformio.org", retries=3, + ): + if api_base_url.endswith("/"): + api_base_url = api_base_url[:-1] + self.api_base_url = api_base_url + self._session = requests.Session() + self._session.headers.update({"User-Agent": app.get_user_agent()}) + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=2, + method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], + ) + adapter = requests.adapters.HTTPAdapter(max_retries=retry) + self._session.mount(api_base_url, adapter) + + def login(self, username, password): + try: + self.fetch_authentication_token() + except: # pylint:disable=bare-except + pass + else: + raise exception.AccountAlreadyAuthenticated( + app.get_state_item("account", {}).get("email", "") + ) + + response = self._session.post( + self.api_base_url + "/v1/login", + data={"username": username, "password": password}, + ) + result = self.raise_error_from_response(response) + app.set_state_item("account", result) + return result + + def logout(self): + try: + refresh_token = self.get_refresh_token() + except: # pylint:disable=bare-except + raise exception.AccountNotAuthenticated() + response = requests.post( + self.api_base_url + "/v1/logout", data={"refresh_token": refresh_token}, + ) + try: + self.raise_error_from_response(response) + except exception.AccountError: + pass + app.delete_state_item("account") + return True + + def change_password(self, old_password, new_password): + try: + token = self.fetch_authentication_token() + except: # pylint:disable=bare-except + raise exception.AccountNotAuthenticated() + response = self._session.post( + self.api_base_url + "/v1/password", + headers={"Authorization": "Bearer %s" % token}, + data={"old_password": old_password, "new_password": new_password}, + ) + self.raise_error_from_response(response) + return True + + def registration( + self, username, email, password, firstname, lastname + ): # pylint:disable=too-many-arguments + try: + self.fetch_authentication_token() + except: # pylint:disable=bare-except + pass + else: + raise exception.AccountAlreadyAuthenticated( + app.get_state_item("account", {}).get("email", "") + ) + + response = self._session.post( + self.api_base_url + "/v1/registration", + data={ + "username": username, + "email": email, + "password": password, + "firstname": firstname, + "lastname": lastname, + }, + ) + return self.raise_error_from_response(response) + + def auth_token(self, password, regenerate): + try: + token = self.fetch_authentication_token() + except: # pylint:disable=bare-except + raise exception.AccountNotAuthenticated() + response = self._session.post( + self.api_base_url + "/v1/token", + headers={"Authorization": "Bearer %s" % token}, + data={"password": password, "regenerate": 1 if regenerate else 0}, + ) + return self.raise_error_from_response(response).get("auth_token") + + def forgot_password(self, username): + response = self._session.post( + self.api_base_url + "/v1/forgot", data={"username": username}, + ) + return self.raise_error_from_response(response).get("auth_token") + + def get_profile(self): + try: + token = self.fetch_authentication_token() + except: # pylint:disable=bare-except + raise exception.AccountNotAuthenticated() + response = self._session.get( + self.api_base_url + "/v1/profile", + headers={"Authorization": "Bearer %s" % token}, + ) + return self.raise_error_from_response(response) + + def update_profile(self, profile, current_password): + try: + token = self.fetch_authentication_token() + except: # pylint:disable=bare-except + raise exception.AccountNotAuthenticated() + profile["current_password"] = current_password + response = self._session.put( + self.api_base_url + "/v1/profile", + headers={"Authorization": "Bearer %s" % token}, + data=profile, + ) + return self.raise_error_from_response(response) + + def get_account_info(self, offline): + if offline: + account = app.get_state_item("account") + if not account: + raise exception.AccountNotAuthenticated() + return { + "profile": { + "email": account.get("email"), + "username": account.get("username"), + } + } + try: + token = self.fetch_authentication_token() + except: # pylint:disable=bare-except + raise exception.AccountNotAuthenticated() + response = self._session.get( + self.api_base_url + "/v1/summary", + headers={"Authorization": "Bearer %s" % token}, + ) + return self.raise_error_from_response(response) + + def fetch_authentication_token(self): + if "PLATFORMIO_AUTH_TOKEN" in os.environ: + return os.environ["PLATFORMIO_AUTH_TOKEN"] + auth = app.get_state_item("account", {}).get("auth", {}) + if auth.get("access_token") and auth.get("access_token_expire"): + if auth.get("access_token_expire") > time.time(): + return auth.get("access_token") + if auth.get("refresh_token"): + response = self._session.post( + self.api_base_url + "/v1/login", + headers={"Authorization": "Bearer %s" % auth.get("refresh_token")}, + ) + result = self.raise_error_from_response(response) + app.set_state_item("account", result) + return result.get("auth").get("access_token") + raise exception.AccountNotAuthenticated() + + @staticmethod + def get_refresh_token(): + try: + auth = app.get_state_item("account").get("auth").get("refresh_token") + return auth + except: # pylint:disable=bare-except + raise exception.AccountNotAuthenticated() + + @staticmethod + def raise_error_from_response(response, expected_codes=(200, 201, 202)): + if response.status_code in expected_codes: + try: + return response.json() + except ValueError: + pass + try: + message = response.json()["message"] + except (KeyError, ValueError): + message = response.text + if "Authorization session has been expired" in message: + app.delete_state_item("account") + raise exception.AccountError(message) diff --git a/platformio/commands/account/command.py b/platformio/commands/account/command.py new file mode 100644 index 0000000000..e24261a61e --- /dev/null +++ b/platformio/commands/account/command.py @@ -0,0 +1,278 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import datetime +import json +import re + +import click +from tabulate import tabulate + +from platformio.commands.account import exception +from platformio.commands.account.client import AccountClient + + +@click.group("account", short_help="Manage PIO Account") +def cli(): + pass + + +def validate_username(value): + value = str(value).strip() + if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){3,38}$", value, flags=re.I): + raise click.BadParameter( + "Invalid username format. " + "Username must contain at least 4 characters including single hyphens," + " and cannot begin or end with a hyphen" + ) + return value + + +def validate_email(value): + value = str(value).strip() + if not re.match(r"^[a-z\d_.+-]+@[a-z\d\-]+\.[a-z\d\-.]+$", value, flags=re.I): + raise click.BadParameter("Invalid E-Mail address") + return value + + +def validate_password(value): + value = str(value).strip() + if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value): + raise click.BadParameter( + "Invalid password format. " + "Password must contain at least 8 characters" + " including a number and a lowercase letter" + ) + return value + + +@cli.command("register", short_help="Create new PIO Account") +@click.option( + "-u", + "--username", + prompt=True, + callback=lambda _, __, value: validate_username(value), +) +@click.option( + "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value) +) +@click.option( + "-p", + "--password", + prompt=True, + hide_input=True, + confirmation_prompt=True, + callback=lambda _, __, value: validate_password(value), +) +@click.option("--firstname", prompt=True) +@click.option("--lastname", prompt=True) +def account_register(username, email, password, firstname, lastname): + client = AccountClient() + client.registration(username, email, password, firstname, lastname) + return click.secho( + "An account has been successfully created. " + "Please check your mail to activate your account and verify your email address.", + fg="green", + ) + + +@cli.command("login", short_help="Log in to PIO Account") +@click.option("-u", "--username", prompt="Username or e-mail") +@click.option("-p", "--password", prompt=True, hide_input=True) +def account_login(username, password): + client = AccountClient() + client.login(username, password) + return click.secho("Successfully logged in!", fg="green") + + +@cli.command("logout", short_help="Log out of PIO Account") +def account_logout(): + client = AccountClient() + client.logout() + return click.secho("Successfully logged out!", fg="green") + + +@cli.command("password", short_help="Change password") +@click.option("--old-password", prompt=True, hide_input=True) +@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True) +def account_password(old_password, new_password): + client = AccountClient() + client.change_password(old_password, new_password) + return click.secho("Password successfully changed!", fg="green") + + +@cli.command("token", short_help="Get or regenerate Authentication Token") +@click.option("-p", "--password", prompt=True, hide_input=True) +@click.option("--regenerate", is_flag=True) +@click.option("--json-output", is_flag=True) +def account_token(password, regenerate, json_output): + client = AccountClient() + auth_token = client.auth_token(password, regenerate) + if json_output: + return click.echo(json.dumps({"status": "success", "result": auth_token})) + return click.secho("Personal Authentication Token: %s" % auth_token, fg="green") + + +@cli.command("forgot", short_help="Forgot password") +@click.option("--username", prompt="Username or e-mail") +def account_forgot(username): + client = AccountClient() + client.forgot_password(username) + return click.secho( + "If this account is registered, we will send the " + "further instructions to your E-Mail.", + fg="green", + ) + + +@cli.command("update", short_help="Update profile information") +@click.option("--current-password", prompt=True, hide_input=True) +@click.option("--username") +@click.option("--email") +@click.option("--firstname") +@click.option("--lastname") +def account_update(current_password, **kwargs): + client = AccountClient() + profile = client.get_profile() + new_profile = profile.copy() + if not any(kwargs.values()): + for field in profile: + new_profile[field] = click.prompt( + field.replace("_", " ").capitalize(), default=profile[field] + ) + if field == "email": + validate_email(new_profile[field]) + if field == "username": + validate_username(new_profile[field]) + else: + new_profile.update({key: value for key, value in kwargs.items() if value}) + client.update_profile(new_profile, current_password) + click.secho("Profile successfully updated!", fg="green") + username_changed = new_profile["username"] != profile["username"] + email_changed = new_profile["email"] != profile["email"] + if not username_changed and not email_changed: + return None + try: + client.logout() + except exception.AccountNotAuthenticated: + pass + if email_changed: + return click.secho( + "Please check your mail to verify your new email address and re-login. ", + fg="yellow", + ) + return click.secho("Please re-login.", fg="yellow") + + +@cli.command("show", short_help="PIO Account information") +@click.option("--offline", is_flag=True) +@click.option("--json-output", is_flag=True) +def account_show(offline, json_output): + client = AccountClient() + info = client.get_account_info(offline) + if json_output: + return click.echo(json.dumps(info)) + click.echo() + if info.get("profile"): + print_profile(info["profile"]) + if info.get("packages"): + print_packages(info["packages"]) + if info.get("subscriptions"): + print_subscriptions(info["subscriptions"]) + return click.echo() + + +def print_profile(profile): + click.secho("Profile", fg="cyan", bold=True) + click.echo("=" * len("Profile")) + data = [] + if profile.get("username"): + data.append(("Username:", profile["username"])) + if profile.get("email"): + data.append(("Email:", profile["email"])) + if profile.get("firstname"): + data.append(("First name:", profile["firstname"])) + if profile.get("lastname"): + data.append(("Last name:", profile["lastname"])) + click.echo(tabulate(data, tablefmt="plain")) + + +def print_packages(packages): + click.echo() + click.secho("Packages", fg="cyan") + click.echo("=" * len("Packages")) + for package in packages: + click.echo() + click.secho(package.get("name"), bold=True) + click.echo("-" * len(package.get("name"))) + if package.get("description"): + click.echo(package.get("description")) + data = [] + expire = "-" + if "subscription" in package: + expire = datetime.datetime.strptime( + ( + package["subscription"].get("end_at") + or package["subscription"].get("next_bill_at") + ), + "%Y-%m-%dT%H:%M:%SZ", + ).strftime("%Y-%m-%d") + data.append(("Expire:", expire)) + services = [] + for key in package: + if not key.startswith("service."): + continue + if isinstance(package[key], dict): + services.append(package[key].get("title")) + else: + services.append(package[key]) + if services: + data.append(("Services:", ", ".join(services))) + click.echo(tabulate(data, tablefmt="plain")) + + +def print_subscriptions(subscriptions): + click.echo() + click.secho("Subscriptions", fg="cyan") + click.echo("=" * len("Subscriptions")) + for subscription in subscriptions: + click.echo() + click.secho(subscription.get("product_name"), bold=True) + click.echo("-" * len(subscription.get("product_name"))) + data = [("State:", subscription.get("status"))] + begin_at = datetime.datetime.strptime( + subscription.get("begin_at"), "%Y-%m-%dT%H:%M:%SZ" + ).strftime("%Y-%m-%d %H:%M:%S") + data.append(("Start date:", begin_at or "-")) + end_at = subscription.get("end_at") + if end_at: + end_at = datetime.datetime.strptime( + subscription.get("end_at"), "%Y-%m-%dT%H:%M:%SZ" + ).strftime("%Y-%m-%d %H:%M:%S") + data.append(("End date:", end_at or "-")) + next_bill_at = subscription.get("next_bill_at") + if next_bill_at: + next_bill_at = datetime.datetime.strptime( + subscription.get("next_bill_at"), "%Y-%m-%dT%H:%M:%SZ" + ).strftime("%Y-%m-%d %H:%M:%S") + data.append(("Next payment:", next_bill_at or "-")) + data.append( + ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-") + ) + data.append( + ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-") + ) + click.echo(tabulate(data, tablefmt="plain")) diff --git a/platformio/commands/account/exception.py b/platformio/commands/account/exception.py new file mode 100644 index 0000000000..213be0e15e --- /dev/null +++ b/platformio/commands/account/exception.py @@ -0,0 +1,30 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException + + +class AccountError(PlatformioException): + + MESSAGE = "{0}" + + +class AccountNotAuthenticated(AccountError): + + MESSAGE = "You are not authenticated! Please login to PIO Account." + + +class AccountAlreadyAuthenticated(AccountError): + + MESSAGE = "You are already authenticated with {0} account." diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 41009c4ed9..9bcef9e54b 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -96,7 +96,7 @@ def _call_generator(args, options=None): to_json = "--json-output" in args try: - if args and args[0] in ("account", "remote"): + if args and args[0] == "remote": result = yield PIOCoreRPC._call_subprocess(args, options) defer.returnValue(PIOCoreRPC._process_result(result, to_json)) else: From be6bf5052e5eaa8fe33a722ea0222859b6d377bb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 19 Apr 2020 19:26:56 +0300 Subject: [PATCH 35/69] Open source PIO Remote client --- .isort.cfg | 2 +- HISTORY.rst | 1 + Makefile | 2 +- docs | 2 +- platformio/__init__.py | 1 + platformio/app.py | 16 ++ platformio/commands/remote/__init__.py | 13 + platformio/commands/remote/ac/__init__.py | 13 + platformio/commands/remote/ac/base.py | 91 ++++++ platformio/commands/remote/ac/process.py | 42 +++ platformio/commands/remote/ac/psync.py | 66 +++++ platformio/commands/remote/ac/serial.py | 60 ++++ platformio/commands/remote/client/__init__.py | 13 + .../commands/remote/client/agent_list.py | 38 +++ .../commands/remote/client/agent_service.py | 222 ++++++++++++++ .../commands/remote/client/async_base.py | 65 +++++ platformio/commands/remote/client/base.py | 182 ++++++++++++ .../commands/remote/client/device_list.py | 54 ++++ .../commands/remote/client/device_monitor.py | 236 +++++++++++++++ .../commands/remote/client/run_or_test.py | 272 ++++++++++++++++++ .../commands/remote/client/update_core.py | 22 ++ .../commands/{remote.py => remote/command.py} | 194 +++++++++++-- .../commands/remote/factory/__init__.py | 13 + platformio/commands/remote/factory/client.py | 73 +++++ platformio/commands/remote/factory/ssl.py | 41 +++ platformio/commands/remote/projectsync.py | 117 ++++++++ platformio/managers/core.py | 51 +--- 27 files changed, 1822 insertions(+), 80 deletions(-) create mode 100644 platformio/commands/remote/__init__.py create mode 100644 platformio/commands/remote/ac/__init__.py create mode 100644 platformio/commands/remote/ac/base.py create mode 100644 platformio/commands/remote/ac/process.py create mode 100644 platformio/commands/remote/ac/psync.py create mode 100644 platformio/commands/remote/ac/serial.py create mode 100644 platformio/commands/remote/client/__init__.py create mode 100644 platformio/commands/remote/client/agent_list.py create mode 100644 platformio/commands/remote/client/agent_service.py create mode 100644 platformio/commands/remote/client/async_base.py create mode 100644 platformio/commands/remote/client/base.py create mode 100644 platformio/commands/remote/client/device_list.py create mode 100644 platformio/commands/remote/client/device_monitor.py create mode 100644 platformio/commands/remote/client/run_or_test.py create mode 100644 platformio/commands/remote/client/update_core.py rename platformio/commands/{remote.py => remote/command.py} (57%) create mode 100644 platformio/commands/remote/factory/__init__.py create mode 100644 platformio/commands/remote/factory/client.py create mode 100644 platformio/commands/remote/factory/ssl.py create mode 100644 platformio/commands/remote/projectsync.py diff --git a/.isort.cfg b/.isort.cfg index 2270008c4c..de9bf40e83 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=88 -known_third_party=SCons, twisted, autobahn, jsonrpc +known_third_party=OpenSSL, SCons, autobahn, jsonrpc, twisted, zope diff --git a/HISTORY.rst b/HISTORY.rst index a7664799ee..7d5de2733e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ PlatformIO Core 4 4.3.2 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +* Open source `PIO Remote `__ client * Fixed PIO Unit Testing for Zephyr RTOS * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) * Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 `_) diff --git a/Makefile b/Makefile index 548f96f3ef..36b5d3963d 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ format: test: py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py -before-commit: isort format lint test +before-commit: isort format lint clean-docs: rm -rf docs/_build diff --git a/docs b/docs index 2bf2daaa0a..ae721948ba 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2bf2daaa0a9b33d88d809d6d529059304e72f855 +Subproject commit ae721948ba4640e64073d2357b9c2bd47d3430c1 diff --git a/platformio/__init__.py b/platformio/__init__.py index 6096e53af6..8573627ea4 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -34,3 +34,4 @@ __copyright__ = "Copyright 2014-present PlatformIO" __apiurl__ = "https://api.platformio.org" +__pioremote_endpoint__ = "ssl:remote.platformio.org:4413" diff --git a/platformio/app.py b/platformio/app.py index e32004ab97..6c7c7b1ad0 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -13,9 +13,11 @@ # limitations under the License. import codecs +import getpass import hashlib import os import platform +import socket import uuid from os import environ, getenv, listdir, remove from os.path import dirname, isdir, isfile, join, realpath @@ -426,3 +428,17 @@ def get_user_agent(): data.append("Python/%s" % platform.python_version()) data.append("Platform/%s" % platform.platform()) return " ".join(data) + + +def get_host_id(): + h = hashlib.sha1(hashlib_encode_data(get_cid())) + try: + username = getpass.getuser() + h.update(hashlib_encode_data(username)) + except: # pylint: disable=bare-except + pass + return h.hexdigest() + + +def get_host_name(): + return str(socket.gethostname())[:255] diff --git a/platformio/commands/remote/__init__.py b/platformio/commands/remote/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/commands/remote/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/remote/ac/__init__.py b/platformio/commands/remote/ac/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/commands/remote/ac/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/remote/ac/base.py b/platformio/commands/remote/ac/base.py new file mode 100644 index 0000000000..7b76a327a7 --- /dev/null +++ b/platformio/commands/remote/ac/base.py @@ -0,0 +1,91 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + + +class AsyncCommandBase(object): + + MAX_BUFFER_SIZE = 1024 * 1024 # 1Mb + + def __init__(self, options=None, on_end_callback=None): + self.options = options or {} + self.on_end_callback = on_end_callback + self._buffer = b"" + self._return_code = None + self._d = None + self._paused = False + + try: + self.start() + except Exception as e: + raise pb.Error(str(e)) + + @property + def id(self): + return id(self) + + def pause(self): + self._paused = True + self.stop() + + def unpause(self): + self._paused = False + self.start() + + def start(self): + raise NotImplementedError + + def stop(self): + self.transport.loseConnection() # pylint: disable=no-member + + def _ac_ended(self): + if self.on_end_callback: + self.on_end_callback() + if not self._d or self._d.called: + self._d = None + return + if self._buffer: + self._d.callback(self._buffer) + else: + self._d.callback(None) + + def _ac_ondata(self, data): + self._buffer += data + if len(self._buffer) > self.MAX_BUFFER_SIZE: + self._buffer = self._buffer[-1 * self.MAX_BUFFER_SIZE :] + if self._paused: + return + if self._d and not self._d.called: + self._d.callback(self._buffer) + self._buffer = b"" + + def ac_read(self): + if self._buffer: + result = self._buffer + self._buffer = b"" + return result + if self._return_code is None: + self._d = defer.Deferred() + return self._d + return None + + def ac_write(self, data): + self.transport.write(data) # pylint: disable=no-member + return len(data) + + def ac_close(self): + self.stop() + return self._return_code diff --git a/platformio/commands/remote/ac/process.py b/platformio/commands/remote/ac/process.py new file mode 100644 index 0000000000..9e4f6989a3 --- /dev/null +++ b/platformio/commands/remote/ac/process.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from twisted.internet import protocol, reactor # pylint: disable=import-error + +from platformio.commands.remote.ac.base import AsyncCommandBase + + +class ProcessAsyncCmd(protocol.ProcessProtocol, AsyncCommandBase): + def start(self): + env = dict(os.environ).copy() + env.update({"PLATFORMIO_FORCE_ANSI": "true"}) + reactor.spawnProcess( + self, self.options["executable"], self.options["args"], env + ) + + def outReceived(self, data): + self._ac_ondata(data) + + def errReceived(self, data): + self._ac_ondata(data) + + def processExited(self, reason): + self._return_code = reason.value.exitCode + + def processEnded(self, reason): + if self._return_code is None: + self._return_code = reason.value.exitCode + self._ac_ended() diff --git a/platformio/commands/remote/ac/psync.py b/platformio/commands/remote/ac/psync.py new file mode 100644 index 0000000000..6773615cbc --- /dev/null +++ b/platformio/commands/remote/ac/psync.py @@ -0,0 +1,66 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import zlib +from io import BytesIO + +from platformio.commands.remote.ac.base import AsyncCommandBase +from platformio.commands.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync + + +class ProjectSyncAsyncCmd(AsyncCommandBase): + def __init__(self, *args, **kwargs): + self.psync = None + self._upstream = None + super(ProjectSyncAsyncCmd, self).__init__(*args, **kwargs) + + def start(self): + project_dir = os.path.join( + self.options["agent_working_dir"], "projects", self.options["id"] + ) + self.psync = ProjectSync(project_dir) + for name in self.options["items"]: + self.psync.add_item(os.path.join(project_dir, name), name) + + def stop(self): + self.psync = None + self._upstream = None + self._return_code = PROJECT_SYNC_STAGE.COMPLETED.value + + def ac_write(self, data): + stage = PROJECT_SYNC_STAGE.lookupByValue(data.get("stage")) + + if stage is PROJECT_SYNC_STAGE.DBINDEX: + self.psync.rebuild_dbindex() + return zlib.compress(json.dumps(self.psync.get_dbindex()).encode()) + + if stage is PROJECT_SYNC_STAGE.DELETE: + return self.psync.delete_dbindex( + json.loads(zlib.decompress(data["dbindex"])) + ) + + if stage is PROJECT_SYNC_STAGE.UPLOAD: + if not self._upstream: + self._upstream = BytesIO() + self._upstream.write(data["chunk"]) + if self._upstream.tell() == data["total"]: + self.psync.decompress_items(self._upstream) + self._upstream = None + return PROJECT_SYNC_STAGE.EXTRACTED.value + + return PROJECT_SYNC_STAGE.UPLOAD.value + + return None diff --git a/platformio/commands/remote/ac/serial.py b/platformio/commands/remote/ac/serial.py new file mode 100644 index 0000000000..d0181f9cf2 --- /dev/null +++ b/platformio/commands/remote/ac/serial.py @@ -0,0 +1,60 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import sleep + +from twisted.internet import protocol, reactor # pylint: disable=import-error +from twisted.internet.serialport import SerialPort # pylint: disable=import-error + +from platformio.commands.remote.ac.base import AsyncCommandBase + + +class SerialPortAsyncCmd(protocol.Protocol, AsyncCommandBase): + def start(self): + SerialPort( + self, + reactor=reactor, + **{ + "deviceNameOrPortNumber": self.options["port"], + "baudrate": self.options["baud"], + "parity": self.options["parity"], + "rtscts": 1 if self.options["rtscts"] else 0, + "xonxoff": 1 if self.options["xonxoff"] else 0, + } + ) + + def connectionMade(self): + self.reset_device() + if self.options.get("rts", None) is not None: + self.transport.setRTS(self.options.get("rts")) + if self.options.get("dtr", None) is not None: + self.transport.setDTR(self.options.get("dtr")) + + def reset_device(self): + self.transport.flushInput() + self.transport.setDTR(False) + self.transport.setRTS(False) + sleep(0.1) + self.transport.setDTR(True) + self.transport.setRTS(True) + sleep(0.1) + + def dataReceived(self, data): + self._ac_ondata(data) + + def connectionLost(self, reason): # pylint: disable=unused-argument + if self._paused: + return + self._return_code = 0 + self._ac_ended() diff --git a/platformio/commands/remote/client/__init__.py b/platformio/commands/remote/client/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/commands/remote/client/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/remote/client/agent_list.py b/platformio/commands/remote/client/agent_list.py new file mode 100644 index 0000000000..df1de28b74 --- /dev/null +++ b/platformio/commands/remote/client/agent_list.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +import click + +from platformio.commands.remote.client.base import RemoteClientBase + + +class AgentListClient(RemoteClientBase): + def agent_pool_ready(self): + d = self.agentpool.callRemote("list", True) + d.addCallback(self._cbResult) + d.addErrback(self.cb_global_error) + + def _cbResult(self, result): + for item in result: + click.secho(item["name"], fg="cyan") + click.echo("-" * len(item["name"])) + click.echo("ID: %s" % item["id"]) + click.echo( + "Started: %s" + % datetime.fromtimestamp(item["started"]).strftime("%Y-%m-%d %H:%M:%S") + ) + click.echo("") + self.disconnect() diff --git a/platformio/commands/remote/client/agent_service.py b/platformio/commands/remote/client/agent_service.py new file mode 100644 index 0000000000..5918d20550 --- /dev/null +++ b/platformio/commands/remote/client/agent_service.py @@ -0,0 +1,222 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from os.path import getatime, getmtime, isdir, isfile, join + +from twisted.logger import LogLevel # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + +from platformio import proc, util +from platformio.commands.remote.ac.process import ProcessAsyncCmd +from platformio.commands.remote.ac.psync import ProjectSyncAsyncCmd +from platformio.commands.remote.ac.serial import SerialPortAsyncCmd +from platformio.commands.remote.client.base import RemoteClientBase +from platformio.project.config import ProjectConfig +from platformio.project.exception import NotPlatformIOProjectError +from platformio.project.helpers import get_project_core_dir + + +class RemoteAgentService(RemoteClientBase): + def __init__(self, name, share, working_dir=None): + RemoteClientBase.__init__(self) + self.log_level = LogLevel.info + self.working_dir = working_dir or join(get_project_core_dir(), "remote") + if not isdir(self.working_dir): + os.makedirs(self.working_dir) + if name: + self.name = str(name)[:50] + self.join_options.update( + {"agent": True, "share": [s.lower().strip()[:50] for s in share]} + ) + + self._acs = {} + + def agent_pool_ready(self): + pass + + def cb_disconnected(self, reason): + for ac in self._acs.values(): + ac.ac_close() + RemoteClientBase.cb_disconnected(self, reason) + + def remote_acread(self, ac_id): + self.log.debug("Async Read: {id}", id=ac_id) + if ac_id not in self._acs: + raise pb.Error("Invalid Async Identifier") + return self._acs[ac_id].ac_read() + + def remote_acwrite(self, ac_id, data): + self.log.debug("Async Write: {id}", id=ac_id) + if ac_id not in self._acs: + raise pb.Error("Invalid Async Identifier") + return self._acs[ac_id].ac_write(data) + + def remote_acclose(self, ac_id): + self.log.debug("Async Close: {id}", id=ac_id) + if ac_id not in self._acs: + raise pb.Error("Invalid Async Identifier") + return_code = self._acs[ac_id].ac_close() + del self._acs[ac_id] + return return_code + + def remote_cmd(self, cmd, options): + self.log.info("Remote command received: {cmd}", cmd=cmd) + self.log.debug("Command options: {options!r}", options=options) + callback = "_process_cmd_%s" % cmd.replace(".", "_") + return getattr(self, callback)(options) + + def _defer_async_cmd(self, ac, pass_agent_name=True): + self._acs[ac.id] = ac + if pass_agent_name: + return (self.id, ac.id, self.name) + return (self.id, ac.id) + + def _process_cmd_device_list(self, _): + return (self.name, util.get_serialports()) + + def _process_cmd_device_monitor(self, options): + if not options["port"]: + for item in util.get_serialports(): + if "VID:PID" in item["hwid"]: + options["port"] = item["port"] + break + + # terminate opened monitors + if options["port"]: + for ac in list(self._acs.values()): + if ( + isinstance(ac, SerialPortAsyncCmd) + and ac.options["port"] == options["port"] + ): + self.log.info( + "Terminate previously opened monitor at {port}", + port=options["port"], + ) + ac.ac_close() + del self._acs[ac.id] + + if not options["port"]: + raise pb.Error("Please specify serial port using `--port` option") + self.log.info("Starting serial monitor at {port}", port=options["port"]) + + return self._defer_async_cmd(SerialPortAsyncCmd(options), pass_agent_name=False) + + def _process_cmd_psync(self, options): + for ac in list(self._acs.values()): + if ( + isinstance(ac, ProjectSyncAsyncCmd) + and ac.options["id"] == options["id"] + ): + self.log.info("Terminate previous Project Sync process") + ac.ac_close() + del self._acs[ac.id] + + options["agent_working_dir"] = self.working_dir + return self._defer_async_cmd( + ProjectSyncAsyncCmd(options), pass_agent_name=False + ) + + def _process_cmd_run(self, options): + return self._process_cmd_run_or_test("run", options) + + def _process_cmd_test(self, options): + return self._process_cmd_run_or_test("test", options) + + def _process_cmd_run_or_test( # pylint: disable=too-many-locals,too-many-branches + self, command, options + ): + assert options and "project_id" in options + project_dir = join(self.working_dir, "projects", options["project_id"]) + origin_pio_ini = join(project_dir, "platformio.ini") + back_pio_ini = join(project_dir, "platformio.ini.bak") + + # remove insecure project options + try: + conf = ProjectConfig(origin_pio_ini) + if isfile(back_pio_ini): + os.remove(back_pio_ini) + os.rename(origin_pio_ini, back_pio_ini) + # cleanup + if conf.has_section("platformio"): + for opt in conf.options("platformio"): + if opt.endswith("_dir"): + conf.remove_option("platformio", opt) + else: + conf.add_section("platformio") + conf.set("platformio", "build_dir", ".pio/build") + conf.save(origin_pio_ini) + + # restore A/M times + os.utime(origin_pio_ini, (getatime(back_pio_ini), getmtime(back_pio_ini))) + except NotPlatformIOProjectError as e: + raise pb.Error(str(e)) + + cmd_args = ["platformio", "--force", command, "-d", project_dir] + for env in options.get("environment", []): + cmd_args.extend(["-e", env]) + for target in options.get("target", []): + cmd_args.extend(["-t", target]) + for ignore in options.get("ignore", []): + cmd_args.extend(["-i", ignore]) + if options.get("upload_port", False): + cmd_args.extend(["--upload-port", options.get("upload_port")]) + if options.get("test_port", False): + cmd_args.extend(["--test-port", options.get("test_port")]) + if options.get("disable_auto_clean", False): + cmd_args.append("--disable-auto-clean") + if options.get("without_building", False): + cmd_args.append("--without-building") + if options.get("without_uploading", False): + cmd_args.append("--without-uploading") + if options.get("silent", False): + cmd_args.append("-s") + if options.get("verbose", False): + cmd_args.append("-v") + + paused_acs = [] + for ac in self._acs.values(): + if not isinstance(ac, SerialPortAsyncCmd): + continue + self.log.info("Pause active monitor at {port}", port=ac.options["port"]) + ac.pause() + paused_acs.append(ac) + + def _cb_on_end(): + if isfile(back_pio_ini): + if isfile(origin_pio_ini): + os.remove(origin_pio_ini) + os.rename(back_pio_ini, origin_pio_ini) + for ac in paused_acs: + ac.unpause() + self.log.info( + "Unpause active monitor at {port}", port=ac.options["port"] + ) + + return self._defer_async_cmd( + ProcessAsyncCmd( + {"executable": proc.where_is_program("platformio"), "args": cmd_args}, + on_end_callback=_cb_on_end, + ) + ) + + def _process_cmd_update(self, options): + cmd_args = ["platformio", "--force", "update"] + if options.get("only_check"): + cmd_args.append("--only-check") + return self._defer_async_cmd( + ProcessAsyncCmd( + {"executable": proc.where_is_program("platformio"), "args": cmd_args} + ) + ) diff --git a/platformio/commands/remote/client/async_base.py b/platformio/commands/remote/client/async_base.py new file mode 100644 index 0000000000..a07e110b0e --- /dev/null +++ b/platformio/commands/remote/client/async_base.py @@ -0,0 +1,65 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +from twisted.spread import pb # pylint: disable=import-error + +from platformio.commands.remote.client.base import RemoteClientBase + + +class AsyncClientBase(RemoteClientBase): + def __init__(self, command, agents, options): + RemoteClientBase.__init__(self) + self.command = command + self.agents = agents + self.options = options + + self._acs_total = 0 + self._acs_ended = 0 + + def agent_pool_ready(self): + pass + + def cb_async_result(self, result): + if self._acs_total == 0: + self._acs_total = len(result) + for (success, value) in result: + if not success: + raise pb.Error(value) + self.acread_data(*value) + + def acread_data(self, agent_id, ac_id, agent_name=None): + d = self.agentpool.callRemote("acread", agent_id, ac_id) + d.addCallback(self.cb_acread_result, agent_id, ac_id, agent_name) + d.addErrback(self.cb_global_error) + + def cb_acread_result(self, result, agent_id, ac_id, agent_name): + if result is None: + self.acclose(agent_id, ac_id) + else: + if self._acs_total > 1 and agent_name: + click.echo("[%s] " % agent_name, nl=False) + click.echo(result, nl=False) + self.acread_data(agent_id, ac_id, agent_name) + + def acclose(self, agent_id, ac_id): + d = self.agentpool.callRemote("acclose", agent_id, ac_id) + d.addCallback(self.cb_acclose_result) + d.addErrback(self.cb_global_error) + + def cb_acclose_result(self, exit_code): + self._acs_ended += 1 + if self._acs_ended != self._acs_total: + return + self.disconnect(exit_code) diff --git a/platformio/commands/remote/client/base.py b/platformio/commands/remote/client/base.py new file mode 100644 index 0000000000..5e0591eb1f --- /dev/null +++ b/platformio/commands/remote/client/base.py @@ -0,0 +1,182 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from time import time + +import click +from twisted.internet import defer, endpoints, reactor # pylint: disable=import-error +from twisted.logger import ILogObserver # pylint: disable=import-error +from twisted.logger import Logger # pylint: disable=import-error +from twisted.logger import LogLevel # pylint: disable=import-error +from twisted.logger import formatEvent # pylint: disable=import-error +from twisted.python import failure # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error +from zope.interface import provider # pylint: disable=import-error + +from platformio import __pioremote_endpoint__, __version__, app, exception, maintenance +from platformio.commands.remote.factory.client import RemoteClientFactory +from platformio.commands.remote.factory.ssl import SSLContextFactory + + +class RemoteClientBase( # pylint: disable=too-many-instance-attributes + pb.Referenceable +): + + PING_DELAY = 60 + PING_MAX_FAILURES = 3 + DEBUG = False + + def __init__(self): + self.log_level = LogLevel.warn + self.log = Logger(namespace="remote", observer=self._log_observer) + self.id = app.get_host_id() + self.name = app.get_host_name() + self.join_options = {"corever": __version__} + self.perspective = None + self.agentpool = None + + self._ping_id = 0 + self._ping_caller = None + self._ping_counter = 0 + self._reactor_stopped = False + self._exit_code = 0 + + @provider(ILogObserver) + def _log_observer(self, event): + if not self.DEBUG and ( + event["log_namespace"] != self.log.namespace + or self.log_level > event["log_level"] + ): + return + msg = formatEvent(event) + click.echo( + "%s [%s] %s" + % ( + datetime.fromtimestamp(event["log_time"]).strftime("%Y-%m-%d %H:%M:%S"), + event["log_level"].name, + msg, + ) + ) + + def connect(self): + self.log.info("Name: {name}", name=self.name) + self.log.info("Connecting to PIO Remote Cloud") + endpoint = endpoints.clientFromString(reactor, __pioremote_endpoint__) + factory = RemoteClientFactory() + factory.remote_client = self + factory.sslContextFactory = None + if __pioremote_endpoint__.startswith("ssl:"): + # pylint: disable=protected-access + factory.sslContextFactory = SSLContextFactory(endpoint._host) + endpoint._sslContextFactory = factory.sslContextFactory + endpoint.connect(factory) + reactor.run() + + if self._exit_code != 0: + raise exception.ReturnErrorCode(self._exit_code) + + def cb_client_authorization_failed(self, err): + msg = "Bad account credentials" + if err.check(pb.Error): + msg = err.getErrorMessage() + self.log.error(msg) + self.disconnect(exit_code=1) + + def cb_client_authorization_made(self, perspective): + self.log.info("Successfully authorized") + self.perspective = perspective + d = perspective.callRemote("join", self.id, self.name, self.join_options) + d.addCallback(self._cb_client_join_made) + d.addErrback(self.cb_global_error) + + def _cb_client_join_made(self, result): + code = result[0] + if code == 1: + self.agentpool = result[1] + self.agent_pool_ready() + self.restart_ping() + elif code == 2: + self.remote_service(*result[1:]) + + def remote_service(self, command, options): + if command == "disconnect": + self.log.error( + "PIO Remote Cloud disconnected: {msg}", msg=options.get("message") + ) + self.disconnect() + + def restart_ping(self, reset_counter=True): + # stop previous ping callers + self.stop_ping(reset_counter) + self._ping_caller = reactor.callLater(self.PING_DELAY, self._do_ping) + + def _do_ping(self): + self._ping_counter += 1 + self._ping_id = int(time()) + d = self.perspective.callRemote("service", "ping", {"id": self._ping_id}) + d.addCallback(self._cb_pong) + d.addErrback(self._cb_pong) + + def stop_ping(self, reset_counter=True): + if reset_counter: + self._ping_counter = 0 + if not self._ping_caller or not self._ping_caller.active(): + return + self._ping_caller.cancel() + self._ping_caller = None + + def _cb_pong(self, result): + if not isinstance(result, failure.Failure) and self._ping_id == result: + self.restart_ping() + return + if self._ping_counter >= self.PING_MAX_FAILURES: + self.stop_ping() + self.perspective.broker.transport.loseConnection() + else: + self.restart_ping(reset_counter=False) + + def agent_pool_ready(self): + raise NotImplementedError + + def disconnect(self, exit_code=None): + self.stop_ping() + if exit_code is not None: + self._exit_code = exit_code + if reactor.running and not self._reactor_stopped: + self._reactor_stopped = True + reactor.stop() + + def cb_disconnected(self, _): + self.stop_ping() + self.perspective = None + self.agentpool = None + + def cb_global_error(self, err): + if err.check(pb.PBConnectionLost, defer.CancelledError): + return + + msg = err.getErrorMessage() + if err.check(pb.DeadReferenceError): + msg = "Remote Client has been terminated" + elif "PioAgentNotStartedError" in str(err.type): + msg = ( + "Could not find active agents. Please start it before on " + "a remote machine using `pio remote agent start` command.\n" + "See http://docs.platformio.org/page/plus/pio-remote.html" + ) + else: + maintenance.on_platformio_exception(Exception(err.type)) + click.secho(msg, fg="red", err=True) + self.disconnect(exit_code=1) diff --git a/platformio/commands/remote/client/device_list.py b/platformio/commands/remote/client/device_list.py new file mode 100644 index 0000000000..dba1729ff5 --- /dev/null +++ b/platformio/commands/remote/client/device_list.py @@ -0,0 +1,54 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click + +from platformio.commands.remote.client.base import RemoteClientBase + + +class DeviceListClient(RemoteClientBase): + def __init__(self, agents, json_output): + RemoteClientBase.__init__(self) + self.agents = agents + self.json_output = json_output + + def agent_pool_ready(self): + d = self.agentpool.callRemote("cmd", self.agents, "device.list") + d.addCallback(self._cbResult) + d.addErrback(self.cb_global_error) + + def _cbResult(self, result): + data = {} + for (success, value) in result: + if not success: + click.secho(value, fg="red", err=True) + continue + (agent_name, devlist) = value + data[agent_name] = devlist + + if self.json_output: + click.echo(json.dumps(data)) + else: + for agent_name, devlist in data.items(): + click.echo("Agent %s" % click.style(agent_name, fg="cyan", bold=True)) + click.echo("=" * (6 + len(agent_name))) + for item in devlist: + click.secho(item["port"], fg="cyan") + click.echo("-" * len(item["port"])) + click.echo("Hardware ID: %s" % item["hwid"]) + click.echo("Description: %s" % item["description"]) + click.echo("") + self.disconnect() diff --git a/platformio/commands/remote/client/device_monitor.py b/platformio/commands/remote/client/device_monitor.py new file mode 100644 index 0000000000..990bb4337e --- /dev/null +++ b/platformio/commands/remote/client/device_monitor.py @@ -0,0 +1,236 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from fnmatch import fnmatch + +import click +from twisted.internet import protocol, reactor, task # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + +from platformio.commands.remote.client.base import RemoteClientBase + + +class SMBridgeProtocol(protocol.Protocol): # pylint: disable=no-init + def connectionMade(self): + self.factory.add_client(self) + + def connectionLost(self, reason): # pylint: disable=unused-argument + self.factory.remove_client(self) + + def dataReceived(self, data): + self.factory.send_to_server(data) + + +class SMBridgeFactory(protocol.ServerFactory): + def __init__(self, cdm): + self.cdm = cdm + self._clients = [] + + def buildProtocol(self, addr): # pylint: disable=unused-argument + p = SMBridgeProtocol() + p.factory = self # pylint: disable=attribute-defined-outside-init + return p + + def add_client(self, client): + self.cdm.log.debug("SMBridge: Client connected") + self._clients.append(client) + self.cdm.acread_data() + + def remove_client(self, client): + self.cdm.log.debug("SMBridge: Client disconnected") + self._clients.remove(client) + if not self._clients: + self.cdm.client_terminal_stopped() + + def has_clients(self): + return len(self._clients) + + def send_to_clients(self, data): + if not self._clients: + return None + for client in self._clients: + client.transport.write(data) + return len(data) + + def send_to_server(self, data): + self.cdm.acwrite_data(data) + + +class DeviceMonitorClient( # pylint: disable=too-many-instance-attributes + RemoteClientBase +): + + MAX_BUFFER_SIZE = 1024 * 1024 + + def __init__(self, agents, **kwargs): + RemoteClientBase.__init__(self) + self.agents = agents + self.cmd_options = kwargs + + self._bridge_factory = SMBridgeFactory(self) + self._agent_id = None + self._ac_id = None + self._d_acread = None + self._d_acwrite = None + self._acwrite_buffer = "" + + def agent_pool_ready(self): + d = task.deferLater( + reactor, 1, self.agentpool.callRemote, "cmd", self.agents, "device.list" + ) + d.addCallback(self._cb_device_list) + d.addErrback(self.cb_global_error) + + def _cb_device_list(self, result): + devices = [] + hwid_devindexes = [] + for (success, value) in result: + if not success: + click.secho(value, fg="red", err=True) + continue + (agent_name, ports) = value + for item in ports: + if "VID:PID" in item["hwid"]: + hwid_devindexes.append(len(devices)) + devices.append((agent_name, item)) + + if len(result) == 1 and self.cmd_options["port"]: + if set(["*", "?", "[", "]"]) & set(self.cmd_options["port"]): + for agent, item in devices: + if fnmatch(item["port"], self.cmd_options["port"]): + return self.start_remote_monitor(agent, item["port"]) + return self.start_remote_monitor(result[0][1][0], self.cmd_options["port"]) + + device = None + if len(hwid_devindexes) == 1: + device = devices[hwid_devindexes[0]] + else: + click.echo("Available ports:") + for i, device in enumerate(devices): + click.echo( + "{index}. {host}{port} \t{description}".format( + index=i + 1, + host=device[0] + ":" if len(result) > 1 else "", + port=device[1]["port"], + description=device[1]["description"] + if device[1]["description"] != "n/a" + else "", + ) + ) + device_index = click.prompt( + "Please choose a port (number in the list above)", + type=click.Choice([str(i + 1) for i, _ in enumerate(devices)]), + ) + device = devices[int(device_index) - 1] + + self.start_remote_monitor(device[0], device[1]["port"]) + + return None + + def start_remote_monitor(self, agent, port): + options = {"port": port} + for key in ("baud", "parity", "rtscts", "xonxoff", "rts", "dtr"): + options[key] = self.cmd_options[key] + + click.echo( + "Starting Serial Monitor on {host}:{port}".format( + host=agent, port=options["port"] + ) + ) + d = self.agentpool.callRemote("cmd", [agent], "device.monitor", options) + d.addCallback(self.cb_async_result) + d.addErrback(self.cb_global_error) + + def cb_async_result(self, result): + if len(result) != 1: + raise pb.Error("Invalid response from Remote Cloud") + success, value = result[0] + if not success: + raise pb.Error(value) + + reconnected = self._agent_id is not None + self._agent_id, self._ac_id = value + + if reconnected: + self.acread_data(force=True) + self.acwrite_data("", force=True) + return + + # start bridge + port = reactor.listenTCP(0, self._bridge_factory) + address = port.getHost() + self.log.debug("Serial Bridge is started on {address!r}", address=address) + if "sock" in self.cmd_options: + with open(os.path.join(self.cmd_options["sock"], "sock"), "w") as fp: + fp.write("socket://localhost:%d" % address.port) + + def client_terminal_stopped(self): + try: + d = self.agentpool.callRemote("acclose", self._agent_id, self._ac_id) + d.addCallback(lambda r: self.disconnect()) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def acread_data(self, force=False): + if force and self._d_acread: + self._d_acread.cancel() + self._d_acread = None + + if ( + self._d_acread and not self._d_acread.called + ) or not self._bridge_factory.has_clients(): + return + + try: + self._d_acread = self.agentpool.callRemote( + "acread", self._agent_id, self._ac_id + ) + self._d_acread.addCallback(self.cb_acread_result) + self._d_acread.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_acread_result(self, result): + if result is None: + self.disconnect(exit_code=1) + else: + self._bridge_factory.send_to_clients(result) + self.acread_data() + + def acwrite_data(self, data, force=False): + if force and self._d_acwrite: + self._d_acwrite.cancel() + self._d_acwrite = None + + self._acwrite_buffer += data + if len(self._acwrite_buffer) > self.MAX_BUFFER_SIZE: + self._acwrite_buffer = self._acwrite_buffer[-1 * self.MAX_BUFFER_SIZE :] + if (self._d_acwrite and not self._d_acwrite.called) or not self._acwrite_buffer: + return + + data = self._acwrite_buffer + self._acwrite_buffer = "" + try: + d = self.agentpool.callRemote("acwrite", self._agent_id, self._ac_id, data) + d.addCallback(self.cb_acwrite_result) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_acwrite_result(self, result): + assert result > 0 + if self._acwrite_buffer: + self.acwrite_data("") diff --git a/platformio/commands/remote/client/run_or_test.py b/platformio/commands/remote/client/run_or_test.py new file mode 100644 index 0000000000..c986ad0a79 --- /dev/null +++ b/platformio/commands/remote/client/run_or_test.py @@ -0,0 +1,272 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json +import os +import zlib +from io import BytesIO + +from twisted.spread import pb # pylint: disable=import-error + +from platformio import util +from platformio.commands.remote.client.async_base import AsyncClientBase +from platformio.commands.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync +from platformio.compat import hashlib_encode_data +from platformio.project.config import ProjectConfig + + +class RunOrTestClient(AsyncClientBase): + + MAX_ARCHIVE_SIZE = 50 * 1024 * 1024 # 50Mb + UPLOAD_CHUNK_SIZE = 256 * 1024 # 256Kb + + PSYNC_SRC_EXTS = [ + "c", + "cpp", + "S", + "spp", + "SPP", + "sx", + "s", + "asm", + "ASM", + "h", + "hpp", + "ipp", + "ino", + "pde", + "json", + "properties", + ] + + PSYNC_SKIP_DIRS = (".git", ".svn", ".hg", "example", "examples", "test", "tests") + + def __init__(self, *args, **kwargs): + AsyncClientBase.__init__(self, *args, **kwargs) + self.project_id = self.generate_project_id(self.options["project_dir"]) + self.psync = ProjectSync(self.options["project_dir"]) + + def generate_project_id(self, path): + h = hashlib.sha1(hashlib_encode_data(self.id)) + h.update(hashlib_encode_data(path)) + return "%s-%s" % (os.path.basename(path), h.hexdigest()) + + def add_project_items(self, psync): + with util.cd(self.options["project_dir"]): + cfg = ProjectConfig.get_instance( + os.path.join(self.options["project_dir"], "platformio.ini") + ) + psync.add_item(cfg.path, "platformio.ini") + psync.add_item(cfg.get_optional_dir("shared"), "shared") + psync.add_item(cfg.get_optional_dir("boards"), "boards") + + if self.options["force_remote"]: + self._add_project_source_items(cfg, psync) + else: + self._add_project_binary_items(cfg, psync) + + if self.command == "test": + psync.add_item(cfg.get_optional_dir("test"), "test") + + def _add_project_source_items(self, cfg, psync): + psync.add_item(cfg.get_optional_dir("lib"), "lib") + psync.add_item( + cfg.get_optional_dir("include"), + "include", + cb_filter=self._cb_tarfile_filter, + ) + psync.add_item( + cfg.get_optional_dir("src"), "src", cb_filter=self._cb_tarfile_filter + ) + if set(["buildfs", "uploadfs", "uploadfsota"]) & set( + self.options.get("target", []) + ): + psync.add_item(cfg.get_optional_dir("data"), "data") + + @staticmethod + def _add_project_binary_items(cfg, psync): + build_dir = cfg.get_optional_dir("build") + for env_name in os.listdir(build_dir): + env_dir = os.path.join(build_dir, env_name) + if not os.path.isdir(env_dir): + continue + for fname in os.listdir(env_dir): + bin_file = os.path.join(env_dir, fname) + bin_exts = (".elf", ".bin", ".hex", ".eep", "program") + if os.path.isfile(bin_file) and fname.endswith(bin_exts): + psync.add_item( + bin_file, os.path.join(".pio", "build", env_name, fname) + ) + + def _cb_tarfile_filter(self, path): + if ( + os.path.isdir(path) + and os.path.basename(path).lower() in self.PSYNC_SKIP_DIRS + ): + return None + if os.path.isfile(path) and not self.is_file_with_exts( + path, self.PSYNC_SRC_EXTS + ): + return None + return path + + @staticmethod + def is_file_with_exts(path, exts): + if path.endswith(tuple(".%s" % e for e in exts)): + return True + return False + + def agent_pool_ready(self): + self.psync_init() + + def psync_init(self): + self.add_project_items(self.psync) + d = self.agentpool.callRemote( + "cmd", + self.agents, + "psync", + dict(id=self.project_id, items=[i[1] for i in self.psync.get_items()]), + ) + d.addCallback(self.cb_psync_init_result) + d.addErrback(self.cb_global_error) + + # build db index while wait for result from agent + self.psync.rebuild_dbindex() + + def cb_psync_init_result(self, result): + self._acs_total = len(result) + for (success, value) in result: + if not success: + raise pb.Error(value) + agent_id, ac_id = value + try: + d = self.agentpool.callRemote( + "acwrite", + agent_id, + ac_id, + dict(stage=PROJECT_SYNC_STAGE.DBINDEX.value), + ) + d.addCallback(self.cb_psync_dbindex_result, agent_id, ac_id) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_psync_dbindex_result(self, result, agent_id, ac_id): + result = set(json.loads(zlib.decompress(result))) + dbindex = set(self.psync.get_dbindex()) + delete = list(result - dbindex) + delta = list(dbindex - result) + + self.log.debug( + "PSync: stats, total={total}, delete={delete}, delta={delta}", + total=len(dbindex), + delete=len(delete), + delta=len(delta), + ) + + if not delete and not delta: + return self.psync_finalize(agent_id, ac_id) + if not delete: + return self.psync_upload(agent_id, ac_id, delta) + + try: + d = self.agentpool.callRemote( + "acwrite", + agent_id, + ac_id, + dict( + stage=PROJECT_SYNC_STAGE.DELETE.value, + dbindex=zlib.compress(json.dumps(delete).encode()), + ), + ) + d.addCallback(self.cb_psync_delete_result, agent_id, ac_id, delta) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + return None + + def cb_psync_delete_result(self, result, agent_id, ac_id, dbindex): + assert result + self.psync_upload(agent_id, ac_id, dbindex) + + def psync_upload(self, agent_id, ac_id, dbindex): + assert dbindex + fileobj = BytesIO() + compressed = self.psync.compress_items(fileobj, dbindex, self.MAX_ARCHIVE_SIZE) + fileobj.seek(0) + self.log.debug( + "PSync: upload project, size={size}", size=len(fileobj.getvalue()) + ) + self.psync_upload_chunk( + agent_id, ac_id, list(set(dbindex) - set(compressed)), fileobj + ) + + def psync_upload_chunk(self, agent_id, ac_id, dbindex, fileobj): + offset = fileobj.tell() + total = fileobj.seek(0, os.SEEK_END) + # unwind + fileobj.seek(offset) + chunk = fileobj.read(self.UPLOAD_CHUNK_SIZE) + assert chunk + try: + d = self.agentpool.callRemote( + "acwrite", + agent_id, + ac_id, + dict( + stage=PROJECT_SYNC_STAGE.UPLOAD.value, + chunk=chunk, + length=len(chunk), + total=total, + ), + ) + d.addCallback( + self.cb_psync_upload_chunk_result, agent_id, ac_id, dbindex, fileobj + ) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_psync_upload_chunk_result( # pylint: disable=too-many-arguments + self, result, agent_id, ac_id, dbindex, fileobj + ): + result = PROJECT_SYNC_STAGE.lookupByValue(result) + self.log.debug("PSync: upload chunk result {r}", r=str(result)) + assert result & (PROJECT_SYNC_STAGE.UPLOAD | PROJECT_SYNC_STAGE.EXTRACTED) + if result is PROJECT_SYNC_STAGE.EXTRACTED: + if dbindex: + self.psync_upload(agent_id, ac_id, dbindex) + else: + self.psync_finalize(agent_id, ac_id) + else: + self.psync_upload_chunk(agent_id, ac_id, dbindex, fileobj) + + def psync_finalize(self, agent_id, ac_id): + try: + d = self.agentpool.callRemote("acclose", agent_id, ac_id) + d.addCallback(self.cb_psync_completed_result, agent_id) + d.addErrback(self.cb_global_error) + except (AttributeError, pb.DeadReferenceError): + self.disconnect(exit_code=1) + + def cb_psync_completed_result(self, result, agent_id): + assert PROJECT_SYNC_STAGE.lookupByValue(result) + options = self.options.copy() + del options["project_dir"] + options["project_id"] = self.project_id + d = self.agentpool.callRemote("cmd", [agent_id], self.command, options) + d.addCallback(self.cb_async_result) + d.addErrback(self.cb_global_error) diff --git a/platformio/commands/remote/client/update_core.py b/platformio/commands/remote/client/update_core.py new file mode 100644 index 0000000000..49e4488cda --- /dev/null +++ b/platformio/commands/remote/client/update_core.py @@ -0,0 +1,22 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.remote.client.async_base import AsyncClientBase + + +class UpdateCoreClient(AsyncClientBase): + def agent_pool_ready(self): + d = self.agentpool.callRemote("cmd", self.agents, self.command, self.options) + d.addCallback(self.cb_async_result) + d.addErrback(self.cb_global_error) diff --git a/platformio/commands/remote.py b/platformio/commands/remote/command.py similarity index 57% rename from platformio/commands/remote.py rename to platformio/commands/remote/command.py index ca296b694b..869890a87f 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote/command.py @@ -12,30 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-arguments, import-outside-toplevel +# pylint: disable=inconsistent-return-statements + import os -import sys +import subprocess import threading from tempfile import mkdtemp from time import sleep import click -from platformio import exception, fs +from platformio import exception, fs, proc from platformio.commands.device import helpers as device_helpers from platformio.commands.device.command import device_monitor as cmd_device_monitor -from platformio.managers.core import pioplus_call +from platformio.commands.run.command import cli as cmd_run +from platformio.commands.test.command import cli as cmd_test +from platformio.compat import PY2 +from platformio.managers.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError -# pylint: disable=unused-argument - @click.group("remote", short_help="PIO Remote") @click.option("-a", "--agent", multiple=True) -def cli(**kwargs): - pass +@click.pass_context +def cli(ctx, agent): + if PY2: + raise exception.UserSideException( + "PIO Remote requires Python 3.5 or above. \nPlease install the latest " + "Python 3 and reinstall PlatformIO Core using installation script:\n" + "https://docs.platformio.org/page/core/installation.html" + ) + ctx.obj = agent + inject_contrib_pysite() -@cli.group("agent", short_help="Start new agent or list active") +@cli.group("agent", short_help="Start a new agent or list active") def remote_agent(): pass @@ -49,18 +61,17 @@ def remote_agent(): envvar="PLATFORMIO_REMOTE_AGENT_DIR", type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), ) -def remote_agent_start(**kwargs): - pioplus_call(sys.argv[1:]) +def remote_agent_start(name, share, working_dir): + from platformio.commands.remote.client.agent_service import RemoteAgentService - -@remote_agent.command("reload", short_help="Reload agents") -def remote_agent_reload(): - pioplus_call(sys.argv[1:]) + RemoteAgentService(name, share, working_dir).connect() @remote_agent.command("list", short_help="List active agents") def remote_agent_list(): - pioplus_call(sys.argv[1:]) + from platformio.commands.remote.client.agent_list import AgentListClient + + AgentListClient().connect() @cli.command("update", short_help="Update installed Platforms, Packages and Libraries") @@ -73,8 +84,11 @@ def remote_agent_list(): @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) -def remote_update(only_check, dry_run): - pioplus_call(sys.argv[1:]) +@click.pass_obj +def remote_update(agents, only_check, dry_run): + from platformio.commands.remote.client.update_core import UpdateCoreClient + + UpdateCoreClient("update", agents, dict(only_check=only_check or dry_run)).connect() @cli.command("run", short_help="Process project environments remotely") @@ -93,8 +107,65 @@ def remote_update(only_check, dry_run): @click.option("-r", "--force-remote", is_flag=True) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) -def remote_run(**kwargs): - pioplus_call(sys.argv[1:]) +@click.pass_obj +@click.pass_context +def remote_run( + ctx, + agents, + environment, + target, + upload_port, + project_dir, + disable_auto_clean, + force_remote, + silent, + verbose, +): + + from platformio.commands.remote.client.run_or_test import RunOrTestClient + + cr = RunOrTestClient( + "run", + agents, + dict( + environment=environment, + target=target, + upload_port=upload_port, + project_dir=project_dir, + disable_auto_clean=disable_auto_clean, + force_remote=force_remote, + silent=silent, + verbose=verbose, + ), + ) + if force_remote: + return cr.connect() + + click.secho("Building project locally", bold=True) + local_targets = [] + if "clean" in target: + local_targets = ["clean"] + elif set(["buildfs", "uploadfs", "uploadfsota"]) & set(target): + local_targets = ["buildfs"] + else: + local_targets = ["checkprogsize", "buildprog"] + ctx.invoke( + cmd_run, + environment=environment, + target=local_targets, + project_dir=project_dir, + # disable_auto_clean=True, + silent=silent, + verbose=verbose, + ) + + if any(["upload" in t for t in target] + ["program" in target]): + click.secho("Uploading firmware remotely", bold=True) + cr.options["target"] += ("nobuild",) + cr.options["disable_auto_clean"] = True + cr.connect() + + return True @cli.command("test", short_help="Remote Unit Testing") @@ -114,8 +185,59 @@ def remote_run(**kwargs): @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @click.option("--verbose", "-v", is_flag=True) -def remote_test(**kwargs): - pioplus_call(sys.argv[1:]) +@click.pass_obj +@click.pass_context +def remote_test( + ctx, + agents, + environment, + ignore, + upload_port, + test_port, + project_dir, + force_remote, + without_building, + without_uploading, + verbose, +): + + from platformio.commands.remote.client.run_or_test import RunOrTestClient + + cr = RunOrTestClient( + "test", + agents, + dict( + environment=environment, + ignore=ignore, + upload_port=upload_port, + test_port=test_port, + project_dir=project_dir, + force_remote=force_remote, + without_building=without_building, + without_uploading=without_uploading, + verbose=verbose, + ), + ) + if force_remote: + return cr.connect() + + click.secho("Building project locally", bold=True) + + ctx.invoke( + cmd_test, + environment=environment, + ignore=ignore, + project_dir=project_dir, + without_uploading=True, + without_testing=True, + verbose=verbose, + ) + + click.secho("Testing project remotely", bold=True) + cr.options["without_building"] = True + cr.connect() + + return True @cli.group("device", short_help="Monitor remote device or list existing") @@ -125,8 +247,11 @@ def remote_device(): @remote_device.command("list", short_help="List remote devices") @click.option("--json-output", is_flag=True) -def device_list(json_output): - pioplus_call(sys.argv[1:]) +@click.pass_obj +def device_list(agents, json_output): + from platformio.commands.remote.client.device_list import DeviceListClient + + DeviceListClient(agents, json_output).connect() @remote_device.command("monitor", short_help="Monitor remote device") @@ -193,8 +318,20 @@ def device_list(json_output): "--environment", help="Load configuration from `platformio.ini` and specified environment", ) +@click.option( + "--sock", + type=click.Path( + exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True + ), +) +@click.pass_obj @click.pass_context -def device_monitor(ctx, **kwargs): +def device_monitor(ctx, agents, **kwargs): + from platformio.commands.remote.client.device_monitor import DeviceMonitorClient + + if kwargs["sock"]: + return DeviceMonitorClient(agents, **kwargs).connect() + project_options = {} try: with fs.cd(kwargs["project_dir"]): @@ -209,12 +346,9 @@ def _tx_target(sock_dir): pioplus_argv = ["remote", "device", "monitor"] pioplus_argv.extend(device_helpers.options_to_argv(kwargs, project_options)) pioplus_argv.extend(["--sock", sock_dir]) - try: - pioplus_call(pioplus_argv) - except exception.ReturnErrorCode: - pass + subprocess.call([proc.where_is_program("platformio")] + pioplus_argv) - sock_dir = mkdtemp(suffix="pioplus") + sock_dir = mkdtemp(suffix="pio") sock_file = os.path.join(sock_dir, "sock") try: t = threading.Thread(target=_tx_target, args=(sock_dir,)) @@ -229,3 +363,5 @@ def _tx_target(sock_dir): t.join(2) finally: fs.rmtree(sock_dir) + + return True diff --git a/platformio/commands/remote/factory/__init__.py b/platformio/commands/remote/factory/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/commands/remote/factory/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/remote/factory/client.py b/platformio/commands/remote/factory/client.py new file mode 100644 index 0000000000..202c7da62f --- /dev/null +++ b/platformio/commands/remote/factory/client.py @@ -0,0 +1,73 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.cred import credentials # pylint: disable=import-error +from twisted.internet import protocol, reactor # pylint: disable=import-error +from twisted.spread import pb # pylint: disable=import-error + +from platformio.app import get_host_id +from platformio.commands.account.client import AccountClient + + +class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory): + def clientConnectionMade(self, broker): + if self.sslContextFactory and not self.sslContextFactory.certificate_verified: + self.remote_client.log.error( + "A remote cloud could not prove that its security certificate is " + "from {host}. This may cause a misconfiguration or an attacker " + "intercepting your connection.", + host=self.sslContextFactory.host, + ) + return self.remote_client.disconnect() + pb.PBClientFactory.clientConnectionMade(self, broker) + protocol.ReconnectingClientFactory.resetDelay(self) + self.remote_client.log.info("Successfully connected") + self.remote_client.log.info("Authenticating") + + d = self.login( + credentials.UsernamePassword( + AccountClient().fetch_authentication_token().encode(), + get_host_id().encode(), + ), + client=self.remote_client, + ) + d.addCallback(self.remote_client.cb_client_authorization_made) + d.addErrback(self.remote_client.cb_client_authorization_failed) + return d + + def clientConnectionFailed(self, connector, reason): + self.remote_client.log.warn( + "Could not connect to PIO Remote Cloud. Reconnecting..." + ) + self.remote_client.cb_disconnected(reason) + protocol.ReconnectingClientFactory.clientConnectionFailed( + self, connector, reason + ) + + def clientConnectionLost( # pylint: disable=arguments-differ + self, connector, unused_reason + ): + if not reactor.running: + self.remote_client.log.info("Successfully disconnected") + return + self.remote_client.log.warn( + "Connection is lost to PIO Remote Cloud. Reconnecting" + ) + pb.PBClientFactory.clientConnectionLost( + self, connector, unused_reason, reconnecting=1 + ) + self.remote_client.cb_disconnected(unused_reason) + protocol.ReconnectingClientFactory.clientConnectionLost( + self, connector, unused_reason + ) diff --git a/platformio/commands/remote/factory/ssl.py b/platformio/commands/remote/factory/ssl.py new file mode 100644 index 0000000000..a4233a6923 --- /dev/null +++ b/platformio/commands/remote/factory/ssl.py @@ -0,0 +1,41 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import certifi +from OpenSSL import SSL # pylint: disable=import-error +from twisted.internet import ssl # pylint: disable=import-error + + +class SSLContextFactory(ssl.ClientContextFactory): + def __init__(self, host): + self.host = host + self.certificate_verified = False + + def getContext(self): + ctx = super(SSLContextFactory, self).getContext() + ctx.set_verify( + SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, self.verifyHostname + ) + ctx.load_verify_locations(certifi.where()) + return ctx + + def verifyHostname( # pylint: disable=unused-argument,too-many-arguments + self, connection, x509, errno, depth, status + ): + cn = x509.get_subject().commonName + if cn.startswith("*"): + cn = cn[1:] + if self.host.endswith(cn): + self.certificate_verified = True + return status diff --git a/platformio/commands/remote/projectsync.py b/platformio/commands/remote/projectsync.py new file mode 100644 index 0000000000..867922bd99 --- /dev/null +++ b/platformio/commands/remote/projectsync.py @@ -0,0 +1,117 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tarfile +from binascii import crc32 +from os.path import getmtime, getsize, isdir, isfile, join + +from twisted.python import constants # pylint: disable=import-error + +from platformio.compat import hashlib_encode_data + + +class PROJECT_SYNC_STAGE(constants.Flags): + INIT = constants.FlagConstant() + DBINDEX = constants.FlagConstant() + DELETE = constants.FlagConstant() + UPLOAD = constants.FlagConstant() + EXTRACTED = constants.FlagConstant() + COMPLETED = constants.FlagConstant() + + +class ProjectSync(object): + def __init__(self, path): + self.path = path + if not isdir(self.path): + os.makedirs(self.path) + self.items = [] + self._db = {} + + def add_item(self, path, relpath, cb_filter=None): + self.items.append((path, relpath, cb_filter)) + + def get_items(self): + return self.items + + def rebuild_dbindex(self): + self._db = {} + for (path, relpath, cb_filter) in self.items: + if cb_filter and not cb_filter(path): + continue + self._insert_to_db(path, relpath) + if not isdir(path): + continue + for (root, _, files) in os.walk(path, followlinks=True): + for name in files: + self._insert_to_db( + join(root, name), join(relpath, root[len(path) + 1 :], name) + ) + + def _insert_to_db(self, path, relpath): + if not isfile(path): + return + index_hash = "%s-%s-%s" % (relpath, getmtime(path), getsize(path)) + index = crc32(hashlib_encode_data(index_hash)) + self._db[index] = (path, relpath) + + def get_dbindex(self): + return list(self._db.keys()) + + def delete_dbindex(self, dbindex): + for index in dbindex: + if index not in self._db: + continue + path = self._db[index][0] + if isfile(path): + os.remove(path) + del self._db[index] + self.delete_empty_folders() + return True + + def delete_empty_folders(self): + deleted = False + for item in self.items: + if not isdir(item[0]): + continue + for root, dirs, files in os.walk(item[0]): + if not dirs and not files and root != item[0]: + deleted = True + os.rmdir(root) + if deleted: + return self.delete_empty_folders() + + return True + + def compress_items(self, fileobj, dbindex, max_size): + compressed = [] + total_size = 0 + tar_opts = dict(fileobj=fileobj, mode="w:gz", bufsize=0, dereference=True) + with tarfile.open(**tar_opts) as tgz: + for index in dbindex: + compressed.append(index) + if index not in self._db: + continue + path, relpath = self._db[index] + tgz.add(path, relpath) + total_size += getsize(path) + if total_size > max_size: + break + return compressed + + def decompress_items(self, fileobj): + fileobj.seek(0) + with tarfile.open(fileobj=fileobj, mode="r:gz") as tgz: + tgz.extractall(self.path) + return True diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 64d4808573..55af5fe193 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -16,12 +16,11 @@ import os import subprocess import sys -from os.path import dirname, join -from platformio import __version__, exception, fs, util -from platformio.compat import PY2, WINDOWS +from platformio import exception, util +from platformio.compat import PY2 from platformio.managers.package import PackageManager -from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path +from platformio.proc import get_pythonexe_path from platformio.project.config import ProjectConfig CORE_PACKAGES = { @@ -185,47 +184,3 @@ def get_contrib_pysite_deps(): ) result[0] = twisted_wheel return result - - -def pioplus_call(args, **kwargs): - if WINDOWS and sys.version_info < (2, 7, 6): - raise exception.PlatformioException( - "PlatformIO Remote v%s does not run under Python version %s.\n" - "Minimum supported version is 2.7.6, please upgrade Python.\n" - "Python 3 is not yet supported.\n" % (__version__, sys.version) - ) - - pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus") - pythonexe_path = get_pythonexe_path() - os.environ["PYTHONEXEPATH"] = pythonexe_path - os.environ["PYTHONPYSITEDIR"] = get_core_package_dir("contrib-pysite") - os.environ["PIOCOREPYSITEDIR"] = dirname(fs.get_source_dir() or "") - if dirname(pythonexe_path) not in os.environ["PATH"].split(os.pathsep): - os.environ["PATH"] = (os.pathsep).join( - [dirname(pythonexe_path), os.environ["PATH"]] - ) - copy_pythonpath_to_osenv() - code = subprocess.call([pioplus_path] + args, **kwargs) - - # handle remote update request - if code == 13: - count_attr = "_update_count" - try: - count_value = getattr(pioplus_call, count_attr) - except AttributeError: - count_value = 0 - setattr(pioplus_call, count_attr, 1) - count_value += 1 - setattr(pioplus_call, count_attr, count_value) - if count_value < PIOPLUS_AUTO_UPDATES_MAX: - update_core_packages() - return pioplus_call(args, **kwargs) - - # handle reload request - elif code == 14: - return pioplus_call(args, **kwargs) - - if code != 0: - raise exception.ReturnErrorCode(1) - - return True From cb1058c6934b3f3980692bec4fa1e8794eaeea03 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 19 Apr 2020 20:03:46 +0300 Subject: [PATCH 36/69] New PIO Account with "username" and profile support --- HISTORY.rst | 1 + docs | 2 +- platformio/__init__.py | 1 + platformio/commands/account/client.py | 4 ++-- platformio/commands/account/command.py | 8 ++++---- platformio/commands/remote/command.py | 8 ++++---- platformio/managers/core.py | 5 +---- 7 files changed, 14 insertions(+), 15 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7d5de2733e..8199e368cd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ PlatformIO Core 4 4.3.2 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +* New `PIO Account `__ with "username" and profile support * Open source `PIO Remote `__ client * Fixed PIO Unit Testing for Zephyr RTOS * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) diff --git a/docs b/docs index ae721948ba..14dee6e0b4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ae721948ba4640e64073d2357b9c2bd47d3430c1 +Subproject commit 14dee6e0b45e174633ea61fd7a2420f5231eec57 diff --git a/platformio/__init__.py b/platformio/__init__.py index 8573627ea4..fd914dcd64 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -34,4 +34,5 @@ __copyright__ = "Copyright 2014-present PlatformIO" __apiurl__ = "https://api.platformio.org" +__pioaccount_api__ = "https://api.accounts.platformio.org" __pioremote_endpoint__ = "ssl:remote.platformio.org:4413" diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py index 4c49922a76..5acef85ec3 100644 --- a/platformio/commands/account/client.py +++ b/platformio/commands/account/client.py @@ -20,13 +20,13 @@ import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error -from platformio import app +from platformio import __pioaccount_api__, app from platformio.commands.account import exception class AccountClient(object): def __init__( - self, api_base_url="https://api.accounts.platformio.org", retries=3, + self, api_base_url=__pioaccount_api__, retries=3, ): if api_base_url.endswith("/"): api_base_url = api_base_url[:-1] diff --git a/platformio/commands/account/command.py b/platformio/commands/account/command.py index e24261a61e..539f1a5b6f 100644 --- a/platformio/commands/account/command.py +++ b/platformio/commands/account/command.py @@ -44,7 +44,7 @@ def validate_username(value): def validate_email(value): value = str(value).strip() if not re.match(r"^[a-z\d_.+-]+@[a-z\d\-]+\.[a-z\d\-.]+$", value, flags=re.I): - raise click.BadParameter("Invalid E-Mail address") + raise click.BadParameter("Invalid email address") return value @@ -90,7 +90,7 @@ def account_register(username, email, password, firstname, lastname): @cli.command("login", short_help="Log in to PIO Account") -@click.option("-u", "--username", prompt="Username or e-mail") +@click.option("-u", "--username", prompt="Username or email") @click.option("-p", "--password", prompt=True, hide_input=True) def account_login(username, password): client = AccountClient() @@ -127,13 +127,13 @@ def account_token(password, regenerate, json_output): @cli.command("forgot", short_help="Forgot password") -@click.option("--username", prompt="Username or e-mail") +@click.option("--username", prompt="Username or email") def account_forgot(username): client = AccountClient() client.forgot_password(username) return click.secho( "If this account is registered, we will send the " - "further instructions to your E-Mail.", + "further instructions to your email.", fg="green", ) diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index 869890a87f..d4a1cc34d2 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -343,10 +343,10 @@ def device_monitor(ctx, agents, **kwargs): kwargs["baud"] = kwargs["baud"] or 9600 def _tx_target(sock_dir): - pioplus_argv = ["remote", "device", "monitor"] - pioplus_argv.extend(device_helpers.options_to_argv(kwargs, project_options)) - pioplus_argv.extend(["--sock", sock_dir]) - subprocess.call([proc.where_is_program("platformio")] + pioplus_argv) + subcmd_argv = ["remote", "device", "monitor"] + subcmd_argv.extend(device_helpers.options_to_argv(kwargs, project_options)) + subcmd_argv.extend(["--sock", sock_dir]) + subprocess.call([proc.where_is_program("platformio")] + subcmd_argv) sock_dir = mkdtemp(suffix="pio") sock_file = os.path.join(sock_dir, "sock") diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 55af5fe193..88419f6b21 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,9 +24,8 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": "~3.1.0", + "contrib-piohome": ">=3.2.0-rc.1", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), - "tool-pioplus": "^2.6.1", "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", "tool-cppcheck": "~1.189.0", @@ -34,8 +33,6 @@ "tool-pvs-studio": "~7.5.0", } -PIOPLUS_AUTO_UPDATES_MAX = 100 - # pylint: disable=arguments-differ From 877e84ea1d3e40dd208bb83aaa419f0fb9a821bb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 19 Apr 2020 20:05:21 +0300 Subject: [PATCH 37/69] Bump version to 4.3.2a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index fd914dcd64..6f17439ee1 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, "2a1") +VERSION = (4, 3, "2a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From c4645a9a964ed3b7d91d6fe33e722b1228c3e1b2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 21 Apr 2020 10:43:34 +0300 Subject: [PATCH 38/69] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 14dee6e0b4..ac566a4492 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 14dee6e0b45e174633ea61fd7a2420f5231eec57 +Subproject commit ac566a4492d8e3c4177bc7df1cfc967c8a8784bb From 2960b73da587331d1e9bff3d9762872daaae9f77 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 21 Apr 2020 12:32:03 +0300 Subject: [PATCH 39/69] Fix an issue when PIO Remote agent was not reconnected automatically --- platformio/__init__.py | 2 +- platformio/commands/remote/client/base.py | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6f17439ee1..381359029e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -35,4 +35,4 @@ __apiurl__ = "https://api.platformio.org" __pioaccount_api__ = "https://api.accounts.platformio.org" -__pioremote_endpoint__ = "ssl:remote.platformio.org:4413" +__pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" diff --git a/platformio/commands/remote/client/base.py b/platformio/commands/remote/client/base.py index 5e0591eb1f..806d7bda23 100644 --- a/platformio/commands/remote/client/base.py +++ b/platformio/commands/remote/client/base.py @@ -73,15 +73,26 @@ def _log_observer(self, event): def connect(self): self.log.info("Name: {name}", name=self.name) self.log.info("Connecting to PIO Remote Cloud") - endpoint = endpoints.clientFromString(reactor, __pioremote_endpoint__) + + # pylint: disable=protected-access + proto, options = endpoints._parse(__pioremote_endpoint__) + proto = proto[0] + factory = RemoteClientFactory() factory.remote_client = self factory.sslContextFactory = None - if __pioremote_endpoint__.startswith("ssl:"): - # pylint: disable=protected-access - factory.sslContextFactory = SSLContextFactory(endpoint._host) - endpoint._sslContextFactory = factory.sslContextFactory - endpoint.connect(factory) + if proto == "ssl": + factory.sslContextFactory = SSLContextFactory(options["host"]) + reactor.connectSSL( + options["host"], + int(options["port"]), + factory, + factory.sslContextFactory, + ) + elif proto == "tcp": + reactor.connectTCP(options["host"], int(options["port"]), factory) + else: + raise exception.PlatformioException("Unknown PIO Remote Cloud protocol") reactor.run() if self._exit_code != 0: From 24c04057e90a0d8a145dd49bb1bfa14a1988eab9 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 21 Apr 2020 17:37:55 +0300 Subject: [PATCH 40/69] CLion: Add paths to libraries specified via lib_extra_dirs option (#3463) * Add paths to libraries specified via lib_extra_dirs option Besides, global folders in SRC_LIST seem a bit unnecessary since there might be unused libraries in these folders * Refactor processing of includes when exporting IDE/Editor projects Split includes according to their source. That will help export includes in a more flexible way. For example some IDEs don't need include paths from toolchains * Add new record to history log * Typo fix --- HISTORY.rst | 1 + platformio/builder/tools/pioide.py | 37 +++++++++--------- platformio/ide/projectgenerator.py | 17 +++++++- platformio/ide/tpls/atom/.clang_complete.tpl | 2 +- platformio/ide/tpls/atom/.gcc-flags.json.tpl | 2 +- .../ide/tpls/clion/CMakeListsPrivate.txt.tpl | 39 ++++++++++++++----- .../ide/tpls/codeblocks/platformio.cbp.tpl | 2 +- platformio/ide/tpls/eclipse/.cproject.tpl | 31 ++++----------- platformio/ide/tpls/emacs/.ccls.tpl | 2 +- platformio/ide/tpls/emacs/.clang_complete.tpl | 2 +- .../netbeans/nbproject/configurations.xml.tpl | 5 ++- .../ide/tpls/qtcreator/platformio.pro.tpl | 2 +- platformio/ide/tpls/vim/.ccls.tpl | 2 +- platformio/ide/tpls/vim/.clang_complete.tpl | 2 +- platformio/ide/tpls/vim/.gcc-flags.json.tpl | 2 +- .../tpls/visualstudio/platformio.vcxproj.tpl | 5 ++- .../vscode/.vscode/c_cpp_properties.json.tpl | 8 +--- 17 files changed, 88 insertions(+), 73 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8199e368cd..ddf9870c7b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ PlatformIO Core 4 * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) * Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 `_) * Fixed an incorrect node path used for pattern matching when processing middleware nodes +* Fixed an issue with missing ``lib_extra_dirs`` option in SRC_LIST for CLion (`issue #3460 `_) 4.3.1 (2020-03-20) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 6d54c66348..fe57e8792c 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -25,17 +25,26 @@ def _dump_includes(env): - includes = [] - - for item in env.get("CPPPATH", []): - includes.append(env.subst(item)) + includes = {} + + includes["build"] = [ + env.subst("$PROJECT_INCLUDE_DIR"), + env.subst("$PROJECT_SRC_DIR"), + ] + includes["build"].extend( + [os.path.realpath(env.subst(item)) for item in env.get("CPPPATH", [])] + ) # installed libs + includes["compatlib"] = [] for lb in env.GetLibBuilders(): - includes.extend(lb.get_include_dirs()) + includes["compatlib"].extend( + [os.path.realpath(inc) for inc in lb.get_include_dirs()] + ) # includes from toolchains p = env.PioPlatform() + includes["toolchain"] = [] for name in p.get_installed_packages(): if p.get_package_type(name) != "toolchain": continue @@ -47,22 +56,14 @@ def _dump_includes(env): os.path.join(toolchain_dir, "lib", "gcc", "*", "*", "include*"), ] for g in toolchain_incglobs: - includes.extend(glob(g)) + includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)]) + includes["unity"] = [] unity_dir = get_core_package_dir("tool-unity") if unity_dir: - includes.append(unity_dir) - - includes.extend([env.subst("$PROJECT_INCLUDE_DIR"), env.subst("$PROJECT_SRC_DIR")]) - - # remove duplicates - result = [] - for item in includes: - item = os.path.realpath(item) - if item not in result: - result.append(item) + includes["unity"].append(unity_dir) - return result + return includes def _get_gcc_defines(env): @@ -158,8 +159,6 @@ def DumpIDEData(env): "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()], "defines": _dump_defines(env), "includes": _dump_includes(env), - "cc_flags": env.subst(LINTCCOM), - "cxx_flags": env.subst(LINTCXXCOM), "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")), "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 34eb59f54c..0ca2db1481 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -57,6 +57,20 @@ def get_best_envname(self, boards=None): return envname + @staticmethod + def filter_includes(includes_map, ignore_scopes=None, to_unix_path=True): + ignore_scopes = ignore_scopes or [] + result = [] + for scope, includes in includes_map.items(): + if scope in ignore_scopes: + continue + for include in includes: + if to_unix_path: + include = fs.to_unix_path(include) + if include not in result: + result.append(include) + return result + def _load_tplvars(self): tpl_vars = { "config": self.config, @@ -92,12 +106,13 @@ def _load_tplvars(self): for key, value in tpl_vars.items(): if key.endswith(("_path", "_dir")): tpl_vars[key] = fs.to_unix_path(value) - for key in ("includes", "src_files", "libsource_dirs"): + for key in ("src_files", "libsource_dirs"): if key not in tpl_vars: continue tpl_vars[key] = [fs.to_unix_path(inc) for inc in tpl_vars[key]] tpl_vars["to_unix_path"] = fs.to_unix_path + tpl_vars["filter_includes"] = self.filter_includes return tpl_vars def get_src_files(self): diff --git a/platformio/ide/tpls/atom/.clang_complete.tpl b/platformio/ide/tpls/atom/.clang_complete.tpl index 3b137e3129..6d8e70ed23 100644 --- a/platformio/ide/tpls/atom/.clang_complete.tpl +++ b/platformio/ide/tpls/atom/.clang_complete.tpl @@ -1,4 +1,4 @@ -% for include in includes: +% for include in filter_includes(includes): -I{{include}} % end % for define in defines: diff --git a/platformio/ide/tpls/atom/.gcc-flags.json.tpl b/platformio/ide/tpls/atom/.gcc-flags.json.tpl index 361c2f0484..85b4e9dabc 100644 --- a/platformio/ide/tpls/atom/.gcc-flags.json.tpl +++ b/platformio/ide/tpls/atom/.gcc-flags.json.tpl @@ -4,6 +4,6 @@ "gccDefaultCFlags": "-fsyntax-only {{! to_unix_path(cc_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", "gccDefaultCppFlags": "-fsyntax-only {{! to_unix_path(cxx_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", "gccErrorLimit": 15, - "gccIncludePaths": "{{ ','.join(includes) }}", + "gccIncludePaths": "{{ ','.join(filter_includes(includes)) }}", "gccSuppressWarnings": false } diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index df8171facc..79da3dc945 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -5,10 +5,12 @@ # please create `CMakeListsUser.txt` in the root of project. # The `CMakeListsUser.txt` will not be overwritten by PlatformIO. -% from platformio.project.helpers import (load_project_ide_data) -% +% import os % import re % +% from platformio.compat import WINDOWS +% from platformio.project.helpers import (load_project_ide_data) +% % def _normalize_path(path): % if project_dir in path: % path = path.replace(project_dir, "${CMAKE_CURRENT_LIST_DIR}") @@ -22,12 +24,24 @@ % return path % end % +% def _fix_lib_dirs(lib_dirs): +% result = [] +% for lib_dir in lib_dirs: +% if not os.path.isabs(lib_dir): +% lib_dir = os.path.join(project_dir, lib_dir) +% end +% result.append(to_unix_path(os.path.normpath(lib_dir))) +% end +% return result +% end +% % def _escape(text): % return to_unix_path(text).replace('"', '\\"') % end % % envs = config.envs() + % if len(envs) > 1: set(CMAKE_CONFIGURATION_TYPES "{{ ";".join(envs) }};" CACHE STRING "Build Types reflect PlatformIO Environments" FORCE) % else: @@ -54,13 +68,13 @@ set(CMAKE_CXX_STANDARD {{ cxx_stds[-1] }}) % end if (CMAKE_BUILD_TYPE MATCHES "{{ env_name }}") -%for define in defines: +% for define in defines: add_definitions(-D'{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}') -%end +% end -%for include in includes: - include_directories("{{ _normalize_path(to_unix_path(include)) }}") -%end +% for include in filter_includes(includes): + include_directories("{{ _normalize_path(include) }}") +% end endif() % leftover_envs = list(set(envs) ^ set([env_name])) @@ -76,9 +90,16 @@ if (CMAKE_BUILD_TYPE MATCHES "{{ env }}") add_definitions(-D'{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}') % end -% for include in data["includes"]: +% for include in filter_includes(data["includes"]): include_directories("{{ _normalize_path(to_unix_path(include)) }}") % end endif() % end -FILE(GLOB_RECURSE SRC_LIST "{{ _normalize_path(project_src_dir) }}/*.*" "{{ _normalize_path(project_lib_dir) }}/*.*" "{{ _normalize_path(project_libdeps_dir) }}/*.*") +% +% lib_extra_dirs = _fix_lib_dirs(config.get("env:" + env_name, "lib_extra_dirs", [])) +% src_paths = [project_src_dir, project_lib_dir, project_libdeps_dir] + lib_extra_dirs +FILE(GLOB_RECURSE SRC_LIST +% for path in src_paths: + {{ _normalize_path(path) + "/*.*" }} +% end +) diff --git a/platformio/ide/tpls/codeblocks/platformio.cbp.tpl b/platformio/ide/tpls/codeblocks/platformio.cbp.tpl index ccbe4d7f11..fcd05d4e5f 100644 --- a/platformio/ide/tpls/codeblocks/platformio.cbp.tpl +++ b/platformio/ide/tpls/codeblocks/platformio.cbp.tpl @@ -52,7 +52,7 @@ % for define in defines: % end - % for include in includes: + % for include in filter_includes(includes): % end diff --git a/platformio/ide/tpls/eclipse/.cproject.tpl b/platformio/ide/tpls/eclipse/.cproject.tpl index 8a237f8d5b..d11e95cf8f 100644 --- a/platformio/ide/tpls/eclipse/.cproject.tpl +++ b/platformio/ide/tpls/eclipse/.cproject.tpl @@ -23,10 +23,8 @@