From e6e14be528daea1ffbfb8388f8aca0698e514acb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Nov 2019 16:58:57 +0200 Subject: [PATCH 001/142] Fix framework name shakti-sdk --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index c7be176730..ab25c72d0e 100644 --- a/README.rst +++ b/README.rst @@ -120,7 +120,7 @@ Frameworks * `mbed `_ * `PULP OS `_ * `Pumbaa `_ -* `Shakti `_ +* `Shakti SDK `_ * `Simba `_ * `SPL `_ * `STM32Cube `_ From 3c796ca7c87212b437cd6cbf30bed1821d72f97d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 8 Nov 2019 17:40:11 +0200 Subject: [PATCH 002/142] Cosmetic fixes --- README.rst | 8 ++++---- docs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index ab25c72d0e..1267200854 100644 --- a/README.rst +++ b/README.rst @@ -36,10 +36,10 @@ PlatformIO `PlatformIO `_ an open source ecosystem for embedded development -* **Cross-platform IDE** and **Unified Debugger** -* **Static Code Analyzer** and **Remote Unit Testing** -* **Multi-platform** and **Multi-architecture Build System** -* **Firmware File Explorer** and **Memory Inspection**. +* Cross-platform IDE and Unified Debugger +* Static Code Analyzer and Remote Unit Testing +* Multi-platform and Multi-architecture Build System +* Firmware File Explorer and Memory Inspection. Get Started ----------- diff --git a/docs b/docs index 28f91efb24..fc2ae00cbb 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 28f91efb24b70301c7357956b2aa88dae0ad6cdd +Subproject commit fc2ae00cbb369d1e81f1caf392782c2f30756707 From 0328037b49f4b753b26ee7186bab961bb3f108af Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 11 Nov 2019 22:38:16 +0200 Subject: [PATCH 003/142] Handle project configuration (monitor, test, and upload options) for PIO Remote commands // Resolve #2591 --- HISTORY.rst | 5 +++ docs | 2 +- platformio/commands/device.py | 70 +++++++++++++++++++++-------------- platformio/commands/remote.py | 45 ++++++++++++++++------ platformio/managers/core.py | 2 +- 5 files changed, 82 insertions(+), 42 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 13a15253e4..36b638bef1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,11 @@ Release Notes PlatformIO Core 4.0 ------------------- +4.1.1 (2019-??-??) +~~~~~~~~~~~~~~~~~~ + +* Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) + 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index fc2ae00cbb..5210aa16d7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit fc2ae00cbb369d1e81f1caf392782c2f30756707 +Subproject commit 5210aa16d729bfdc654d2062e109f211e0493fad diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 45f94ae7bb..67c7f05ac9 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -172,18 +172,11 @@ def device_list( # pylint: disable=too-many-branches help="Load configuration from `platformio.ini` and specified environment", ) def device_monitor(**kwargs): # pylint: disable=too-many-branches - env_options = {} + project_options = {} try: with fs.cd(kwargs["project_dir"]): - env_options = get_project_options(kwargs["environment"]) - for k in ("port", "speed", "rts", "dtr"): - k2 = "monitor_%s" % k - if k == "speed": - k = "baud" - if kwargs[k] is None and k2 in env_options: - kwargs[k] = env_options[k2] - if k != "port": - kwargs[k] = int(kwargs[k]) + project_options = get_project_options(kwargs["environment"]) + kwargs = apply_project_monitor_options(kwargs, project_options) except exception.NotPlatformIOProject: pass @@ -191,29 +184,19 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches ports = util.get_serial_ports(filter_hwid=True) if len(ports) == 1: kwargs["port"] = ports[0]["port"] - - sys.argv = ["monitor"] + env_options.get("monitor_flags", []) - for k, v in kwargs.items(): - if k in ("port", "baud", "rts", "dtr", "environment", "project_dir"): - continue - k = "--" + k.replace("_", "-") - if k in env_options.get("monitor_flags", []): - continue - if isinstance(v, bool): - if v: - sys.argv.append(k) - elif isinstance(v, tuple): - for i in v: - sys.argv.extend([k, i]) - else: - sys.argv.extend([k, str(v)]) - - if kwargs["port"] and (set(["*", "?", "[", "]"]) & set(kwargs["port"])): + elif kwargs["port"] and (set(["*", "?", "[", "]"]) & set(kwargs["port"])): for item in util.get_serial_ports(): if fnmatch(item["port"], kwargs["port"]): kwargs["port"] = item["port"] break + # override system argv with patched options + sys.argv = ["monitor"] + options_to_argv( + kwargs, + project_options, + ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"), + ) + try: miniterm.main( default_port=kwargs["port"], @@ -225,6 +208,37 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches raise exception.MinitermException(e) +def apply_project_monitor_options(cli_options, project_options): + for k in ("port", "speed", "rts", "dtr"): + k2 = "monitor_%s" % k + if k == "speed": + k = "baud" + if cli_options[k] is None and k2 in project_options: + cli_options[k] = project_options[k2] + if k != "port": + cli_options[k] = int(cli_options[k]) + return cli_options + + +def options_to_argv(cli_options, project_options, ignore=None): + result = project_options.get("monitor_flags", []) + for k, v in cli_options.items(): + if v is None or (ignore and k in ignore): + continue + k = "--" + k.replace("_", "-") + if k in project_options.get("monitor_flags", []): + continue + if isinstance(v, bool): + if v: + result.append(k) + elif isinstance(v, tuple): + for i in v: + result.extend([k, i]) + else: + result.extend([k, str(v)]) + return result + + def get_project_options(environment=None): config = ProjectConfig.get_instance() config.validate(envs=[environment] if environment else None) diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index a0707c1f15..f4b17ade0d 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -12,17 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import sys import threading -from os import getcwd -from os.path import isfile, join from tempfile import mkdtemp from time import sleep import click from platformio import exception, fs -from platformio.commands.device import device_monitor as cmd_device_monitor +from platformio.commands import device from platformio.managers.core import pioplus_call # pylint: disable=unused-argument @@ -83,7 +82,7 @@ def remote_update(only_check, dry_run): @click.option( "-d", "--project-dir", - default=getcwd, + default=os.getcwd, type=click.Path( exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True ), @@ -104,7 +103,7 @@ def remote_run(**kwargs): @click.option( "-d", "--project-dir", - default=getcwd, + default=os.getcwd, type=click.Path( exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True ), @@ -130,9 +129,7 @@ def device_list(json_output): @remote_device.command("monitor", short_help="Monitor remote device") @click.option("--port", "-p", help="Port, a number or a device name") -@click.option( - "--baud", "-b", type=int, default=9600, help="Set baud rate, default=9600" -) +@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600") @click.option( "--parity", default="N", @@ -183,25 +180,49 @@ def device_list(json_output): is_flag=True, help="Diagnostics: suppress non-error messages, default=Off", ) +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), +) +@click.option( + "-e", + "--environment", + help="Load configuration from `platformio.ini` and specified environment", +) @click.pass_context def device_monitor(ctx, **kwargs): + project_options = {} + try: + with fs.cd(kwargs["project_dir"]): + project_options = device.get_project_options(kwargs["environment"]) + kwargs = device.apply_project_monitor_options(kwargs, project_options) + except exception.NotPlatformIOProject: + pass + + kwargs["baud"] = kwargs["baud"] or 9600 + def _tx_target(sock_dir): + pioplus_argv = ["remote", "device", "monitor"] + pioplus_argv.extend(device.options_to_argv(kwargs, project_options)) + pioplus_argv.extend(["--sock", sock_dir]) try: - pioplus_call(sys.argv[1:] + ["--sock", sock_dir]) + pioplus_call(pioplus_argv) except exception.ReturnErrorCode: pass sock_dir = mkdtemp(suffix="pioplus") - sock_file = join(sock_dir, "sock") + sock_file = os.path.join(sock_dir, "sock") try: t = threading.Thread(target=_tx_target, args=(sock_dir,)) t.start() - while t.is_alive() and not isfile(sock_file): + while t.is_alive() and not os.path.isfile(sock_file): sleep(0.1) if not t.is_alive(): return kwargs["port"] = fs.get_file_contents(sock_file) - ctx.invoke(cmd_device_monitor, **kwargs) + ctx.invoke(device.device_monitor, **kwargs) t.join(2) finally: fs.rmtree(sock_dir) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 8e66aa1916..1f8142b1e2 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -26,7 +26,7 @@ CORE_PACKAGES = { "contrib-piohome": "~3.0.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.5.8", + "tool-pioplus": "^2.6.0", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0", "tool-cppcheck": "~1.189.0", From 56ceee220b74f6de97fc70fde49554accd9aa388 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 11 Nov 2019 22:48:29 +0200 Subject: [PATCH 004/142] Fix invalid build status for unit test when remote is used --- platformio/commands/test/embedded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index dc0d9ef0eb..6f47eafcab 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -46,7 +46,7 @@ def process(self): return False if self.options["without_testing"]: - return None + return True self.print_progress("Testing...") return self.run() From 703b29a05e2481a744e3c5913db814ebf7bdb499 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 11 Nov 2019 23:19:47 +0200 Subject: [PATCH 005/142] Fixed missed descriptions for project options --- platformio/project/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/project/options.py b/platformio/project/options.py index b4c1814fa4..16ada25259 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -582,11 +582,11 @@ def ConfigEnvOption(*args, **kwargs): description="A connection speed (baud rate) to communicate with a target device", type=click.INT, ), - ConfigEnvOption(group="test", name="test_transport", description="",), + ConfigEnvOption(group="test", name="test_transport", description="A transport to communicate with a target device",), ConfigEnvOption( group="test", name="test_build_project_src", - description="", + description="Build project source code in a pair with test code", type=click.BOOL, default=False, ), From 1f796ca0e5a2c2469880a269425b753a8fb96f02 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 11 Nov 2019 23:22:17 +0200 Subject: [PATCH 006/142] Bump version to 4.1.1a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 893d83dbd5..0e626e181e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, 0) +VERSION = (4, 1, "1a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From cbb7869da18c50539ed84732dd626919927e6213 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 12 Nov 2019 13:09:35 +0200 Subject: [PATCH 007/142] Fixed an issue with the broken latest news for PIO Home --- HISTORY.rst | 1 + platformio/commands/home/rpc/handlers/misc.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 36b638bef1..0c25869ea4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ PlatformIO Core 4.0 ~~~~~~~~~~~~~~~~~~ * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) +* Fixed an issue with the broken latest news for PIO Home 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index 992eeebe07..7458500e55 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -23,7 +23,7 @@ class MiscRPC(object): def load_latest_tweets(self, data_url): - cache_key = data_url + cache_key = app.ContentCache.key_from_args(data_url) cache_valid = "7d" with app.ContentCache() as cc: cache_data = cc.get(cache_key) From c6a8e0336773d44fbc1ec3d645821962fffc4d52 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 12 Nov 2019 13:41:39 +0200 Subject: [PATCH 008/142] Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries // Resolve #3264 --- HISTORY.rst | 1 + platformio/builder/tools/pioplatform.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0c25869ea4..ceef275e7e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ PlatformIO Core 4.0 * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Fixed an issue with the broken latest news for PIO Home +* Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index f910aaedaf..a7fbb8b966 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -44,7 +44,8 @@ def BoardConfig(env, board=None): try: board = board or env.get("BOARD") assert board, "BoardConfig: Board is not defined" - config = p.board_config(board) + with fs.cd(env.subst("$PROJECT_DIR")): + config = p.board_config(board) except (AssertionError, exception.UnknownBoard) as e: sys.stderr.write("Error: %s\n" % str(e)) env.Exit(1) From f1d20f591a2cc0ec50bec69f7edc0e49add30172 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 12 Nov 2019 13:41:54 +0200 Subject: [PATCH 009/142] Sync docs --- docs | 2 +- platformio/project/options.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 5210aa16d7..71222071ec 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5210aa16d729bfdc654d2062e109f211e0493fad +Subproject commit 71222071ecbbf0a3db886a6dcc48d9d73e20b107 diff --git a/platformio/project/options.py b/platformio/project/options.py index 16ada25259..3788ba550b 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -582,7 +582,11 @@ def ConfigEnvOption(*args, **kwargs): description="A connection speed (baud rate) to communicate with a target device", type=click.INT, ), - ConfigEnvOption(group="test", name="test_transport", description="A transport to communicate with a target device",), + ConfigEnvOption( + group="test", + name="test_transport", + description="A transport to communicate with a target device", + ), ConfigEnvOption( group="test", name="test_build_project_src", From 7c481291dc60f441b53da5ee8bc6fa4c97942dfa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 12 Nov 2019 18:14:06 +0200 Subject: [PATCH 010/142] Warn about about broken library manifest when scanning dependencies // Resolve #3268 --- HISTORY.rst | 1 + platformio/builder/tools/piolib.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ceef275e7e..d2462c7999 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ PlatformIO Core 4.0 ~~~~~~~~~~~~~~~~~~ * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) +* Warn about about broken library manifest when scanning dependencies (`issue #3268 `_) * Fixed an issue with the broken latest news for PIO Home * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index d5c58ff830..bd4bcf7482 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -33,7 +33,10 @@ from platformio.builder.tools import platformio as piotool from platformio.compat import WINDOWS, hashlib_encode_data, string_types from platformio.managers.lib import LibraryManager -from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.manifest.parser import ( + ManifestParserError, + ManifestParserFactory, +) from platformio.project.options import ProjectOptions @@ -108,7 +111,14 @@ def __init__(self, env, path, manifest=None, verbose=False): self.path = realpath(env.subst(path)) self.verbose = verbose - self._manifest = manifest if manifest else self.load_manifest() + try: + self._manifest = manifest if manifest else self.load_manifest() + except ManifestParserError: + click.secho( + "Warning! Ignoring broken library manifest in " + self.path, fg="yellow" + ) + self._manifest = {} + self._is_dependent = False self._is_built = False self._depbuilders = list() From f0eb177a8e1b97d1abbd9ad4f25aa11be8e160c0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 12 Nov 2019 18:32:10 +0200 Subject: [PATCH 011/142] Check if directory exists before fetching project info --- .../commands/home/rpc/handlers/project.py | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 80340a5a2a..6eb81929a8 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -17,7 +17,6 @@ import os import shutil import time -from os.path import basename, getmtime, isdir, isfile, join, realpath, sep import jsonrpc # pylint: disable=import-error @@ -38,7 +37,7 @@ def config_call(init_kwargs, method, *args): assert isinstance(init_kwargs, dict) assert "path" in init_kwargs project_dir = get_project_dir() - if isfile(init_kwargs["path"]): + if os.path.isfile(init_kwargs["path"]): project_dir = os.path.dirname(init_kwargs["path"]) with fs.cd(project_dir): return getattr(ProjectConfig(**init_kwargs), method)(*args) @@ -86,7 +85,7 @@ def _get_project_data(): for section in config.sections(): if not section.startswith("env:"): continue - data["envLibdepsDirs"].append(join(libdeps_dir, section[4:])) + data["envLibdepsDirs"].append(os.path.join(libdeps_dir, section[4:])) if config.has_option(section, "board"): data["boards"].append(config.get(section, "board")) data["libExtraDirs"].extend(config.get(section, "lib_extra_dirs", [])) @@ -94,15 +93,15 @@ def _get_project_data(): # skip non existing folders and resolve full path for key in ("envLibdepsDirs", "libExtraDirs"): data[key] = [ - fs.expanduser(d) if d.startswith("~") else realpath(d) + fs.expanduser(d) if d.startswith("~") else os.path.realpath(d) for d in data[key] - if isdir(d) + if os.path.isdir(d) ] return data def _path_to_name(path): - return (sep).join(path.split(sep)[-2:]) + return (os.path.sep).join(path.split(os.path.sep)[-2:]) if not project_dirs: project_dirs = AppRPC.load_state()["storage"]["recentProjects"] @@ -110,6 +109,8 @@ def _path_to_name(path): result = [] pm = PlatformManager() for project_dir in project_dirs: + if not os.path.isdir(project_dir): + continue data = {} boards = [] try: @@ -130,12 +131,12 @@ def _path_to_name(path): { "path": project_dir, "name": _path_to_name(project_dir), - "modified": int(getmtime(project_dir)), + "modified": int(os.path.getmtime(project_dir)), "boards": boards, "description": data.get("description"), "envs": data.get("envs", []), "envLibStorages": [ - {"name": basename(d), "path": d} + {"name": os.path.basename(d), "path": d} for d in data.get("envLibdepsDirs", []) ], "extraLibStorages": [ @@ -153,20 +154,20 @@ def get_projects(self, project_dirs=None): def get_project_examples(): result = [] for manifest in PlatformManager().get_installed(): - examples_dir = join(manifest["__pkg_dir"], "examples") - if not isdir(examples_dir): + examples_dir = os.path.join(manifest["__pkg_dir"], "examples") + if not os.path.isdir(examples_dir): continue items = [] for project_dir, _, __ in os.walk(examples_dir): project_description = None try: - config = ProjectConfig(join(project_dir, "platformio.ini")) + config = ProjectConfig(os.path.join(project_dir, "platformio.ini")) config.validate(silent=True) project_description = config.get("platformio", "description") except exception.PlatformIOProjectException: continue - path_tokens = project_dir.split(sep) + path_tokens = project_dir.split(os.path.sep) items.append( { "name": "/".join( @@ -190,7 +191,7 @@ def get_project_examples(): def init(self, board, framework, project_dir): assert project_dir state = AppRPC.load_state() - if not isdir(project_dir): + if not os.path.isdir(project_dir): os.makedirs(project_dir) args = ["init", "--board", board] if framework: @@ -243,10 +244,10 @@ def _generate_project_main(_, project_dir, framework): with fs.cd(project_dir): config = ProjectConfig() src_dir = config.get_optional_dir("src") - main_path = join(src_dir, "main.cpp") - if isfile(main_path): + main_path = os.path.join(src_dir, "main.cpp") + if os.path.isfile(main_path): return project_dir - if not isdir(src_dir): + if not os.path.isdir(src_dir): os.makedirs(src_dir) fs.write_file_contents(main_path, main_content.strip()) return project_dir @@ -261,10 +262,10 @@ def import_arduino(self, board, use_arduino_libs, arduino_project_dir): is_arduino_project = any( [ - isfile( - join( + os.path.isfile( + os.path.join( arduino_project_dir, - "%s.%s" % (basename(arduino_project_dir), ext), + "%s.%s" % (os.path.basename(arduino_project_dir), ext), ) ) for ext in ("ino", "pde") @@ -276,10 +277,10 @@ def import_arduino(self, board, use_arduino_libs, arduino_project_dir): ) state = AppRPC.load_state() - project_dir = join( + project_dir = os.path.join( state["storage"]["projectsDir"], time.strftime("%y%m%d-%H%M%S-") + board ) - if not isdir(project_dir): + if not os.path.isdir(project_dir): os.makedirs(project_dir) args = ["init", "--board", board] args.extend(["--project-option", "framework = arduino"]) @@ -301,7 +302,7 @@ def _finalize_arduino_import(_, project_dir, arduino_project_dir): with fs.cd(project_dir): config = ProjectConfig() src_dir = config.get_optional_dir("src") - if isdir(src_dir): + if os.path.isdir(src_dir): fs.rmtree(src_dir) shutil.copytree(arduino_project_dir, src_dir) return project_dir @@ -312,9 +313,9 @@ def import_pio(project_dir): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4001, message="Not an PlatformIO project: %s" % project_dir ) - new_project_dir = join( + new_project_dir = os.path.join( AppRPC.load_state()["storage"]["projectsDir"], - time.strftime("%y%m%d-%H%M%S-") + basename(project_dir), + time.strftime("%y%m%d-%H%M%S-") + os.path.basename(project_dir), ) shutil.copytree(project_dir, new_project_dir) From be628051a7e404e420247f1faa48b4be98dc9f94 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 12 Nov 2019 18:35:07 +0200 Subject: [PATCH 012/142] Fix typo --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index d2462c7999..9aaf947a5c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,7 +10,7 @@ PlatformIO Core 4.0 ~~~~~~~~~~~~~~~~~~ * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) -* Warn about about broken library manifest when scanning dependencies (`issue #3268 `_) +* Warn about broken library manifest when scanning dependencies (`issue #3268 `_) * Fixed an issue with the broken latest news for PIO Home * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) From 6e9429dbbfc6057a0913a27c7ab224156913f889 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 12 Nov 2019 18:52:25 +0200 Subject: [PATCH 013/142] Split BuildProgram into ProcessProgramDeps and ProcessProjectDeps --- platformio/builder/tools/platformio.py | 114 ++++++++++++++----------- 1 file changed, 63 insertions(+), 51 deletions(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 818c07ff79..82b81a0f9d 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -44,48 +44,40 @@ def scons_patched_match_splitext(path, suffixes=None): return tokens -def _build_project_deps(env): - project_lib_builder = env.ConfigureProjectLibBuilder() - - # prepend project libs to the beginning of list - env.Prepend(LIBS=project_lib_builder.build()) - # prepend extra linker related options from libs - env.PrependUnique( - **{ - key: project_lib_builder.env.get(key) - for key in ("LIBS", "LIBPATH", "LINKFLAGS") - if project_lib_builder.env.get(key) - } +def GetBuildType(env): + return ( + "debug" + if ( + set(["debug", "sizedata"]) & set(COMMAND_LINE_TARGETS) + or env.GetProjectOption("build_type") == "debug" + ) + else "release" ) - projenv = env.Clone() - # CPPPATH from dependencies - projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH")) - # extra build flags from `platformio.ini` - projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS")) +def BuildProgram(env): + env.ProcessProgramDeps() + env.ProcessProjectDeps() - is_test = "__test" in COMMAND_LINE_TARGETS - if is_test: - projenv.BuildSources( - "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" - ) - if not is_test or env.GetProjectOption("test_build_project_src"): - projenv.BuildSources( - "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") - ) + program = env.Program( + os.path.join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"] + ) + env.Replace(PIOMAINPROG=program) - if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS: - sys.stderr.write( - "Error: Nothing to build. Please put your source code files " - "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR") + AlwaysBuild( + env.Alias( + "checkprogsize", + program, + env.VerboseAction(env.CheckUploadSize, "Checking size $PIOMAINPROG"), ) - env.Exit(1) + ) - Export("projenv") + print("Building in %s mode" % env.GetBuildType()) + return program -def BuildProgram(env): + +def ProcessProgramDeps(env): def _append_pio_macros(): env.AppendUnique( CPPDEFINES=[ @@ -114,11 +106,7 @@ def _append_pio_macros(): # process framework scripts env.BuildFrameworks(env.get("PIOFRAMEWORK")) - is_build_type_debug = ( - set(["debug", "sizedata"]) & set(COMMAND_LINE_TARGETS) - or env.GetProjectOption("build_type") == "debug" - ) - if is_build_type_debug: + if env.GetBuildType() == "debug": env.ConfigureDebugFlags() # remove specified flags @@ -136,25 +124,46 @@ def _append_pio_macros(): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") - # build project with dependencies - _build_project_deps(env) - program = env.Program( - os.path.join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"] +def ProcessProjectDeps(env): + project_lib_builder = env.ConfigureProjectLibBuilder() + + # prepend project libs to the beginning of list + env.Prepend(LIBS=project_lib_builder.build()) + # prepend extra linker related options from libs + env.PrependUnique( + **{ + key: project_lib_builder.env.get(key) + for key in ("LIBS", "LIBPATH", "LINKFLAGS") + if project_lib_builder.env.get(key) + } ) - env.Replace(PIOMAINPROG=program) - AlwaysBuild( - env.Alias( - "checkprogsize", - program, - env.VerboseAction(env.CheckUploadSize, "Checking size $PIOMAINPROG"), + projenv = env.Clone() + + # CPPPATH from dependencies + projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH")) + # extra build flags from `platformio.ini` + projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS")) + + is_test = "__test" in COMMAND_LINE_TARGETS + if is_test: + projenv.BuildSources( + "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" + ) + if not is_test or env.GetProjectOption("test_build_project_src"): + projenv.BuildSources( + "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") ) - ) - print("Building in %s mode" % ("debug" if is_build_type_debug else "release")) + if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS: + sys.stderr.write( + "Error: Nothing to build. Please put your source code files " + "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR") + ) + env.Exit(1) - return program + Export("projenv") def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches @@ -343,7 +352,10 @@ def exists(_): def generate(env): + env.AddMethod(GetBuildType) env.AddMethod(BuildProgram) + env.AddMethod(ProcessProgramDeps) + env.AddMethod(ProcessProjectDeps) env.AddMethod(ParseFlagsExtended) env.AddMethod(ProcessFlags) env.AddMethod(ProcessUnFlags) From 15f142fc7024daca124002d20f8486579c6880cc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 12 Nov 2019 23:48:39 +0200 Subject: [PATCH 014/142] Ensure content cache key does not contain path separators --- platformio/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/app.py b/platformio/app.py index 42dac9c068..34ac498626 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -199,6 +199,7 @@ def _unlock_dbindex(self): return True def get_cache_path(self, key): + assert "/" not in key and "\\" not in key key = str(key) assert len(key) > 3 return join(self.cache_dir, key[-2:], key) From a9f9f4ef045d44863792b5fbc820baa3c9eea886 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 12 Nov 2019 23:52:43 +0200 Subject: [PATCH 015/142] Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries // Resolve #3264 --- platformio/builder/tools/pioplatform.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index a7fbb8b966..f7a085dffd 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -40,16 +40,15 @@ def PioPlatform(env): def BoardConfig(env, board=None): - p = env.PioPlatform() - try: - board = board or env.get("BOARD") - assert board, "BoardConfig: Board is not defined" - with fs.cd(env.subst("$PROJECT_DIR")): - config = p.board_config(board) - except (AssertionError, exception.UnknownBoard) as e: - sys.stderr.write("Error: %s\n" % str(e)) - env.Exit(1) - return config + with fs.cd(env.subst("$PROJECT_DIR")): + try: + p = env.PioPlatform() + board = board or env.get("BOARD") + assert board, "BoardConfig: Board is not defined" + return p.board_config(board) + except (AssertionError, exception.UnknownBoard) as e: + sys.stderr.write("Error: %s\n" % str(e)) + env.Exit(1) def GetFrameworkScript(env, framework): From 53e6cf3e4a99eabc99d149620ae365a547bd1ce7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 13 Nov 2019 15:32:14 +0200 Subject: [PATCH 016/142] Drop support for "project_dirs" argument to Project RPC --- platformio/commands/home/rpc/handlers/project.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 6eb81929a8..8a58d88633 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -73,7 +73,7 @@ def get_config_schema(): return get_config_options_schema() @staticmethod - def _get_projects(project_dirs=None): + def get_projects(): def _get_project_data(): data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} config = ProjectConfig() @@ -103,12 +103,9 @@ def _get_project_data(): def _path_to_name(path): return (os.path.sep).join(path.split(os.path.sep)[-2:]) - if not project_dirs: - project_dirs = AppRPC.load_state()["storage"]["recentProjects"] - result = [] pm = PlatformManager() - for project_dir in project_dirs: + for project_dir in AppRPC.load_state()["storage"]["recentProjects"]: if not os.path.isdir(project_dir): continue data = {} @@ -147,9 +144,6 @@ def _path_to_name(path): ) return result - def get_projects(self, project_dirs=None): - return self._get_projects(project_dirs) - @staticmethod def get_project_examples(): result = [] From b1577d101cbc8e9539a6adf24524cedc67fe1b33 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 13 Nov 2019 15:32:57 +0200 Subject: [PATCH 017/142] Update PIO Home to 3.0.1 --- platformio/commands/home/rpc/handlers/misc.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index 7458500e55..d1851d1397 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -23,7 +23,7 @@ class MiscRPC(object): def load_latest_tweets(self, data_url): - cache_key = app.ContentCache.key_from_args(data_url) + cache_key = app.ContentCache.key_from_args(data_url, "tweets") cache_valid = "7d" with app.ContentCache() as cc: cache_data = cc.get(cache_key) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 1f8142b1e2..59700f22a0 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": "~3.0.0", + "contrib-piohome": "~3.0.1", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.0", "tool-unity": "~1.20403.0", From 9bbaba3d59f31b38cb8880f4abb9f95dbb467514 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 13 Nov 2019 15:35:32 +0200 Subject: [PATCH 018/142] Bump version to 4.1.1a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0e626e181e..da377bbfe9 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1a1") +VERSION = (4, 1, "1a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 053160a6eb60181033da9e858f4e33beb309c312 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 13 Nov 2019 16:47:39 +0200 Subject: [PATCH 019/142] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 71222071ec..e34f09606a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 71222071ecbbf0a3db886a6dcc48d9d73e20b107 +Subproject commit e34f09606a71d3d38ff6900881b82e1bcaef6a29 From 728fd7f5b9e53f3d6e57036726f848bb6967e315 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 13 Nov 2019 16:48:04 +0200 Subject: [PATCH 020/142] Catch exception when can not get a default locale --- platformio/compat.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/compat.py b/platformio/compat.py index ac21016ea3..a8ff28e2b9 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -32,7 +32,10 @@ def get_filesystem_encoding(): def get_locale_encoding(): - return locale.getdefaultlocale()[1] + try: + return locale.getdefaultlocale()[1] + except ValueError: + return None def get_class_attributes(cls): From a077081e46fb77d0e8d640ae26b4f18ef12d5568 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 15 Nov 2019 15:10:43 +0200 Subject: [PATCH 021/142] Init generic C/C++ SCons tools by default --- platformio/builder/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 6286bd574e..8f7586e41a 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -50,10 +50,10 @@ DEFAULT_ENV_OPTIONS = dict( tools=[ "ar", - "gas", - "gcc", - "g++", - "gnulink", + "as", + "cc", + "c++", + "link", "platformio", "pioplatform", "pioproject", From 44a9de6dcb51b748f36d19b0a5c77962c326fabb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 15 Nov 2019 15:11:13 +0200 Subject: [PATCH 022/142] Pass -m and -i flags to VSCode Intellisense analyzer --- platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index 7dd647fcf0..cb05573015 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -56,7 +56,7 @@ % cxx_stds = STD_RE.findall(cxx_flags) % % # pass only architecture specific flags -% cc_m_flags = " ".join([f.strip() for f in cc_flags.split(" ") if f.strip().startswith("-m")]) +% cc_m_flags = " ".join([f.strip() for f in cc_flags.split(" ") if f.strip().startswith(("-m", "-i"))]) % % if cc_stds: "cStandard": "c{{ cc_stds[-1] }}", @@ -68,4 +68,4 @@ } ], "version": 4 -} \ No newline at end of file +} From fbdcbf17c753340dd348d2aef079136bb5e5471c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 15 Nov 2019 15:52:39 +0200 Subject: [PATCH 023/142] Rename Data/Flash to RAM/ROM --- platformio/builder/tools/pioplatform.py | 4 ++-- platformio/builder/tools/pioupload.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index f7a085dffd..9384e64c9e 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -163,9 +163,9 @@ def _get_hardware_data(): if not board_config: return data ram = board_config.get("upload", {}).get("maximum_ram_size") - flash = board_config.get("upload", {}).get("maximum_size") + rom = board_config.get("upload", {}).get("maximum_size") data.append( - "%s RAM, %s Flash" % (fs.format_filesize(ram), fs.format_filesize(flash)) + "%s RAM, %s ROM" % (fs.format_filesize(ram), fs.format_filesize(rom)) ) return data diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 153564343f..c1ba11e5f2 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -251,9 +251,9 @@ def _format_availale_bytes(value, total): print('Advanced Memory Usage is available via "PlatformIO Home > Project Inspect"') if data_max_size and data_size > -1: - print("DATA: %s" % _format_availale_bytes(data_size, data_max_size)) + print("RAM: %s" % _format_availale_bytes(data_size, data_max_size)) if program_size > -1: - print("PROGRAM: %s" % _format_availale_bytes(program_size, program_max_size)) + print("ROM: %s" % _format_availale_bytes(program_size, program_max_size)) if int(ARGUMENTS.get("PIOVERBOSE", 0)): print(output) From 6809da0353aa1986f4d8682489dae5f391341dde Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 15 Nov 2019 16:02:15 +0200 Subject: [PATCH 024/142] Replace os.path.abspath by realpath --- platformio/app.py | 4 ++-- platformio/builder/tools/pioide.py | 24 ++++++++++++------------ platformio/commands/check/defect.py | 6 +++--- platformio/commands/check/tools/base.py | 2 +- platformio/commands/ci.py | 6 +++--- platformio/commands/debug/client.py | 4 ++-- platformio/fs.py | 4 ++-- platformio/ide/projectgenerator.py | 4 ++-- platformio/lockfile.py | 15 +++++++-------- platformio/managers/package.py | 4 ++-- 10 files changed, 36 insertions(+), 37 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 34ac498626..f53b17d0d8 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -17,7 +17,7 @@ import os import uuid from os import environ, getenv, listdir, remove -from os.path import abspath, dirname, isdir, isfile, join +from os.path import dirname, isdir, isfile, join, realpath from time import time import requests @@ -34,7 +34,7 @@ def projects_dir_validate(projects_dir): assert isdir(projects_dir) - return abspath(projects_dir) + return realpath(projects_dir) DEFAULT_SETTINGS = { diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 26544cfeea..253ce61de1 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -14,9 +14,8 @@ from __future__ import absolute_import +import os from glob import glob -from os import environ -from os.path import abspath, isfile, join from SCons.Defaults import processDefines # pylint: disable=import-error @@ -42,10 +41,10 @@ def _dump_includes(env): continue toolchain_dir = glob_escape(p.get_package_dir(name)) toolchain_incglobs = [ - join(toolchain_dir, "*", "include*"), - join(toolchain_dir, "*", "include", "c++", "*"), - join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"), - join(toolchain_dir, "lib", "gcc", "*", "*", "include*"), + os.path.join(toolchain_dir, "*", "include*"), + os.path.join(toolchain_dir, "*", "include", "c++", "*"), + os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"), + os.path.join(toolchain_dir, "lib", "gcc", "*", "*", "include*"), ] for g in toolchain_incglobs: includes.extend(glob(g)) @@ -59,8 +58,9 @@ def _dump_includes(env): # remove duplicates result = [] for item in includes: + item = os.path.realpath(item) if item not in result: - result.append(abspath(item)) + result.append(item) return result @@ -68,7 +68,7 @@ def _dump_includes(env): def _get_gcc_defines(env): items = [] try: - sysenv = environ.copy() + sysenv = os.environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) result = exec_command( "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True @@ -119,7 +119,7 @@ def _dump_defines(env): def _get_svd_path(env): svd_path = env.GetProjectOption("debug_svd_path") if svd_path: - return abspath(svd_path) + return os.path.realpath(svd_path) if "BOARD" not in env: return None @@ -129,12 +129,12 @@ def _get_svd_path(env): except (AssertionError, KeyError): return None # custom path to SVD file - if isfile(svd_path): + if os.path.isfile(svd_path): return svd_path # default file from ./platform/misc/svd folder p = env.PioPlatform() - if isfile(join(p.get_dir(), "misc", "svd", svd_path)): - return abspath(join(p.get_dir(), "misc", "svd", svd_path)) + if os.path.isfile(os.path.join(p.get_dir(), "misc", "svd", svd_path)): + return os.path.realpath(os.path.join(p.get_dir(), "misc", "svd", svd_path)) return None diff --git a/platformio/commands/check/defect.py b/platformio/commands/check/defect.py index e864356ea8..32b7dc2cd9 100644 --- a/platformio/commands/check/defect.py +++ b/platformio/commands/check/defect.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import abspath, relpath +import os import click @@ -52,7 +52,7 @@ def __init__( self.id = id self.file = file if file.startswith(get_project_dir()): - self.file = relpath(file, get_project_dir()) + self.file = os.path.relpath(file, get_project_dir()) def __repr__(self): defect_color = None @@ -86,7 +86,7 @@ def as_dict(self): "severity": self.SEVERITY_LABELS[self.severity], "category": self.category, "message": self.message, - "file": abspath(self.file), + "file": os.path.realpath(self.file), "line": self.line, "column": self.column, "callstack": self.callstack, diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index 59951288d5..4c381a0575 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -132,7 +132,7 @@ def get_project_target_files(self): def _add_file(path): if not path.endswith(allowed_extensions): return - result.append(os.path.abspath(path)) + result.append(os.path.realpath(path)) for pattern in self.options["patterns"]: for item in glob.glob(pattern): diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 4cdf227cfa..7c57f46559 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -14,7 +14,7 @@ from glob import glob from os import getenv, makedirs, remove -from os.path import abspath, basename, isdir, isfile, join +from os.path import basename, isdir, isfile, join, realpath from shutil import copyfile, copytree from tempfile import mkdtemp @@ -35,7 +35,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument for i, p in enumerate(value): if p.startswith("~"): value[i] = fs.expanduser(p) - value[i] = abspath(value[i]) + value[i] = realpath(value[i]) if not glob(value[i]): invalid_path = p break @@ -158,7 +158,7 @@ def _exclude_contents(dst_dir, patterns): for p in patterns: contents += glob(join(glob_escape(dst_dir), p)) for path in contents: - path = abspath(path) + path = realpath(path) if isdir(path): fs.rmtree(path) elif isfile(path): diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 72a12a84ba..5650f3029b 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -18,7 +18,7 @@ import signal import time from hashlib import sha1 -from os.path import abspath, basename, dirname, isdir, join, splitext +from os.path import basename, dirname, isdir, join, realpath, splitext from tempfile import mkdtemp from twisted.internet import protocol # pylint: disable=import-error @@ -108,7 +108,7 @@ def spawn(self, gdb_path, prog_path): def _get_data_dir(gdb_path): if "msp430" in gdb_path: return None - gdb_data_dir = abspath(join(dirname(gdb_path), "..", "share", "gdb")) + gdb_data_dir = realpath(join(dirname(gdb_path), "..", "share", "gdb")) return gdb_data_dir if isdir(gdb_data_dir) else None def generate_pioinit(self, dst_dir, patterns): diff --git a/platformio/fs.py b/platformio/fs.py index 18ad38b2c8..ca561b10ac 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -40,7 +40,7 @@ def __exit__(self, etype, value, traceback): def get_source_dir(): - curpath = os.path.abspath(__file__) + curpath = os.path.realpath(__file__) if not os.path.isfile(curpath): for p in sys.path: if os.path.isfile(os.path.join(p, __file__)): @@ -117,7 +117,7 @@ def _rules_to_set(rules_path): if not any(os.path.isfile(p) for p in installed_rules): raise exception.MissedUdevRules - origin_path = os.path.abspath( + origin_path = os.path.realpath( os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules") ) if not os.path.isfile(origin_path): diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index d4e13e5aea..6e775099bf 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -15,7 +15,7 @@ import io import os import sys -from os.path import abspath, basename, isdir, isfile, join, relpath +from os.path import basename, isdir, isfile, join, realpath, relpath import bottle @@ -64,7 +64,7 @@ def _load_tplvars(self): "project_name": basename(self.project_dir), "project_dir": self.project_dir, "env_name": self.env_name, - "user_home_dir": abspath(fs.expanduser("~")), + "user_home_dir": realpath(fs.expanduser("~")), "platformio_path": sys.argv[0] if isfile(sys.argv[0]) else where_is_program("platformio"), diff --git a/platformio/lockfile.py b/platformio/lockfile.py index 8c7a46581f..44d2e4cf2f 100644 --- a/platformio/lockfile.py +++ b/platformio/lockfile.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import remove -from os.path import abspath, exists, getmtime +import os from time import sleep, time from platformio import exception @@ -45,15 +44,15 @@ class LockFile(object): def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY): self.timeout = timeout self.delay = delay - self._lock_path = abspath(path) + ".lock" + self._lock_path = os.path.realpath(path) + ".lock" self._fp = None def _lock(self): - if not LOCKFILE_CURRENT_INTERFACE and exists(self._lock_path): + if not LOCKFILE_CURRENT_INTERFACE and os.path.exists(self._lock_path): # remove stale lock - if time() - getmtime(self._lock_path) > 10: + if time() - os.path.getmtime(self._lock_path) > 10: try: - remove(self._lock_path) + os.remove(self._lock_path) except: # pylint: disable=bare-except pass else: @@ -93,9 +92,9 @@ def acquire(self): def release(self): self._unlock() - if exists(self._lock_path): + if os.path.exists(self._lock_path): try: - remove(self._lock_path) + os.remove(self._lock_path) except: # pylint: disable=bare-except pass diff --git a/platformio/managers/package.py b/platformio/managers/package.py index de87904b83..10dadd3cd3 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -17,7 +17,7 @@ import os import re import shutil -from os.path import abspath, basename, getsize, isdir, isfile, islink, join +from os.path import basename, getsize, isdir, isfile, islink, join, realpath from tempfile import mkdtemp import click @@ -423,7 +423,7 @@ def get_package_dir(self, name, requirements=None, url=None): def get_package_by_dir(self, pkg_dir): for manifest in self.get_installed(): - if manifest["__pkg_dir"] == abspath(pkg_dir): + if manifest["__pkg_dir"] == realpath(pkg_dir): return manifest return None From c7202154dea76d41d88a7cb09ac150dbba8fcf69 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 15 Nov 2019 20:05:09 +0200 Subject: [PATCH 025/142] Project Manager and Project Configuration UI for "platformio.ini" --- HISTORY.rst | 8 +++++++- platformio/managers/core.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9aaf947a5c..864ef433b0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,11 @@ PlatformIO Core 4.0 4.1.1 (2019-??-??) ~~~~~~~~~~~~~~~~~~ +* `PlatformIO Home 3.0 `__: + + - Project Manager + - Project Configuration UI for `"platformio.ini" `__ + * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Warn about broken library manifest when scanning dependencies (`issue #3268 `_) * Fixed an issue with the broken latest news for PIO Home @@ -26,8 +31,9 @@ PlatformIO Core 4.0 - Unused variables or functions - Out of scope memory usage. -* `PlatformIO Home 3.0 `__ and Project Inspection +* `PlatformIO Home 3.0 `__: + - Project Inspection - Static Code Analysis - Firmware File Explorer - Firmware Memory Inspection diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 59700f22a0..3bd2e56c4e 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": "~3.0.1", + "contrib-piohome": ">=3.1.0-beta.1,<3.2.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.0", "tool-unity": "~1.20403.0", From a7b91872348e7f7464497a97658bb5b4ce0a3fda Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 15 Nov 2019 20:05:52 +0200 Subject: [PATCH 026/142] Bump version to 4.1.1b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index da377bbfe9..22b21671f8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1a2") +VERSION = (4, 1, "1b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From d1b46c838ee984dd242a819d2c4c4ee5c2b10250 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 16 Nov 2019 14:09:16 +0200 Subject: [PATCH 027/142] Revert back "Flash" instead of "ROM" --- platformio/builder/tools/pioplatform.py | 4 ++-- platformio/builder/tools/pioupload.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 9384e64c9e..f7a085dffd 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -163,9 +163,9 @@ def _get_hardware_data(): if not board_config: return data ram = board_config.get("upload", {}).get("maximum_ram_size") - rom = board_config.get("upload", {}).get("maximum_size") + flash = board_config.get("upload", {}).get("maximum_size") data.append( - "%s RAM, %s ROM" % (fs.format_filesize(ram), fs.format_filesize(rom)) + "%s RAM, %s Flash" % (fs.format_filesize(ram), fs.format_filesize(flash)) ) return data diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index c1ba11e5f2..5e1117b501 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -251,9 +251,9 @@ def _format_availale_bytes(value, total): print('Advanced Memory Usage is available via "PlatformIO Home > Project Inspect"') if data_max_size and data_size > -1: - print("RAM: %s" % _format_availale_bytes(data_size, data_max_size)) + print("RAM: %s" % _format_availale_bytes(data_size, data_max_size)) if program_size > -1: - print("ROM: %s" % _format_availale_bytes(program_size, program_max_size)) + print("Flash: %s" % _format_availale_bytes(program_size, program_max_size)) if int(ARGUMENTS.get("PIOVERBOSE", 0)): print(output) From e9f9871c1e7b42573f2865876070988379ba4980 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 16 Nov 2019 17:25:27 +0200 Subject: [PATCH 028/142] Show to user the last exception when can't install a package --- platformio/managers/package.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 10dadd3cd3..32405d77f5 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -439,6 +439,7 @@ def _install_from_piorepo(self, name, requirements): pkg_dir = None pkgdata = None versions = None + last_exc = None for versions in PackageRepoIterator(name, self.repositories): pkgdata = self.max_satisfying_repo_version(versions, requirements) if not pkgdata: @@ -449,12 +450,15 @@ def _install_from_piorepo(self, name, requirements): ) break except Exception as e: # pylint: disable=broad-except + last_exc = e click.secho("Warning! Package Mirror: %s" % e, fg="yellow") click.secho("Looking for another mirror...", fg="yellow") if versions is None: util.internet_on(raise_exception=True) - raise exception.UnknownPackage(name) + raise exception.UnknownPackage( + name + (". Error -> %s" % last_exc if last_exc else "") + ) if not pkgdata: raise exception.UndefinedPackageVersion( requirements or "latest", util.get_systype() From 0029c7fe09144d9a3ddace39293c80c1e969cc44 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 16 Nov 2019 17:45:57 +0200 Subject: [PATCH 029/142] Advanced Serial Monitor with UI --- docs | 2 +- platformio/commands/device.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs b/docs index e34f09606a..d297721097 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e34f09606a71d3d38ff6900881b82e1bcaef6a29 +Subproject commit d2977210978165793d4e98c7a94b4ef045f89eae diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 67c7f05ac9..6a547ae880 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -172,6 +172,10 @@ def device_list( # pylint: disable=too-many-branches help="Load configuration from `platformio.ini` and specified environment", ) def device_monitor(**kwargs): # pylint: disable=too-many-branches + click.echo( + "Looking for advanced Serial Monitor with UI? " + "Check http://bit.ly/pio-advanced-monitor" + ) project_options = {} try: with fs.cd(kwargs["project_dir"]): From c1394b290dcd9257334c021856cf5e6c336c61fb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 16 Nov 2019 22:43:25 +0200 Subject: [PATCH 030/142] Fix issue with unknown dev/platform when symlink is used --- platformio/managers/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 32405d77f5..b253ea0ee2 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -364,7 +364,7 @@ def load_manifest(self, pkg_dir): # pylint: disable=too-many-branches if "version" not in manifest: manifest["version"] = "0.0.0" - manifest["__pkg_dir"] = pkg_dir + manifest["__pkg_dir"] = realpath(pkg_dir) self.cache_set(cache_key, manifest) return manifest From 464e1a509fc4a29303b53e2c403627cc4f54cbe2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 16 Nov 2019 22:57:56 +0200 Subject: [PATCH 031/142] Add PIO Home "os.get_file_mtime" RPC method --- platformio/commands/home/rpc/handlers/os.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index c019fbc7d3..2d7ce609c6 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -19,7 +19,6 @@ import os import shutil from functools import cmp_to_key -from os.path import isdir, isfile, join import click from twisted.internet import defer # pylint: disable=import-error @@ -67,7 +66,7 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None): def request_content(self, uri, data=None, headers=None, cache_valid=None): if uri.startswith("http"): return self.fetch_content(uri, data, headers, cache_valid) - if not isfile(uri): + if not os.path.isfile(uri): return None with codecs.open(uri, encoding="utf-8") as fp: return fp.read() @@ -88,16 +87,20 @@ def open_file(path): @staticmethod def is_file(path): - return isfile(path) + return os.path.isfile(path) @staticmethod def is_dir(path): - return isdir(path) + return os.path.isdir(path) @staticmethod def make_dirs(path): return os.makedirs(path) + @staticmethod + def get_file_mtime(path): + return os.path.getmtime(path) + @staticmethod def rename(src, dst): return os.rename(src, dst) @@ -112,7 +115,7 @@ def glob(pathnames, root=None): pathnames = [pathnames] result = set() for pathname in pathnames: - result |= set(glob.glob(join(root, pathname) if root else pathname)) + result |= set(glob.glob(os.path.join(root, pathname) if root else pathname)) return list(result) @staticmethod @@ -131,13 +134,13 @@ def _cmp(x, y): items = [] if path.startswith("~"): path = fs.expanduser(path) - if not isdir(path): + if not os.path.isdir(path): return items for item in os.listdir(path): try: - item_is_dir = isdir(join(path, item)) + item_is_dir = os.path.isdir(os.path.join(path, item)) if item_is_dir: - os.listdir(join(path, item)) + os.listdir(os.path.join(path, item)) items.append((item, item_is_dir)) except OSError: pass From 5b1c6daa2a7f9681566d86a71e7dbc77d03a5e65 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 16 Nov 2019 23:09:18 +0200 Subject: [PATCH 032/142] Fix a space in config header --- platformio/project/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 27b116c19c..f4b5d20646 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -29,7 +29,7 @@ except ImportError: import configparser as ConfigParser -CONFIG_HEADER = """;PlatformIO Project Configuration File +CONFIG_HEADER = """; PlatformIO Project Configuration File ; ; Build options: build flags, source filter ; Upload options: custom upload port, speed and extra flags From 7eab5d567e745a6f6fbf204083dfdb2c381dce9f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 17 Nov 2019 00:35:56 +0200 Subject: [PATCH 033/142] Fix CLion generator when one env is used --- platformio/ide/tpls/clion/CMakeLists.txt.tpl | 3 ++- platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 302fb6e19f..83d4745f44 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -6,10 +6,11 @@ # The `CMakeListsUser.txt` will not be overwritten by PlatformIO. cmake_minimum_required(VERSION 3.2) -project("{{project_name}}") include(CMakeListsPrivate.txt) +project("{{project_name}}") + if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/CMakeListsUser.txt) include(CMakeListsUser.txt) endif() diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index c7c7c1635f..e4bc39e23e 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -25,7 +25,7 @@ % envs = config.envs() % if len(envs) > 1: -set(CMAKE_CONFIGURATION_TYPES "{{ ";".join(envs) }}" CACHE STRING "" FORCE) +set(CMAKE_CONFIGURATION_TYPES "{{ ";".join(envs) }};" CACHE STRING "" FORCE) % else: set(CMAKE_CONFIGURATION_TYPES "{{ env_name }}" CACHE STRING "" FORCE) % end From f19c7dc575e529bc5c47fa657735c1c7f2df3751 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Nov 2019 17:59:19 +0200 Subject: [PATCH 034/142] Support for Atmel megaAVR --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index d297721097..83abdaf224 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d2977210978165793d4e98c7a94b4ef045f89eae +Subproject commit 83abdaf22481779f99d40d5e1aacd2bdf03f07b7 diff --git a/examples b/examples index 9070288cff..f453ee540e 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 9070288cff31e353ee307b1b108662ff8c06a9b2 +Subproject commit f453ee540e35ea3222e076b597ab866fb7fd15a6 From 88ead0aed100f4c378987007652f2ad10dec5090 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 19 Nov 2019 18:55:21 +0200 Subject: [PATCH 035/142] Catch all debug errors when killing debug server --- platformio/commands/debug/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index cdd7fa32c2..b0ddb9083a 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -15,7 +15,6 @@ import os from os.path import isdir, isfile, join -from twisted.internet import error # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error from platformio import exception, fs, util @@ -134,5 +133,5 @@ def terminate(self): return try: self._transport.signalProcess("KILL") - except (OSError, error.ProcessExitedAlready): + except: # pylint: disable=bare-except pass From 18a9866a02d4bc025a76393f3f4526759eed071f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 19 Nov 2019 19:56:10 +0200 Subject: [PATCH 036/142] An example with custom debug flags in advanced scripting --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 83abdaf224..bda7eb6840 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 83abdaf22481779f99d40d5e1aacd2bdf03f07b7 +Subproject commit bda7eb684061cb933f0bd94ccd65887d7e70a602 From 895aa0cb34ee98131ce53f861ac721f56f838fd3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 19 Nov 2019 20:01:50 +0200 Subject: [PATCH 037/142] Update PlatformIO slogan --- README.rst | 3 ++- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 1267200854..56d553283a 100644 --- a/README.rst +++ b/README.rst @@ -34,8 +34,9 @@ PlatformIO .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png :target: https://platformio.org?utm_source=github&utm_medium=core -`PlatformIO `_ an open source ecosystem for embedded development +`PlatformIO `_ a new generation ecosystem for embedded development +* Open source, maximum permissive Apache 2.0 license * Cross-platform IDE and Unified Debugger * Static Code Analyzer and Remote Unit Testing * Multi-platform and Multi-architecture Build System diff --git a/docs b/docs index bda7eb6840..d8e26d9b97 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit bda7eb684061cb933f0bd94ccd65887d7e70a602 +Subproject commit d8e26d9b97d75e1071c518b75f13633b6612b57c diff --git a/platformio/__init__.py b/platformio/__init__.py index 22b21671f8..71f070fb22 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -17,7 +17,7 @@ __title__ = "platformio" __description__ = ( - "An open source ecosystem for embedded development. " + "A new generation ecosystem for embedded development. " "Cross-platform IDE and Unified Debugger. " "Static Code Analyzer and Remote Unit Testing. " "Multi-platform and Multi-architecture Build System. " From b37814976c7946b25f3161727af3462351ea44d3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Nov 2019 23:31:35 +0200 Subject: [PATCH 038/142] Fix project generator for CLion --- platformio/ide/tpls/clion/CMakeLists.txt.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 83d4745f44..528483785e 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -7,10 +7,10 @@ cmake_minimum_required(VERSION 3.2) -include(CMakeListsPrivate.txt) - project("{{project_name}}") +include(CMakeListsPrivate.txt) + if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/CMakeListsUser.txt) include(CMakeListsUser.txt) endif() From 2b9d8dba5f421ad854574e9d1b7004578dd78346 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Nov 2019 23:32:19 +0200 Subject: [PATCH 039/142] Bump version to 4.1.1b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 71f070fb22..03828749d1 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1b1") +VERSION = (4, 1, "1b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 8675d3fa4684830c91f85e75ff257c28b79a01d1 Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 21 Nov 2019 00:39:05 +0200 Subject: [PATCH 040/142] Use proper CMake variables for setting compilation flags --- platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index e4bc39e23e..daf5f2bca0 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -37,8 +37,8 @@ set(SVD_PATH "{{ _normalize_path(svd_path) }}") SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}") SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}") -SET(CMAKE_CXX_FLAGS_DISTRIBUTION "{{cxx_flags}}") -SET(CMAKE_C_FLAGS_DISTRIBUTION "{{cc_flags}}") +SET(CMAKE_CXX_FLAGS "{{cxx_flags}}") +SET(CMAKE_C_FLAGS "{{cc_flags}}") % STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)") % cc_stds = STD_RE.findall(cc_flags) From 9cf777b4e502714a60ac6d6d4defa72532cc10e4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 21 Nov 2019 14:19:32 +0200 Subject: [PATCH 041/142] Fixed an issue with "start-group/end-group" linker flags on Native development platform // Resolve #3282 --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 864ef433b0..3fded02680 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ PlatformIO Core 4.0 * Warn about broken library manifest when scanning dependencies (`issue #3268 `_) * Fixed an issue with the broken latest news for PIO Home * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) +* Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 82b81a0f9d..5d8222f63e 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -59,6 +59,15 @@ def BuildProgram(env): env.ProcessProgramDeps() env.ProcessProjectDeps() + # append into the beginning a main LD script + if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]): + env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")]) + + # enable "cyclic reference" for linker + if env.get("LIBS") and env.GetCompilerType() == "gcc": + env.Prepend(_LIBFLAGS="-Wl,--start-group ") + env.Append(_LIBFLAGS=" -Wl,--end-group") + program = env.Program( os.path.join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"] ) @@ -115,15 +124,6 @@ def _append_pio_macros(): if "__test" in COMMAND_LINE_TARGETS: env.ConfigureTestTarget() - # append into the beginning a main LD script - if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]): - env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")]) - - # enable "cyclic reference" for linker - if env.get("LIBS") and env.GetCompilerType() == "gcc": - env.Prepend(_LIBFLAGS="-Wl,--start-group ") - env.Append(_LIBFLAGS=" -Wl,--end-group") - def ProcessProjectDeps(env): project_lib_builder = env.ConfigureProjectLibBuilder() From 47099190f463a78bb344c2c0616d871157292df4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Nov 2019 20:49:09 +0200 Subject: [PATCH 042/142] Bump PIO Home version to 3.1.0-beta.2 --- platformio/managers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 3bd2e56c4e..ea1baa1af6 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.1.0-beta.1,<3.2.0", + "contrib-piohome": ">=3.1.0-beta.2,<3.2.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.0", "tool-unity": "~1.20403.0", From f87322591d608fbc6836b64b3d7bcee2fb758146 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Nov 2019 20:49:44 +0200 Subject: [PATCH 043/142] Bump version to 4.1.1b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 03828749d1..2f7e7fc8f2 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1b2") +VERSION = (4, 1, "1b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 17b4f0b4dd9a0f9f15b2142277455f383d752908 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Nov 2019 13:56:33 +0200 Subject: [PATCH 044/142] Docs: Sync Atmel AVR dev/platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index d8e26d9b97..8ca74a35b6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d8e26d9b97d75e1071c518b75f13633b6612b57c +Subproject commit 8ca74a35b686fc681a9affdf8eb72eeba8c352a4 From fdd73552ead50a537bf9cfacd1e5db095f32a718 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Nov 2019 14:44:29 +0200 Subject: [PATCH 045/142] Docs for Zephyr // Issue #1613 --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 8ca74a35b6..a8416e56cf 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8ca74a35b686fc681a9affdf8eb72eeba8c352a4 +Subproject commit a8416e56cf05e2157d97cc781e7a16789e13b21d diff --git a/examples b/examples index f453ee540e..f5835d10b3 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit f453ee540e35ea3222e076b597ab866fb7fd15a6 +Subproject commit f5835d10b38e9928e3378585caaf3f9de4eeb533 From 2fbe33bca0965d9b5c76215683d4dc0b529d903d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Nov 2019 14:44:54 +0200 Subject: [PATCH 046/142] Move "Board" Options to "Platform --- docs | 2 +- platformio/project/options.py | 10 +++++----- tests/test_projectconf.py | 23 +++++++++++++++++++---- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/docs b/docs index a8416e56cf..02d20c83d2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a8416e56cf05e2157d97cc781e7a16789e13b21d +Subproject commit 02d20c83d2b43268eb5a8a287f91c0eb9f0d84f1 diff --git a/platformio/project/options.py b/platformio/project/options.py index 3788ba550b..50a264faf3 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -293,34 +293,34 @@ def ConfigEnvOption(*args, **kwargs): ), # Board ConfigEnvOption( - group="board", + group="platform", name="board", description="A board ID", buildenvvar="BOARD", ), ConfigEnvOption( - group="board", + group="platform", name="board_build.mcu", description="A custom board MCU", oldnames=["board_mcu"], buildenvvar="BOARD_MCU", ), ConfigEnvOption( - group="board", + group="platform", name="board_build.f_cpu", description="A custom MCU frequency", oldnames=["board_f_cpu"], buildenvvar="BOARD_F_CPU", ), ConfigEnvOption( - group="board", + group="platform", name="board_build.f_flash", description="A custom flash frequency", oldnames=["board_f_flash"], buildenvvar="BOARD_F_FLASH", ), ConfigEnvOption( - group="board", + group="platform", name="board_build.flash_mode", description="A custom flash mode", oldnames=["board_flash_mode"], diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index cfede890f4..3719348267 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -65,7 +65,11 @@ EXTRA_ENVS_CONFIG = """ [env:extra_1] -build_flags = ${custom.lib_flags} ${custom.debug_flags} +build_flags = + -fdata-sections + -Wl,--gc-sections + ${custom.lib_flags} + ${custom.debug_flags} lib_install = 574 [env:extra_2] @@ -226,7 +230,10 @@ def test_getraw_value(config): # known assert config.getraw("env:base", "targets") == "" assert config.getraw("env:extra_1", "lib_deps") == "574" - assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1" + assert ( + config.getraw("env:extra_1", "build_flags") + == "\n-fdata-sections\n-Wl,--gc-sections\n-lc -lm\n-D DEBUG=1" + ) # extended assert config.getraw("env:test_extends", "lib_ldf_mode") == "chain+" @@ -236,7 +243,12 @@ def test_getraw_value(config): def test_get_value(config): assert config.get("custom", "debug_flags") == "-D DEBUG=1" - assert config.get("env:extra_1", "build_flags") == ["-lc -lm -D DEBUG=1"] + assert config.get("env:extra_1", "build_flags") == [ + "-fdata-sections", + "-Wl,--gc-sections", + "-lc -lm", + "-D DEBUG=1", + ] assert config.get("env:extra_2", "build_flags") == ["-Og"] assert config.get("env:extra_2", "monitor_speed") == 9600 assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] @@ -259,7 +271,10 @@ def test_items(config): ("lib_ignore", ["LibIgnoreCustom"]), ] assert config.items(env="extra_1") == [ - ("build_flags", ["-lc -lm -D DEBUG=1"]), + ( + "build_flags", + ["-fdata-sections", "-Wl,--gc-sections", "-lc -lm", "-D DEBUG=1"], + ), ("lib_deps", ["574"]), ("monitor_speed", 9600), ("custom_monitor_speed", "115200"), From 0615ff6dd8cd8e1418ede657372f5dbf8678fd4e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Nov 2019 17:40:09 +0200 Subject: [PATCH 047/142] Docs: Change "Global/Local scope" configuration to "Common/Environment" --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 02d20c83d2..ad87be4bdd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 02d20c83d2b43268eb5a8a287f91c0eb9f0d84f1 +Subproject commit ad87be4bdd04c134d981ed0e302f6868f4694308 From 742f60b48d5b6ca72351bf7d76b1598a76b5f47a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Nov 2019 18:00:33 +0200 Subject: [PATCH 048/142] Bump version to 4.1.1b4 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 2f7e7fc8f2..21816d0f34 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1b3") +VERSION = (4, 1, "1b4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index ea1baa1af6..7996d02a0b 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.1.0-beta.2,<3.2.0", + "contrib-piohome": ">=3.1.0-beta.3,<3.2.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.0", "tool-unity": "~1.20403.0", From 47469e87594d8067ce9728cc0e40cf6902e5ad0b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Nov 2019 18:08:32 +0200 Subject: [PATCH 049/142] Fix issue when None value is passed to config.set --- platformio/project/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/project/config.py b/platformio/project/config.py index f4b5d20646..7fdbcdf8ad 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -228,6 +228,8 @@ def items(self, section=None, env=None, as_dict=False): return [(option, self.get(section, option)) for option in self.options(section)] def set(self, section, option, value): + if value is None: + value = "" if isinstance(value, (list, tuple)): value = "\n".join(value) elif isinstance(value, bool): From 49ceadc6ad218af16a9c423e53751a93125b2c53 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 Nov 2019 16:15:54 +0200 Subject: [PATCH 050/142] Move exceptions to their components --- HISTORY.rst | 1 - platformio/commands/debug/client.py | 32 ++--- platformio/commands/debug/command.py | 8 +- platformio/commands/debug/exception.py | 33 +++++ platformio/commands/debug/helpers.py | 5 +- platformio/commands/debug/server.py | 5 +- platformio/commands/device.py | 3 +- .../commands/home/rpc/handlers/project.py | 5 +- platformio/commands/remote.py | 3 +- platformio/commands/run/processor.py | 5 +- platformio/exception.py | 59 +-------- platformio/maintenance.py | 4 +- platformio/managers/core.py | 2 +- platformio/managers/package.py | 25 +--- platformio/managers/platform.py | 11 +- platformio/project/config.py | 13 +- platformio/project/exception.py | 53 ++++++++ platformio/telemetry.py | 123 ++++++++++-------- 18 files changed, 213 insertions(+), 177 deletions(-) create mode 100644 platformio/commands/debug/exception.py create mode 100644 platformio/project/exception.py diff --git a/HISTORY.rst b/HISTORY.rst index 3fded02680..290a04f4ba 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,7 +16,6 @@ PlatformIO Core 4.0 * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Warn about broken library manifest when scanning dependencies (`issue #3268 `_) -* Fixed an issue with the broken latest news for PIO Home * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) * Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 5650f3029b..2c6d972425 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import os import re import signal @@ -26,13 +25,13 @@ from twisted.internet import stdio # pylint: disable=import-error from twisted.internet import task # pylint: disable=import-error -from platformio import app, exception, fs, proc, util +from platformio import app, fs, proc, telemetry, util from platformio.commands.debug import helpers, initcfgs +from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer from platformio.compat import hashlib_encode_data, is_bytes from platformio.project.helpers import get_project_cache_dir -from platformio.telemetry import MeasurementProtocol LOG_FILE = None @@ -58,6 +57,7 @@ def __init__(self, project_dir, args, debug_options, env_options): self._target_is_run = False self._last_server_activity = 0 self._auto_continue_timer = None + self._errors_buffer = b"" def spawn(self, gdb_path, prog_path): session_hash = gdb_path + prog_path @@ -94,7 +94,7 @@ def spawn(self, gdb_path, prog_path): ] args.extend(self.args) if not gdb_path: - raise exception.DebugInvalidOptions("GDB client is not configured") + raise DebugInvalidOptionsError("GDB client is not configured") gdb_data_dir = self._get_data_dir(gdb_path) if gdb_data_dir: args.extend(["--data-directory", gdb_data_dir]) @@ -215,6 +215,9 @@ def outReceived(self, data): self._handle_error(data) # go to init break automatically if self.INIT_COMPLETED_BANNER.encode() in data: + telemetry.send_event( + "Debug", "Started", telemetry.encode_run_environment(self.env_options) + ) self._auto_continue_timer = task.LoopingCall(self._auto_exec_continue) self._auto_continue_timer.start(0.1) @@ -250,20 +253,19 @@ def _auto_exec_continue(self): self._target_is_run = True def _handle_error(self, data): + self._errors_buffer += data if self.PIO_SRC_NAME.encode() not in data or b"Error in sourced" not in data: return - configuration = {"debug": self.debug_options, "env": self.env_options} - exd = re.sub(r'\\(?!")', "/", json.dumps(configuration)) - exd = re.sub( - r'"(?:[a-z]\:)?((/[^"/]+)+)"', - lambda m: '"%s"' % join(*m.group(1).split("/")[-2:]), - exd, - re.I | re.M, + + last_erros = self._errors_buffer.decode() + last_erros = " ".join(reversed(last_erros.split("\n"))) + last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M) + + err = "%s -> %s" % ( + telemetry.encode_run_environment(self.env_options), + last_erros, ) - mp = MeasurementProtocol() - mp["exd"] = "DebugGDBPioInitError: %s" % exd - mp["exf"] = 1 - mp.send("exception") + telemetry.send_exception("DebugInitError: %s" % err, is_fatal=True) self.transport.loseConnection() def _kill_previous_session(self): diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index ab27306319..c3c29dd166 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -23,8 +23,10 @@ from platformio import app, exception, fs, proc, util from platformio.commands.debug import helpers +from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.managers.core import inject_contrib_pysite from platformio.project.config import ProjectConfig +from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project, load_project_ide_data @@ -70,7 +72,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro env_name = environment or helpers.get_default_debug_env(config) env_options = config.items(env=env_name, as_dict=True) if not set(env_options.keys()) >= set(["platform", "board"]): - raise exception.ProjectEnvsNotAvailable() + raise ProjectEnvsNotAvailableError() debug_options = helpers.validate_debug_options(ctx, env_options) assert debug_options @@ -79,7 +81,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro configuration = load_project_ide_data(project_dir, env_name) if not configuration: - raise exception.DebugInvalidOptions("Could not load debug configuration") + raise DebugInvalidOptionsError("Could not load debug configuration") if "--version" in __unprocessed: result = proc.exec_command([configuration["gdb_path"], "--version"]) @@ -140,7 +142,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro helpers.is_prog_obsolete(configuration["prog_path"]) if not isfile(configuration["prog_path"]): - raise exception.DebugInvalidOptions("Program/firmware is missed") + raise DebugInvalidOptionsError("Program/firmware is missed") # run debugging client inject_contrib_pysite() diff --git a/platformio/commands/debug/exception.py b/platformio/commands/debug/exception.py new file mode 100644 index 0000000000..a1269b2fd4 --- /dev/null +++ b/platformio/commands/debug/exception.py @@ -0,0 +1,33 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException, UserSideException + + +class DebugError(PlatformioException): + pass + + +class DebugSupportError(DebugError, UserSideException): + + MESSAGE = ( + "Currently, PlatformIO does not support debugging for `{0}`.\n" + "Please request support at https://github.com/platformio/" + "platformio-core/issues \nor visit -> https://docs.platformio.org" + "/page/plus/debugging.html" + ) + + +class DebugInvalidOptionsError(DebugError, UserSideException): + pass diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index e8e6c525c1..5edba11abe 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -22,6 +22,7 @@ from platformio import exception, fs, util from platformio.commands import PlatformioCLI +from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.run.command import cli as cmd_run from platformio.compat import is_bytes @@ -301,7 +302,5 @@ def _look_for_serial_port(hwids): debug_port = _look_for_serial_port(tool_settings.get("hwids", [])) if not debug_port: - raise exception.DebugInvalidOptions( - "Please specify `debug_port` for environment" - ) + raise DebugInvalidOptionsError("Please specify `debug_port` for environment") return debug_port diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index b0ddb9083a..3b16b61dce 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -17,7 +17,8 @@ from twisted.internet import reactor # pylint: disable=import-error -from platformio import exception, fs, util +from platformio import fs, util +from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode from platformio.commands.debug.process import BaseProcess from platformio.proc import where_is_program @@ -53,7 +54,7 @@ def spawn(self, patterns): # pylint: disable=too-many-branches if not isfile(server_executable): server_executable = where_is_program(server_executable) if not isfile(server_executable): - raise exception.DebugInvalidOptions( + raise DebugInvalidOptionsError( "\nCould not launch Debug Server '%s'. Please check that it " "is installed and is included in a system PATH\n\n" "See documentation or contact contact@platformio.org:\n" diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 6a547ae880..72c7ee8e4a 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -22,6 +22,7 @@ from platformio import exception, fs, util from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig +from platformio.project.exception import NotPlatformIOProjectError @click.group(short_help="Monitor device or list existing") @@ -181,7 +182,7 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches with fs.cd(kwargs["project_dir"]): project_options = get_project_options(kwargs["environment"]) kwargs = apply_project_monitor_options(kwargs, project_options) - except exception.NotPlatformIOProject: + except NotPlatformIOProjectError: pass if not kwargs["port"]: diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 8a58d88633..51e47dcb69 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -27,6 +27,7 @@ from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig +from platformio.project.exception import ProjectError from platformio.project.helpers import get_project_dir, is_platformio_project from platformio.project.options import get_config_options_schema @@ -113,7 +114,7 @@ def _path_to_name(path): try: with fs.cd(project_dir): data = _get_project_data() - except exception.PlatformIOProjectException: + except ProjectError: continue for board_id in data.get("boards", []): @@ -158,7 +159,7 @@ def get_project_examples(): config = ProjectConfig(os.path.join(project_dir, "platformio.ini")) config.validate(silent=True) project_description = config.get("platformio", "description") - except exception.PlatformIOProjectException: + except ProjectError: continue path_tokens = project_dir.split(os.path.sep) diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index f4b17ade0d..0ebbf856cd 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -23,6 +23,7 @@ from platformio import exception, fs from platformio.commands import device from platformio.managers.core import pioplus_call +from platformio.project.exception import NotPlatformIOProjectError # pylint: disable=unused-argument @@ -198,7 +199,7 @@ def device_monitor(ctx, **kwargs): with fs.cd(kwargs["project_dir"]): project_options = device.get_project_options(kwargs["environment"]) kwargs = device.apply_project_monitor_options(kwargs, project_options) - except exception.NotPlatformIOProject: + except NotPlatformIOProjectError: pass kwargs["baud"] = kwargs["baud"] or 9600 diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 3366c1e1ff..75b09b40f1 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -16,6 +16,7 @@ from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME from platformio.managers.platform import PlatformFactory +from platformio.project.exception import UndefinedEnvPlatformError # pylint: disable=too-many-instance-attributes @@ -56,12 +57,12 @@ def get_build_targets(self): def process(self): if "platform" not in self.options: - raise exception.UndefinedEnvPlatform(self.name) + raise UndefinedEnvPlatformError(self.name) build_vars = self.get_build_variables() build_targets = list(self.get_build_targets()) - telemetry.on_run_environment(self.options, build_targets) + telemetry.send_run_environment(self.options, build_targets) # skip monitor target, we call it above if "monitor" in build_targets: diff --git a/platformio/exception.py b/platformio/exception.py index 6e5910f8e3..cfd357a4b3 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -152,49 +152,6 @@ class FDSHASumMismatch(PlatformIOPackageException): ) -# -# Project -# - - -class PlatformIOProjectException(PlatformioException): - pass - - -class NotPlatformIOProject(PlatformIOProjectException): - - MESSAGE = ( - "Not a PlatformIO project. `platformio.ini` file has not been " - "found in current working directory ({0}). To initialize new project " - "please use `platformio init` command" - ) - - -class InvalidProjectConf(PlatformIOProjectException): - - MESSAGE = "Invalid '{0}' (project configuration file): '{1}'" - - -class UndefinedEnvPlatform(PlatformIOProjectException): - - MESSAGE = "Please specify platform for '{0}' environment" - - -class ProjectEnvsNotAvailable(PlatformIOProjectException): - - MESSAGE = "Please setup environments in `platformio.ini` file" - - -class UnknownEnvNames(PlatformIOProjectException): - - MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" - - -class ProjectOptionValueError(PlatformIOProjectException): - - MESSAGE = "{0} for option `{1}` in section [{2}]" - - # # Library # @@ -319,7 +276,7 @@ class UpgradeError(PlatformioException): """ -class HomeDirPermissionsError(PlatformioException): +class HomeDirPermissionsError(UserSideException): MESSAGE = ( "The directory `{0}` or its parent directory is not owned by the " @@ -338,20 +295,6 @@ class CygwinEnvDetected(PlatformioException): ) -class DebugSupportError(PlatformioException): - - MESSAGE = ( - "Currently, PlatformIO does not support debugging for `{0}`.\n" - "Please request support at https://github.com/platformio/" - "platformio-core/issues \nor visit -> https://docs.platformio.org" - "/page/plus/debugging.html" - ) - - -class DebugInvalidOptions(PlatformioException): - pass - - class TestDirNotExists(PlatformioException): MESSAGE = ( diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 94a9140a17..c9b3d74263 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -151,7 +151,7 @@ def after_upgrade(ctx): "PlatformIO has been successfully upgraded to %s!\n" % __version__, fg="green", ) - telemetry.on_event( + telemetry.send_event( category="Auto", action="Upgrade", label="%s > %s" % (last_version, __version__), @@ -315,7 +315,7 @@ def check_internal_updates(ctx, what): ctx.invoke(cmd_lib_update, libraries=outdated_items) click.echo() - telemetry.on_event(category="Auto", action="Update", label=what.title()) + telemetry.send_event(category="Auto", action="Update", label=what.title()) click.echo("*" * terminal_width) click.echo("") diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 7996d02a0b..2397a34d7f 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -26,7 +26,7 @@ CORE_PACKAGES = { "contrib-piohome": ">=3.1.0-beta.3,<3.2.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.6.0", + "tool-pioplus": "^2.6.1", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0", "tool-cppcheck": "~1.189.0", diff --git a/platformio/managers/package.py b/platformio/managers/package.py index b253ea0ee2..6df5276d7f 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -24,7 +24,7 @@ import requests import semantic_version -from platformio import __version__, app, exception, fs, telemetry, util +from platformio import __version__, app, exception, fs, util from platformio.compat import hashlib_encode_data from platformio.downloader import FileDownloader from platformio.lockfile import LockFile @@ -660,7 +660,7 @@ def outdated(self, pkg_dir, requirements=None): def install( self, name, requirements=None, silent=False, after_update=False, force=False - ): + ): # pylint: disable=unused-argument pkg_dir = None # interprocess lock with LockFile(self.package_dir): @@ -709,13 +709,6 @@ def install( manifest = self.load_manifest(pkg_dir) assert manifest - if not after_update: - telemetry.on_event( - category=self.__class__.__name__, - action="Install", - label=manifest["name"], - ) - click.secho( "{name} @ {version} has been successfully installed!".format( **manifest @@ -725,7 +718,9 @@ def install( return pkg_dir - def uninstall(self, package, requirements=None, after_update=False): + def uninstall( + self, package, requirements=None, after_update=False + ): # pylint: disable=unused-argument # interprocess lock with LockFile(self.package_dir): self.cache_reset() @@ -764,13 +759,6 @@ def uninstall(self, package, requirements=None, after_update=False): click.echo("[%s]" % click.style("OK", fg="green")) - if not after_update: - telemetry.on_event( - category=self.__class__.__name__, - action="Uninstall", - label=manifest["name"], - ) - return True def update(self, package, requirements=None, only_check=False): @@ -819,9 +807,6 @@ def update(self, package, requirements=None, only_check=False): self.uninstall(pkg_dir, after_update=True) self.install(name, latest, after_update=True) - telemetry.on_event( - category=self.__class__.__name__, action="Update", label=manifest["name"] - ) return True diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index ec7bd2a266..5d61829ee3 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -23,7 +23,11 @@ import click import semantic_version -from platformio import __version__, app, exception, fs, proc, util +from platformio import __version__, app, exception, fs, proc, telemetry, util +from platformio.commands.debug.exception import ( + DebugInvalidOptionsError, + DebugSupportError, +) from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager @@ -799,11 +803,12 @@ def get_debug_tool_name(self, custom=None): if tool_name == "custom": return tool_name if not debug_tools: - raise exception.DebugSupportError(self._manifest["name"]) + telemetry.send_event("Debug", "Request", self.id) + raise DebugSupportError(self._manifest["name"]) if tool_name: if tool_name in debug_tools: return tool_name - raise exception.DebugInvalidOptions( + raise DebugInvalidOptionsError( "Unknown debug tool `%s`. Please use one of `%s` or `custom`" % (tool_name, ", ".join(sorted(list(debug_tools)))) ) diff --git a/platformio/project/config.py b/platformio/project/config.py index 7fdbcdf8ad..739429f6a3 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -20,8 +20,9 @@ import click -from platformio import exception, fs +from platformio import fs from platformio.compat import PY2, WINDOWS, hashlib_encode_data +from platformio.project import exception from platformio.project.options import ProjectOptions try: @@ -104,7 +105,7 @@ def read(self, path, parse_extra=True): try: self._parser.read(path) except ConfigParser.Error as e: - raise exception.InvalidProjectConf(path, str(e)) + raise exception.InvalidProjectConfError(path, str(e)) if not parse_extra: return @@ -273,7 +274,7 @@ def get(self, section, option, default=None): # pylint: disable=too-many-branch except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): pass # handle value from system environment except ConfigParser.Error as e: - raise exception.InvalidProjectConf(self.path, str(e)) + raise exception.InvalidProjectConfError(self.path, str(e)) option_meta = ProjectOptions.get("%s.%s" % (section.split(":", 1)[0], option)) if not option_meta: @@ -327,14 +328,14 @@ def default_envs(self): def validate(self, envs=None, silent=False): if not os.path.isfile(self.path): - raise exception.NotPlatformIOProject(self.path) + raise exception.NotPlatformIOProjectError(self.path) # check envs known = set(self.envs()) if not known: - raise exception.ProjectEnvsNotAvailable() + raise exception.ProjectEnvsNotAvailableError() unknown = set(list(envs or []) + self.default_envs()) - known if unknown: - raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) + raise exception.UnknownEnvNamesError(", ".join(unknown), ", ".join(known)) if not silent: for warning in self.warnings: click.secho("Warning! %s" % warning, fg="yellow") diff --git a/platformio/project/exception.py b/platformio/project/exception.py new file mode 100644 index 0000000000..c2d7fd09ca --- /dev/null +++ b/platformio/project/exception.py @@ -0,0 +1,53 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException, UserSideException + + +class ProjectError(PlatformioException): + pass + + +class NotPlatformIOProjectError(ProjectError, UserSideException): + + MESSAGE = ( + "Not a PlatformIO project. `platformio.ini` file has not been " + "found in current working directory ({0}). To initialize new project " + "please use `platformio init` command" + ) + + +class InvalidProjectConfError(ProjectError, UserSideException): + + MESSAGE = "Invalid '{0}' (project configuration file): '{1}'" + + +class UndefinedEnvPlatformError(ProjectError, UserSideException): + + MESSAGE = "Please specify platform for '{0}' environment" + + +class ProjectEnvsNotAvailableError(ProjectError, UserSideException): + + MESSAGE = "Please setup environments in `platformio.ini` file" + + +class UnknownEnvNamesError(ProjectError, UserSideException): + + MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" + + +class ProjectOptionValueError(ProjectError, UserSideException): + + MESSAGE = "{0} for option `{1}` in section [{2}]" diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 4a55f96866..f765e06d18 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -13,13 +13,12 @@ # limitations under the License. import atexit +import os import platform import re import sys import threading from collections import deque -from os import getenv, sep -from os.path import join from time import sleep, time from traceback import format_exc @@ -99,8 +98,8 @@ def _prefill_appinfo(self): dpdata.append("PlatformIO/%s" % __version__) if app.get_session_var("caller_id"): dpdata.append("Caller/%s" % app.get_session_var("caller_id")) - if getenv("PLATFORMIO_IDE"): - dpdata.append("IDE/%s" % getenv("PLATFORMIO_IDE")) + if os.getenv("PLATFORMIO_IDE"): + dpdata.append("IDE/%s" % os.getenv("PLATFORMIO_IDE")) self["an"] = " ".join(dpdata) def _prefill_custom_data(self): @@ -179,13 +178,10 @@ def _first_arg_from_list(args_, list_): cmd_path.append(sub_cmd) self["screen_name"] = " ".join([p.title() for p in cmd_path]) - @staticmethod - def _ignore_hit(): + def _ignore_hit(self): if not app.get_setting("enable_telemetry"): return True - if app.get_session_var("caller_id") and all( - c in sys.argv for c in ("run", "idedata") - ): + if all(c in sys.argv for c in ("run", "idedata")) or self["ea"] == "Idedata": return True return False @@ -296,29 +292,64 @@ def on_command(): measure_ci() +def on_exception(e): + skip_conditions = [ + isinstance(e, cls) + for cls in (IOError, exception.ReturnErrorCode, exception.UserSideException,) + ] + try: + skip_conditions.append("[API] Account: " in str(e)) + except UnicodeEncodeError as ue: + e = ue + if any(skip_conditions): + return + is_fatal = any( + [ + not isinstance(e, exception.PlatformioException), + "Error" in e.__class__.__name__, + ] + ) + description = "%s: %s" % ( + type(e).__name__, + " ".join(reversed(format_exc().split("\n"))) if is_fatal else str(e), + ) + send_exception(description, is_fatal) + + def measure_ci(): event = {"category": "CI", "action": "NoName", "label": None} known_cis = ("TRAVIS", "APPVEYOR", "GITLAB_CI", "CIRCLECI", "SHIPPABLE", "DRONE") for name in known_cis: - if getenv(name, "false").lower() == "true": + if os.getenv(name, "false").lower() == "true": event["action"] = name break - on_event(**event) + send_event(**event) + + +def encode_run_environment(options): + non_sensative_keys = [ + "platform", + "framework", + "board", + "upload_protocol", + "check_tool", + "debug_tool", + ] + safe_options = [ + "%s=%s" % (k, v) for k, v in sorted(options.items()) if k in non_sensative_keys + ] + return "&".join(safe_options) -def on_run_environment(options, targets): - non_sensative_values = ["board", "platform", "framework"] - safe_options = [] - for key, value in sorted(options.items()): - if key in non_sensative_values: - safe_options.append("%s=%s" % (key, value)) - else: - safe_options.append(key) - targets = [t.title() for t in targets or ["run"]] - on_event("Env", " ".join(targets), "&".join(safe_options)) +def send_run_environment(options, targets): + send_event( + "Env", + " ".join([t.title() for t in targets or ["run"]]), + encode_run_environment(options), + ) -def on_event(category, action, label=None, value=None, screen_name=None): +def send_event(category, action, label=None, value=None, screen_name=None): mp = MeasurementProtocol() mp["event_category"] = category[:150] mp["event_action"] = action[:500] @@ -331,43 +362,21 @@ def on_event(category, action, label=None, value=None, screen_name=None): mp.send("event") -def on_exception(e): - def _cleanup_description(text): - text = text.replace("Traceback (most recent call last):", "") - text = re.sub( - r'File "([^"]+)"', - lambda m: join(*m.group(1).split(sep)[-2:]), - text, - flags=re.M, - ) - text = re.sub(r"\s+", " ", text, flags=re.M) - return text.strip() - - skip_conditions = [ - isinstance(e, cls) - for cls in ( - IOError, - exception.ReturnErrorCode, - exception.UserSideException, - exception.PlatformIOProjectException, - ) - ] - try: - skip_conditions.append("[API] Account: " in str(e)) - except UnicodeEncodeError as ue: - e = ue - if any(skip_conditions): - return - is_crash = any( - [ - not isinstance(e, exception.PlatformioException), - "Error" in e.__class__.__name__, - ] +def send_exception(description, is_fatal=False): + # cleanup sensitive information, such as paths + description = description.replace("Traceback (most recent call last):", "") + description = description.replace("\\", "/") + description = re.sub( + r'(^|\s+|")(?:[a-z]\:)?((/[^"/]+)+)(\s+|"|$)', + lambda m: " %s " % os.path.join(*m.group(2).split("/")[-2:]), + description, + re.I | re.M, ) + description = re.sub(r"\s+", " ", description, flags=re.M) + mp = MeasurementProtocol() - description = _cleanup_description(format_exc() if is_crash else str(e)) - mp["exd"] = ("%s: %s" % (type(e).__name__, description))[:2048] - mp["exf"] = 1 if is_crash else 0 + mp["exd"] = description[:8192].strip() + mp["exf"] = 1 if is_fatal else 0 mp.send("exception") From f9cae602257bec191ccbc5cd2f64d217a9f38188 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 Nov 2019 18:14:10 +0200 Subject: [PATCH 051/142] Fix tests --- tests/commands/test_init.py | 4 ++-- tests/test_projectconf.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 4310fc419d..5e0172a786 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -16,10 +16,10 @@ from os import getcwd, makedirs from os.path import getsize, isdir, isfile, join -from platformio import exception from platformio.commands.boards import cli as cmd_boards from platformio.commands.init import cli as cmd_init from platformio.project.config import ProjectConfig +from platformio.project.exception import ProjectEnvsNotAvailableError def validate_pioproject(pioproject_dir): @@ -59,7 +59,7 @@ def test_init_ide_without_board(clirunner, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke(cmd_init, ["--ide", "atom"]) assert result.exit_code != 0 - assert isinstance(result.exception, exception.ProjectEnvsNotAvailable) + assert isinstance(result.exception, ProjectEnvsNotAvailableError) def test_init_ide_atom(clirunner, validate_cliresult, tmpdir): diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 3719348267..b601ceceb2 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -16,8 +16,8 @@ import pytest -from platformio.exception import UnknownEnvNames from platformio.project.config import ConfigParser, ProjectConfig +from platformio.project.exception import UnknownEnvNamesError BASE_CONFIG = """ [platformio] @@ -115,7 +115,7 @@ def test_warnings(config): assert len(config.warnings) == 2 assert "lib_install" in config.warnings[1] - with pytest.raises(UnknownEnvNames): + with pytest.raises(UnknownEnvNamesError): config.validate(["non-existing-env"]) From 5c388d4271d390a95d549b0e83c255512cfd6b0c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 Nov 2019 18:14:53 +0200 Subject: [PATCH 052/142] Fix issue with invalid checking used port for PIO Home server --- platformio/commands/home/command.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 56298ea12c..ddbcd039fd 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -88,12 +88,13 @@ def cli(port, host, no_open, shutdown_timeout): return # if already started - already_started = False - socket.setdefaulttimeout(1) + already_started = True try: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) - already_started = True - except: # pylint: disable=bare-except + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind((host, port)) + s.close() + already_started = False + except (OSError, socket.error): pass home_url = "http://%s:%d" % (host, port) From f852f9fa89324835d3490467cdb0e76b6c656369 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 Nov 2019 18:23:00 +0200 Subject: [PATCH 053/142] Small refacoring --- platformio/commands/home/command.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index ddbcd039fd..1c1fff3fad 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -87,16 +87,7 @@ def cli(port, host, no_open, shutdown_timeout): if host == "__do_not_start__": return - # if already started - already_started = True - try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.bind((host, port)) - s.close() - already_started = False - except (OSError, socket.error): - pass - + already_started = is_port_used(host, port) home_url = "http://%s:%d" % (host, port) if not no_open: if already_started: @@ -126,3 +117,14 @@ def cli(port, host, no_open, shutdown_timeout): reactor.listenTCP(port, site, interface=host) reactor.run() + + +def is_port_used(host, port): + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind((host, port)) + s.close() + return False + except (OSError, socket.error): + pass + return True From 4f1ccfe58ff5719cb72c213093a173023b960782 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 Nov 2019 18:58:09 +0200 Subject: [PATCH 054/142] Bump version to 4.1.1b5 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 21816d0f34..55ab0d67d9 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1b4") +VERSION = (4, 1, "1b5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 2397a34d7f..d1edb2c38f 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.1.0-beta.3,<3.2.0", + "contrib-piohome": ">=3.1.0-beta.4,<3.2.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.1", "tool-unity": "~1.20403.0", From d901cc875a75ce7026db54fbb8415815a02fbd01 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Dec 2019 19:28:21 +0200 Subject: [PATCH 055/142] Minor improvements for docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index ad87be4bdd..2589fe1fbe 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ad87be4bdd04c134d981ed0e302f6868f4694308 +Subproject commit 2589fe1fbe24887fb715655e2eff773b8ab768b1 From 5768fcd4293e10dc445c57e482b47cba5133a9b2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Dec 2019 19:34:19 +0200 Subject: [PATCH 056/142] Inform that PlatformIO Home server is already started in another process --- platformio/commands/home/command.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 1c1fff3fad..5d231421da 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -108,9 +108,12 @@ def cli(port, host, no_open, shutdown_timeout): ) ) click.echo("") - click.echo("Open PIO Home in your browser by this URL => %s" % home_url) + click.echo("Open PlatformIO Home in your browser by this URL => %s" % home_url) if already_started: + click.secho( + "PlatformIO Home server is already started in another process.", fg="yellow" + ) return click.echo("PIO Home has been started. Press Ctrl+C to shutdown.") From 09852dcada44d5e16d5a2c591bdd5777ee6ec1c5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 9 Dec 2019 14:46:28 -0800 Subject: [PATCH 057/142] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 2589fe1fbe..4fbd867444 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2589fe1fbe24887fb715655e2eff773b8ab768b1 +Subproject commit 4fbd867444b7de1ebc37c6abeac583be9a1c1091 From 31eed6c5e5bdba7d47e036e96f232e471890e565 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 9 Dec 2019 14:47:00 -0800 Subject: [PATCH 058/142] Improve detecting of a run PIO Home Server --- platformio/commands/home/command.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 5d231421da..c5d5dc147f 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -21,6 +21,7 @@ import click from platformio import exception +from platformio.compat import WINDOWS from platformio.managers.core import get_core_package_dir, inject_contrib_pysite @@ -123,11 +124,20 @@ def cli(port, host, no_open, shutdown_timeout): def is_port_used(host, port): - try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.bind((host, port)) - s.close() - return False - except (OSError, socket.error): - pass + socket.setdefaulttimeout(1) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if WINDOWS: + try: + s.bind((host, port)) + s.close() + return False + except (OSError, socket.error): + pass + else: + try: + s.connect((host, port)) + s.close() + except socket.error: + return False + return True From c57f68aee3a092b181d649fa27a521ecbf9e0dcd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Dec 2019 15:22:02 +0200 Subject: [PATCH 059/142] Change order between board/framework for configuration file --- platformio/project/options.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/platformio/project/options.py b/platformio/project/options.py index 50a264faf3..7d4c5a71b6 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -284,13 +284,6 @@ def ConfigEnvOption(*args, **kwargs): description="Custom packages and specifications", multiple=True, ), - ConfigEnvOption( - group="platform", - name="framework", - description="A list of project dependent frameworks", - multiple=True, - buildenvvar="PIOFRAMEWORK", - ), # Board ConfigEnvOption( group="platform", @@ -298,6 +291,13 @@ def ConfigEnvOption(*args, **kwargs): description="A board ID", buildenvvar="BOARD", ), + ConfigEnvOption( + group="platform", + name="framework", + description="A list of project dependent frameworks", + multiple=True, + buildenvvar="PIOFRAMEWORK", + ), ConfigEnvOption( group="platform", name="board_build.mcu", From b4088a6d00c92bec1cce389e06d54c0d2e844991 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Dec 2019 22:39:15 +0200 Subject: [PATCH 060/142] Drop support for Samsung Artik dev/platform // Resolve #3309 --- README.rst | 3 --- docs | 2 +- examples | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 56d553283a..03fd04c60f 100644 --- a/README.rst +++ b/README.rst @@ -95,7 +95,6 @@ Development Platforms * `NXP LPC `_ * `RISC-V `_ * `RISC-V GAP `_ -* `Samsung ARTIK `_ * `Shakti `_ * `Silicon Labs EFM32 `_ * `ST STM32 `_ @@ -109,7 +108,6 @@ Frameworks ---------- * `Arduino `_ -* `ARTIK SDK `_ * `CMSIS `_ * `Energia `_ * `ESP-IDF `_ @@ -125,7 +123,6 @@ Frameworks * `Simba `_ * `SPL `_ * `STM32Cube `_ -* `Tizen RT `_ * `WiringPi `_ Contributing diff --git a/docs b/docs index 4fbd867444..5aaba30d88 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4fbd867444b7de1ebc37c6abeac583be9a1c1091 +Subproject commit 5aaba30d88cc49c7bdf9c593006e14189d13d9ce diff --git a/examples b/examples index f5835d10b3..5e6a75b2bd 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit f5835d10b38e9928e3378585caaf3f9de4eeb533 +Subproject commit 5e6a75b2bd2280780b96445b7eb845d7639ac93b From b15ddc00a5d8b284b512d1ce907f8d801a82a6ab Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Dec 2019 22:40:28 +0200 Subject: [PATCH 061/142] Pass extra files with compiler options to VSCode IntelliSense tool --- platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index cb05573015..5df41c544d 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -56,7 +56,7 @@ % cxx_stds = STD_RE.findall(cxx_flags) % % # pass only architecture specific flags -% cc_m_flags = " ".join([f.strip() for f in cc_flags.split(" ") if f.strip().startswith(("-m", "-i"))]) +% cc_m_flags = " ".join([f.strip() for f in cc_flags.split(" ") if f.strip().startswith(("-m", "-i", "@"))]) % % if cc_stds: "cStandard": "c{{ cc_stds[-1] }}", From 1bd6e898ad425206a32ae5203fac0b089cbe4a3f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 18 Dec 2019 14:03:10 +0200 Subject: [PATCH 062/142] Fix tests --- docs | 2 +- tests/commands/test_lib.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs b/docs index 5aaba30d88..c25f5d453c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5aaba30d88cc49c7bdf9c593006e14189d13d9ce +Subproject commit c25f5d453c2dfd1bb3ac82c107082c0ed52179bb diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index b40935eee6..dc5dc00a53 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -230,7 +230,9 @@ def test_global_lib_update_check(clirunner, validate_cliresult): ) validate_cliresult(result) output = json.loads(result.output) - assert set(["RFcontrol", "NeoPixelBus"]) == set([l["name"] for l in output]) + assert set(["RFcontrol", "ESPAsyncTCP", "NeoPixelBus"]) == set( + [l["name"] for l in output] + ) def test_global_lib_update(clirunner, validate_cliresult): @@ -250,7 +252,7 @@ def test_global_lib_update(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "update"]) validate_cliresult(result) assert result.output.count("[Detached]") == 5 - assert result.output.count("[Up-to-date]") == 11 + assert result.output.count("[Up-to-date]") == 10 assert "Uninstalling RFcontrol @ 77d4eb3f8a" in result.output # update unknown library From 135ef8701cfa2bf9b69159159c2de98fd73a4233 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Dec 2019 01:19:23 +0200 Subject: [PATCH 063/142] Better representation of dependent packages --- platformio/builder/tools/pioplatform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index f7a085dffd..be5e1fdafa 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -213,7 +213,9 @@ def _get_packages_data(): if extra: info += " (%s)" % ", ".join(extra) data.append(info) - return ["PACKAGES:", ", ".join(data)] + if not data: + return None + return ["PACKAGES:"] + ["\n - %s" % d for d in sorted(data)] for data in ( _get_configuration_data(), From f9494c940eb63a505231354efacc17840e4d370f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Dec 2019 20:20:09 +0200 Subject: [PATCH 064/142] Add support for "downloadUrl" in package manifest --- Makefile | 2 +- platformio/package/manifest/schema.py | 4 +++- tests/test_pkgmanifest.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 088bae1108..30cfe1781a 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ black: black --target-version py27 ./tests test: - py.test --verbose --capture=no --exitfirst -n 3 --dist=loadscope tests --ignore tests/test_examples.py + py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py before-commit: isort black lint test diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index f1d68e0876..790f322634 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -92,8 +92,11 @@ class ManifestSchema(Schema): homepage = fields.Url(validate=validate.Length(min=1, max=255)) license = fields.Str(validate=validate.Length(min=1, max=255)) repository = fields.Nested(RepositorySchema) + + # library.json export = fields.Nested(ExportSchema) examples = fields.Nested(ExampleSchema, many=True) + downloadUrl = fields.Url(validate=validate.Length(min=1, max=255)) keywords = StrictListField( fields.Str( @@ -105,7 +108,6 @@ class ManifestSchema(Schema): ] ) ) - platforms = StrictListField( fields.Str( validate=[ diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 18078a7f09..72b347c587 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -202,6 +202,7 @@ def test_library_json_schema(): "name": "Benoit Blanchon", "url": "https://blog.benoitblanchon.fr" }, + "downloadUrl": "https://example.com/package.tar.gz", "exclude": [ "fuzzing", "scripts", @@ -251,6 +252,7 @@ def test_library_json_schema(): "authors": [ {"name": "Benoit Blanchon", "url": "https://blog.benoitblanchon.fr"} ], + "downloadUrl": "https://example.com/package.tar.gz", "export": {"exclude": ["fuzzing", "scripts", "test", "third-party"]}, "frameworks": ["arduino"], "platforms": ["*"], From 7d90c468ae8d7ebaba181eda06f240901ac9037a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 22 Dec 2019 01:27:51 +0200 Subject: [PATCH 065/142] Updated SCons tool to 3.1.2 --- HISTORY.rst | 1 + docs | 2 +- platformio/managers/core.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 290a04f4ba..a15503025f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ PlatformIO Core 4.0 - Project Configuration UI for `"platformio.ini" `__ * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) +* Updated SCons tool to 3.1.2 * Warn about broken library manifest when scanning dependencies (`issue #3268 `_) * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) * Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) diff --git a/docs b/docs index c25f5d453c..b4e4f10e4d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c25f5d453c2dfd1bb3ac82c107082c0ed52179bb +Subproject commit b4e4f10e4d40783c6e2d15e3e33b984d619c26f9 diff --git a/platformio/managers/core.py b/platformio/managers/core.py index d1edb2c38f..4703de92a6 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -28,7 +28,7 @@ "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.1", "tool-unity": "~1.20403.0", - "tool-scons": "~2.20501.7" if PY2 else "~3.30101.0", + "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", "tool-cppcheck": "~1.189.0", "tool-clangtidy": "^1.80000.0", } From 26e7069099ab948352a40a9c93f38d9b6b0f73db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 22 Dec 2019 12:33:08 +0200 Subject: [PATCH 066/142] Fixed default PIO Unified Debugger configuration for J-Link probe --- HISTORY.rst | 1 + platformio/commands/debug/initcfgs.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index a15503025f..574426028f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ PlatformIO Core 4.0 * Warn about broken library manifest when scanning dependencies (`issue #3268 `_) * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) * Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) +* Fixed default PIO Unified Debugger configuration for `J-Link probe `__ 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index b241efc60a..50da1779c5 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -59,8 +59,8 @@ target extended-remote $DEBUG_PORT monitor clrbp monitor speed auto -$LOAD_CMDS pio_reset_halt_target +$LOAD_CMDS $INIT_BREAK """ From 7e0fb43dbe440e7b49b69f2e43548aa63b961fb0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 22 Dec 2019 23:45:12 +0200 Subject: [PATCH 067/142] Normalise repository url when parsing package manifest --- Makefile | 4 +-- platformio/package/manifest/parser.py | 48 ++++++++++++++++++--------- tests/test_pkgmanifest.py | 20 +++++++++-- 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 30cfe1781a..548f96f3ef 100644 --- a/Makefile +++ b/Makefile @@ -5,14 +5,14 @@ isort: isort -rc ./platformio isort -rc ./tests -black: +format: black --target-version py27 ./platformio black --target-version py27 ./tests test: py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py -before-commit: isort black lint test +before-commit: isort format lint test clean-docs: rm -rf docs/_build diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 22c21b7b4c..3e2ff5ac3a 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -125,6 +125,8 @@ def __init__(self, contents, remote_url=None, package_dir=None): self._data = self.parse(contents) except Exception as e: raise ManifestParserError("Could not parse manifest -> %s" % e) + + self._data = self.normalize_repository(self._data) self._data = self.parse_examples(self._data) # remove None fields @@ -139,7 +141,7 @@ def as_dict(self): return self._data @staticmethod - def cleanup_author(author): + def normalize_author(author): assert isinstance(author, dict) if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) @@ -160,6 +162,22 @@ def parse_author_name_and_email(raw): email = raw[raw.index(ldel) + 1 : raw.index(rdel)] return (name.strip(), email.strip() if email else None) + @staticmethod + def normalize_repository(data): + url = (data.get("repository") or {}).get("url") + if not url or "://" not in url: + return data + url_attrs = urlparse(url) + if url_attrs.netloc not in ("github.com", "bitbucket.org", "gitlab.com"): + return data + url = "https://%s%s" % (url_attrs.netloc, url_attrs.path) + if url.endswith("/"): + url = url[:-1] + if not url.endswith(".git"): + url += ".git" + data["repository"]["url"] = url + return data + def parse_examples(self, data): examples = data.get("examples") if ( @@ -305,7 +323,7 @@ def _parse_authors(self, raw): # normalize Union[dict, list] fields if not isinstance(raw, list): raw = [raw] - return [self.cleanup_author(author) for author in raw] + return [self.normalize_author(author) for author in raw] @staticmethod def _parse_platforms(raw): @@ -352,7 +370,7 @@ def _parse_authors(self, raw): name, email = self.parse_author_name_and_email(author) if not name: continue - result.append(self.cleanup_author(dict(name=name, email=email))) + result.append(self.normalize_author(dict(name=name, email=email))) return result @staticmethod @@ -451,7 +469,7 @@ def _parse_authors(self, properties): name, email = self.parse_author_name_and_email(author) if not name: continue - authors.append(self.cleanup_author(dict(name=name, email=email))) + authors.append(self.normalize_author(dict(name=name, email=email))) for author in properties.get("maintainer", "").split(","): name, email = self.parse_author_name_and_email(author) if not name: @@ -466,27 +484,25 @@ def _parse_authors(self, properties): item["email"] = email if not found: authors.append( - self.cleanup_author(dict(name=name, email=email, maintainer=True)) + self.normalize_author(dict(name=name, email=email, maintainer=True)) ) return authors def _parse_repository(self, properties): if self.remote_url: - repo_parse = urlparse(self.remote_url) - repo_path_tokens = repo_parse.path[1:].split("/")[:-1] - if "github" in repo_parse.netloc: + url_attrs = urlparse(self.remote_url) + repo_path_tokens = url_attrs.path[1:].split("/")[:-1] + if "github" in url_attrs.netloc: return dict( type="git", - url="%s://github.com/%s" - % (repo_parse.scheme, "/".join(repo_path_tokens[:2])), + url="https://github.com/" + "/".join(repo_path_tokens[:2]), ) if "raw" in repo_path_tokens: return dict( type="git", - url="%s://%s/%s" + url="https://%s/%s" % ( - repo_parse.scheme, - repo_parse.netloc, + url_attrs.netloc, "/".join(repo_path_tokens[: repo_path_tokens.index("raw")]), ), ) @@ -498,9 +514,9 @@ def _parse_export(self): result = {"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]} include = None if self.remote_url: - repo_parse = urlparse(self.remote_url) - repo_path_tokens = repo_parse.path[1:].split("/")[:-1] - if "github" in repo_parse.netloc: + url_attrs = urlparse(self.remote_url) + repo_path_tokens = url_attrs.path[1:].split("/")[:-1] + if "github" in url_attrs.netloc: include = "/".join(repo_path_tokens[3:]) or None elif "raw" in repo_path_tokens: include = ( diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 72b347c587..204ee07f6f 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -29,6 +29,10 @@ def test_library_json_parser(): "name": "TestPackage", "keywords": "kw1, KW2, kw3", "platforms": ["atmelavr", "espressif"], + "repository": { + "type": "git", + "url": "http://github.com/username/repo/" + }, "url": "http://old.url.format", "exclude": [".gitignore", "tests"], "include": "mylib", @@ -44,6 +48,10 @@ def test_library_json_parser(): { "name": "TestPackage", "platforms": ["atmelavr", "espressif8266"], + "repository": { + "type": "git", + "url": "https://github.com/username/repo.git", + }, "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, "keywords": ["kw1", "kw2", "kw3"], "homepage": "http://old.url.format", @@ -172,7 +180,7 @@ def test_library_properties_parser(): "include": ["libraries/TestPackage"], } assert data["repository"] == { - "url": "https://github.com/username/reponame", + "url": "https://github.com/username/reponame.git", "type": "git", } @@ -299,7 +307,10 @@ def test_library_properties_schema(): "A library for monochrome TFTs and OLEDs. Supported display " "controller: SSD1306, SSD1309, SSD1322, SSD1325" ), - "repository": {"url": "https://github.com/olikraus/u8glib", "type": "git"}, + "repository": { + "url": "https://github.com/olikraus/u8glib.git", + "type": "git", + }, "frameworks": ["arduino"], "platforms": ["atmelavr", "atmelsam"], "version": "1.19.1", @@ -350,7 +361,10 @@ def test_library_properties_schema(): "sounds using familiar synthesis units like oscillators, delays, " "filters and envelopes." ), - "repository": {"url": "https://github.com/sensorium/Mozzi", "type": "git"}, + "repository": { + "url": "https://github.com/sensorium/Mozzi.git", + "type": "git", + }, "platforms": ["*"], "frameworks": ["arduino"], "export": { From 2f050400815771e8c4a8d01ea88332dfe43835e7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 24 Dec 2019 14:36:44 +0200 Subject: [PATCH 068/142] Fixed an issue with LDF when header files not found if "libdeps_dir" is within a subdirectory of "lib_extra_dirs" // Resolve #3311 --- HISTORY.rst | 1 + platformio/builder/tools/piolib.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 574426028f..b924871845 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,7 @@ PlatformIO Core 4.0 * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) * Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ +* Fixed an issue with LDF when header files not found if "libdeps_dir" is within a subdirectory of "lib_extra_dirs" (`issue #3311 `_) 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index bd4bcf7482..58d965e0e3 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -886,7 +886,7 @@ def process_dependencies(self): # pylint: disable=too-many-branches if not lib_dir: continue for lb in self.env.GetLibBuilders(): - if lib_dir not in lb: + if lib_dir != lb.path: continue if lb not in self.depbuilders: self.depend_recursive(lb) From 20a10c7fc517e2a9ec1d35463e2b50e65f35d80a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 24 Dec 2019 23:43:21 +0200 Subject: [PATCH 069/142] Fixed an issue "Import of non-existent variable 'projenv''" // Resolve #3315 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/main.py | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b924871845..a681588c3d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -21,6 +21,7 @@ PlatformIO Core 4.0 * Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ * Fixed an issue with LDF when header files not found if "libdeps_dir" is within a subdirectory of "lib_extra_dirs" (`issue #3311 `_) +* Fixed an issue "Import of non-existent variable 'projenv''" when development platform does not call "env.BuildProgram()" (`issue #3315 `_) 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index b4e4f10e4d..f53767e918 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b4e4f10e4d40783c6e2d15e3e33b984d619c26f9 +Subproject commit f53767e91846040f4373ef7ca743d5edd34ff0ac diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 8f7586e41a..a49017a03c 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -188,7 +188,10 @@ env.Exit(0) if "idedata" in COMMAND_LINE_TARGETS: - Import("projenv") + try: + Import("projenv") + except: # pylint: disable=bare-except + projenv = env click.echo( "\n%s\n" % dump_json_to_unicode( From f7385e8e88e1e3f969d7b668fadd388ce0707ac6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 25 Dec 2019 00:44:01 +0200 Subject: [PATCH 070/142] Bump version to 4.1.1b6 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 55ab0d67d9..dc0c5bf018 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1b5") +VERSION = (4, 1, "1b6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 4703de92a6..c90dbdb48d 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.1.0-beta.4,<3.2.0", + "contrib-piohome": ">=3.1.0-beta.8,<3.2.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.1", "tool-unity": "~1.20403.0", From 442a7e357636522e844d95375c246644b21a7802 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 29 Dec 2019 13:13:04 +0200 Subject: [PATCH 071/142] Made package ManifestSchema compatible with marshmallow >= 3 // Resolve #3296 --- HISTORY.rst | 1 + platformio/commands/lib.py | 6 +-- platformio/package/exception.py | 7 +-- platformio/package/manifest/schema.py | 61 +++++++++++++++++++++------ platformio/util.py | 3 +- setup.py | 2 +- tests/test_pkgmanifest.py | 56 ++++++++++++------------ 7 files changed, 84 insertions(+), 52 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a681588c3d..1ede584dd1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ PlatformIO Core 4.0 * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Updated SCons tool to 3.1.2 +* Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) * Warn about broken library manifest when scanning dependencies (`issue #3268 `_) * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) * Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 58cb85ed4e..d7c42d4c75 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -26,7 +26,7 @@ from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib from platformio.package.manifest.parser import ManifestParserFactory -from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError +from platformio.package.manifest.schema import ManifestSchema from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project @@ -495,11 +495,9 @@ def lib_register(config_url): raise exception.InvalidLibConfURL(config_url) # Validate manifest - data, error = ManifestSchema(strict=False).load( + ManifestSchema().load_manifest( ManifestParserFactory.new_from_url(config_url).as_dict() ) - if error: - raise ManifestValidationError(error, data) result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) if "message" in result and result["message"]: diff --git a/platformio/package/exception.py b/platformio/package/exception.py index 81753ad15a..f2597dd71b 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -24,13 +24,14 @@ class ManifestParserError(ManifestException): class ManifestValidationError(ManifestException): - def __init__(self, error, data): + def __init__(self, messages, data, valid_data): super(ManifestValidationError, self).__init__() - self.error = error + self.messages = messages self.data = data + self.valid_data = valid_data def __str__(self): return ( "Invalid manifest fields: %s. \nPlease check specification -> " - "http://docs.platformio.org/page/librarymanager/config.html" % self.error + "http://docs.platformio.org/page/librarymanager/config.html" % self.messages ) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 790f322634..7c1b34c445 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import marshmallow import requests import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates @@ -19,23 +20,61 @@ from platformio.package.exception import ManifestValidationError from platformio.util import memoized +MARSHMALLOW_2 = marshmallow.__version_info__ < (3,) -class StrictSchema(Schema): - def handle_error(self, error, data): + +if MARSHMALLOW_2: + + class CompatSchema(Schema): + pass + + +else: + + class CompatSchema(Schema): + class Meta: + unknown = marshmallow.EXCLUDE + + def handle_error( # pylint: disable=arguments-differ + self, error, data, **kwargs + ): + raise ManifestValidationError( + error.messages, + data, + error.valid_data if hasattr(error, "valid_data") else error.data, + ) + + +class BaseSchema(CompatSchema): + def load_manifest(self, data): + if MARSHMALLOW_2: + data, errors = self.load(data) + if errors: + raise ManifestValidationError(errors, data, data) + return data + return self.load(data) + + +class StrictSchema(BaseSchema): + def handle_error(self, error, data, **kwargs): # pylint: disable=arguments-differ # skip broken records if self.many: - error.data = [ + error.valid_data = [ item for idx, item in enumerate(data) if idx not in error.messages ] else: - error.data = None + error.valid_data = None + if MARSHMALLOW_2: + error.data = error.valid_data raise error class StrictListField(fields.List): - def _deserialize(self, value, attr, data): + def _deserialize(self, value, attr, data, **kwargs): try: - return super(StrictListField, self)._deserialize(value, attr, data) + return super(StrictListField, self)._deserialize( + value, attr, data, **kwargs + ) except ValidationError as exc: if exc.data: exc.data = [item for item in exc.data if item is not None] @@ -61,7 +100,7 @@ class RepositorySchema(StrictSchema): branch = fields.Str(validate=validate.Length(min=1, max=50)) -class ExportSchema(Schema): +class ExportSchema(BaseSchema): include = StrictListField(fields.Str) exclude = StrictListField(fields.Str) @@ -80,7 +119,7 @@ class ExampleSchema(StrictSchema): files = StrictListField(fields.Str, required=True) -class ManifestSchema(Schema): +class ManifestSchema(BaseSchema): # Required fields name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) @@ -144,10 +183,6 @@ class ManifestSchema(Schema): ) ) - def handle_error(self, error, data): - if self.strict: - raise ManifestValidationError(error, data) - @validates("version") def validate_version(self, value): # pylint: disable=no-self-use try: @@ -178,7 +213,7 @@ def validate_license(self, value): def load_spdx_licenses(): r = requests.get( "https://raw.githubusercontent.com/spdx/license-list-data" - "/v3.6/json/licenses.json" + "/v3.7/json/licenses.json" ) r.raise_for_status() return r.json() diff --git a/platformio/util.py b/platformio/util.py index 26529dbbf5..04bb723e6e 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -368,8 +368,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): PING_INTERNET_IPS = [ "192.30.253.113", # github.com - "31.28.1.238", # dl.platformio.org - "193.222.52.25", # dl.platformio.org + "78.46.220.20", # dl.platformio.org ] diff --git a/setup.py b/setup.py index 64a83fbf16..6413ffbd0e 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ "semantic_version>=2.8.1,<3", "tabulate>=0.8.3,<1", "pyelftools>=0.25,<1", - "marshmallow>=2.20.5,<3" + "marshmallow>=2.20.5", ] setup( diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 204ee07f6f..978ba1b431 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -238,8 +238,7 @@ def test_library_json_schema(): contents, parser.ManifestFileType.LIBRARY_JSON ).as_dict() - data, errors = ManifestSchema(strict=True).load(raw_data) - assert not errors + data = ManifestSchema().load_manifest(raw_data) assert data["repository"]["url"] == "https://github.com/bblanchon/ArduinoJson.git" assert data["examples"][1]["base"] == "examples/JsonHttpClient" @@ -297,8 +296,7 @@ def test_library_properties_schema(): contents, parser.ManifestFileType.LIBRARY_PROPERTIES ).as_dict() - data, errors = ManifestSchema(strict=True).load(raw_data) - assert not errors + data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, @@ -348,7 +346,12 @@ def test_library_properties_schema(): ), ).as_dict() - data, errors = ManifestSchema(strict=False).load(raw_data) + try: + ManifestSchema().load_manifest(raw_data) + except ManifestValidationError as e: + data = e.valid_data + errors = e.messages + assert errors["authors"] assert not jsondiff.diff( @@ -437,8 +440,7 @@ def test_platform_json_schema(): contents, parser.ManifestFileType.PLATFORM_JSON ).as_dict() raw_data["frameworks"] = sorted(raw_data["frameworks"]) - data, errors = ManifestSchema(strict=False).load(raw_data) - assert not errors + data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, @@ -477,8 +479,7 @@ def test_package_json_schema(): contents, parser.ManifestFileType.PACKAGE_JSON ).as_dict() - data, errors = ManifestSchema(strict=False).load(raw_data) - assert not errors + data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, @@ -580,8 +581,7 @@ def _sort_examples(items): raw_data["examples"] = _sort_examples(raw_data["examples"]) - data, errors = ManifestSchema(strict=True).load(raw_data) - assert not errors + data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( data, @@ -637,34 +637,32 @@ def _sort_examples(items): def test_broken_schemas(): - # non-strict mode - data, errors = ManifestSchema(strict=False).load(dict(name="MyPackage")) - assert set(errors.keys()) == set(["version"]) - assert data.get("version") is None - - # invalid keywords - data, errors = ManifestSchema(strict=False).load(dict(keywords=["kw1", "*^[]"])) - assert errors - assert data["keywords"] == ["kw1"] - - # strict mode + # missing required field + with pytest.raises( + ManifestValidationError, match=("Invalid semantic versioning format") + ) as exc_info: + ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version")) + assert exc_info.value.valid_data == {"name": "MyPackage"} + # invalid StrictList with pytest.raises( - ManifestValidationError, match="Missing data for required field" - ): - ManifestSchema(strict=True).load(dict(name="MyPackage")) + ManifestValidationError, match=("Invalid manifest fields.+keywords") + ) as exc_info: + ManifestSchema().load_manifest( + dict(name="MyPackage", version="1.0.0", keywords=["kw1", "*^[]"]) + ) + assert list(exc_info.value.messages.keys()) == ["keywords"] + assert exc_info.value.valid_data["keywords"] == ["kw1"] # broken SemVer with pytest.raises( ManifestValidationError, match=("Invalid semantic versioning format") ): - ManifestSchema(strict=True).load( - dict(name="MyPackage", version="broken_version") - ) + ManifestSchema().load_manifest(dict(name="MyPackage", version="broken_version")) # broken value for Nested with pytest.raises(ManifestValidationError, match=r"authors.*Invalid input type"): - ManifestSchema(strict=True).load( + ManifestSchema().load_manifest( dict( name="MyPackage", description="MyDescription", From ea0b462d0bc0e8723ae9b64a5628794c38d5513b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 29 Dec 2019 14:18:43 +0200 Subject: [PATCH 072/142] PyLint fixes --- platformio/package/manifest/schema.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 7c1b34c445..fe16413b9a 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-ancestors + import marshmallow import requests import semantic_version @@ -32,12 +34,10 @@ class CompatSchema(Schema): else: class CompatSchema(Schema): - class Meta: - unknown = marshmallow.EXCLUDE + class Meta(object): # pylint: disable=no-init + unknown = marshmallow.EXCLUDE # pylint: disable=no-member - def handle_error( # pylint: disable=arguments-differ - self, error, data, **kwargs - ): + def handle_error(self, error, data, **_): # pylint: disable=arguments-differ raise ManifestValidationError( error.messages, data, @@ -56,7 +56,7 @@ def load_manifest(self, data): class StrictSchema(BaseSchema): - def handle_error(self, error, data, **kwargs): # pylint: disable=arguments-differ + def handle_error(self, error, data, **_): # pylint: disable=arguments-differ # skip broken records if self.many: error.valid_data = [ @@ -70,7 +70,9 @@ def handle_error(self, error, data, **kwargs): # pylint: disable=arguments-diff class StrictListField(fields.List): - def _deserialize(self, value, attr, data, **kwargs): + def _deserialize( # pylint: disable=arguments-differ + self, value, attr, data, **kwargs + ): try: return super(StrictListField, self)._deserialize( value, attr, data, **kwargs From 643f2900572a83d68ac8ce2bb695aeee999cbd4f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 29 Dec 2019 17:43:50 +0200 Subject: [PATCH 073/142] Fix issue with library.properties manifest parser when author email is not specified --- platformio/package/manifest/parser.py | 2 +- tests/test_pkgmanifest.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 3e2ff5ac3a..8abed35b27 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -480,7 +480,7 @@ def _parse_authors(self, properties): continue found = True item["maintainer"] = True - if not item.get("email"): + if not item.get("email") and email: item["email"] = email if not found: authors.append( diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 978ba1b431..de59e0db4d 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -161,6 +161,7 @@ def test_library_properties_parser(): "architectures=*\n" + contents ).as_dict() assert data["platforms"] == ["*"] + # Platforms specific data = parser.LibraryPropertiesManifestParser( "architectures=avr, esp32\n" + contents @@ -184,7 +185,7 @@ def test_library_properties_parser(): "type": "git", } - # Hope page + # Home page data = parser.LibraryPropertiesManifestParser( "url=https://github.com/username/reponame.git\n" + contents ).as_dict() @@ -193,6 +194,17 @@ def test_library_properties_parser(): "url": "https://github.com/username/reponame.git", } + # Author + Maintainer + data = parser.LibraryPropertiesManifestParser( + """ +author=Rocket Scream Electronics +maintainer=Rocket Scream Electronics +""" + ).as_dict() + assert data["authors"] == [ + {"name": "Rocket Scream Electronics", "maintainer": True} + ] + def test_library_json_schema(): contents = """ From 209040fdc1dec17ea71faaba1adafcfba76d748d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Dec 2019 14:11:27 +0200 Subject: [PATCH 074/142] Initial support for Zephyr framework // Issue #1613 --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index f53767e918..9f4505000a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f53767e91846040f4373ef7ca743d5edd34ff0ac +Subproject commit 9f4505000aadd467731991f56accecfd5c71bd50 diff --git a/examples b/examples index 5e6a75b2bd..5f9db822f4 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 5e6a75b2bd2280780b96445b7eb845d7639ac93b +Subproject commit 5f9db822f4dbbf0aa9cf996438dcff291688cfc5 From 0ac5cd6789371305849a27f1cc031765d0ad0f92 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 31 Dec 2019 10:21:20 +0200 Subject: [PATCH 075/142] Skip empty "export" values when parsing library.json manifest --- platformio/package/manifest/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 8abed35b27..028d84a7d0 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -342,7 +342,7 @@ def _parse_export(raw): return None result = {} for k in ("include", "exclude"): - if k not in raw: + if not raw.get(k): continue result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]] return result From 682114d6f1a5ebba3a0838267d1640f7c06b0f26 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 31 Dec 2019 20:11:14 +0200 Subject: [PATCH 076/142] Bump version to 4.1.1b7 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index dc0c5bf018..6492ea68ba 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1b6") +VERSION = (4, 1, "1b7") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index c90dbdb48d..6166d8fe63 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.1.0-beta.8,<3.2.0", + "contrib-piohome": ">=3.1.0-rc.1,<3.2.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.1", "tool-unity": "~1.20403.0", From 8d7b775875bf2465e7c3a706077e6d9c2f9804dc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Jan 2020 15:52:54 +0200 Subject: [PATCH 077/142] Implement package packer --- docs | 2 +- platformio/commands/platform.py | 11 ++ platformio/compat.py | 2 +- platformio/fs.py | 6 +- platformio/managers/package.py | 8 +- platformio/package/exception.py | 4 + platformio/package/manifest/parser.py | 101 ++++++++------- platformio/package/pack.py | 92 ++++++++++++++ platformio/unpacker.py | 10 +- .../test_manifest.py} | 0 tests/package/test_pack.py | 116 ++++++++++++++++++ 11 files changed, 294 insertions(+), 58 deletions(-) create mode 100644 platformio/package/pack.py rename tests/{test_pkgmanifest.py => package/test_manifest.py} (100%) create mode 100644 tests/package/test_pack.py diff --git a/docs b/docs index 9f4505000a..3622e35819 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9f4505000aadd467731991f56accecfd5c71bd50 +Subproject commit 3622e3581951a4b193f7ce8f3aa5e05f59da29b5 diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 968f978b36..c094b560cd 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -20,6 +20,7 @@ from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.pack import PackagePacker @click.group(short_help="Platform Manager") @@ -403,3 +404,13 @@ def platform_update( # pylint: disable=too-many-locals click.echo() return True + + +@cli.command( + "pack", short_help="Create a tarball from development platform/tool package" +) +@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +def platform_pack(package): + p = PackagePacker(package) + tarball_path = p.pack() + click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green") diff --git a/platformio/compat.py b/platformio/compat.py index a8ff28e2b9..9107f8b19c 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -39,7 +39,7 @@ def get_locale_encoding(): def get_class_attributes(cls): - attributes = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a))) + attributes = inspect.getmembers(cls, lambda a: not inspect.isroutine(a)) return { a[0]: a[1] for a in attributes diff --git a/platformio/fs.py b/platformio/fs.py index ca561b10ac..57809ba676 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -143,10 +143,10 @@ def path_endswith_ext(path, extensions): return False -def match_src_files(src_dir, src_filter=None, src_exts=None): +def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True): def _append_build_item(items, item, src_dir): if not src_exts or path_endswith_ext(item, src_exts): - items.add(item.replace(src_dir + os.sep, "")) + items.add(os.path.relpath(item, src_dir)) src_filter = src_filter or "" if isinstance(src_filter, (list, tuple)): @@ -159,7 +159,7 @@ def _append_build_item(items, item, src_dir): items = set() for item in glob(os.path.join(glob_escape(src_dir), pattern)): if os.path.isdir(item): - for root, _, files in os.walk(item, followlinks=True): + for root, _, files in os.walk(item, followlinks=followlinks): for f in files: _append_build_item(items, os.path.join(root, f), src_dir) else: diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 6df5276d7f..d6481541b1 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -28,10 +28,8 @@ from platformio.compat import hashlib_encode_data from platformio.downloader import FileDownloader from platformio.lockfile import LockFile -from platformio.package.manifest.parser import ( - ManifestParserError, - ManifestParserFactory, -) +from platformio.package.exception import ManifestException +from platformio.package.manifest.parser import ManifestParserFactory from platformio.unpacker import FileUnpacker from platformio.vcsclient import VCSClientFactory @@ -347,7 +345,7 @@ def load_manifest(self, pkg_dir): # pylint: disable=too-many-branches try: manifest = ManifestParserFactory.new_from_file(manifest_path).as_dict() - except ManifestParserError: + except ManifestException: pass if src_manifest: diff --git a/platformio/package/exception.py b/platformio/package/exception.py index f2597dd71b..3927505d6c 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -19,6 +19,10 @@ class ManifestException(PlatformioException): pass +class UnknownManifestError(ManifestException): + pass + + class ManifestParserError(ManifestException): pass diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 028d84a7d0..71f63c7939 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect import json import os import re @@ -20,7 +21,7 @@ from platformio.compat import get_class_attributes, string_types from platformio.fs import get_file_contents -from platformio.package.exception import ManifestParserError +from platformio.package.exception import ManifestParserError, UnknownManifestError from platformio.project.helpers import is_platformio_project try: @@ -36,36 +37,36 @@ class ManifestFileType(object): MODULE_JSON = "module.json" PACKAGE_JSON = "package.json" + @classmethod + def items(cls): + return get_class_attributes(ManifestFileType) + @classmethod def from_uri(cls, uri): - if uri.endswith(".properties"): - return ManifestFileType.LIBRARY_PROPERTIES - if uri.endswith("platform.json"): - return ManifestFileType.PLATFORM_JSON - if uri.endswith("module.json"): - return ManifestFileType.MODULE_JSON - if uri.endswith("package.json"): - return ManifestFileType.PACKAGE_JSON - if uri.endswith("library.json"): - return ManifestFileType.LIBRARY_JSON + for t in sorted(cls.items().values()): + if uri.endswith(t): + return t return None + @classmethod + def from_dir(cls, path): + for t in sorted(cls.items().values()): + if os.path.isfile(os.path.join(path, t)): + return t + return None -class ManifestParserFactory(object): - @staticmethod - def type_to_clsname(t): - t = t.replace(".", " ") - t = t.title() - return "%sManifestParser" % t.replace(" ", "") +class ManifestParserFactory(object): @staticmethod def new_from_file(path, remote_url=False): if not path or not os.path.isfile(path): - raise ManifestParserError("Manifest file does not exist %s" % path) - for t in get_class_attributes(ManifestFileType).values(): - if path.endswith(t): - return ManifestParserFactory.new(get_file_contents(path), t, remote_url) - raise ManifestParserError("Unknown manifest file type %s" % path) + raise UnknownManifestError("Manifest file does not exist %s" % path) + type_from_uri = ManifestFileType.from_uri(path) + if not type_from_uri: + raise UnknownManifestError("Unknown manifest file type %s" % path) + return ManifestParserFactory.new( + get_file_contents(path), type_from_uri, remote_url + ) @staticmethod def new_from_dir(path, remote_url=None): @@ -80,23 +81,17 @@ def new_from_dir(path, remote_url=None): package_dir=path, ) - file_order = [ - ManifestFileType.PLATFORM_JSON, - ManifestFileType.LIBRARY_JSON, - ManifestFileType.LIBRARY_PROPERTIES, - ManifestFileType.MODULE_JSON, - ManifestFileType.PACKAGE_JSON, - ] - for t in file_order: - if not os.path.isfile(os.path.join(path, t)): - continue - return ManifestParserFactory.new( - get_file_contents(os.path.join(path, t)), - t, - remote_url=remote_url, - package_dir=path, + type_from_dir = ManifestFileType.from_dir(path) + if not type_from_dir: + raise UnknownManifestError( + "Unknown manifest file type in %s directory" % path ) - raise ManifestParserError("Unknown manifest file type in %s directory" % path) + return ManifestParserFactory.new( + get_file_contents(os.path.join(path, type_from_dir)), + type_from_dir, + remote_url=remote_url, + package_dir=path, + ) @staticmethod def new_from_url(remote_url): @@ -109,12 +104,18 @@ def new_from_url(remote_url): ) @staticmethod - def new(contents, type, remote_url=None, package_dir=None): - # pylint: disable=redefined-builtin - clsname = ManifestParserFactory.type_to_clsname(type) - if clsname not in globals(): - raise ManifestParserError("Unknown manifest file type %s" % clsname) - return globals()[clsname](contents, remote_url, package_dir) + def new( # pylint: disable=redefined-builtin + contents, type, remote_url=None, package_dir=None + ): + for _, cls in globals().items(): + if ( + inspect.isclass(cls) + and issubclass(cls, BaseManifestParser) + and cls != BaseManifestParser + and cls.manifest_type == type + ): + return cls(contents, remote_url, package_dir) + raise UnknownManifestError("Unknown manifest file type %s" % type) class BaseManifestParser(object): @@ -268,6 +269,8 @@ def parse_examples_from_dir(package_dir): class LibraryJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.LIBRARY_JSON + def parse(self, contents): data = json.loads(contents) data = self._process_renamed_fields(data) @@ -349,6 +352,8 @@ def _parse_export(raw): class ModuleJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.MODULE_JSON + def parse(self, contents): data = json.loads(contents) data["frameworks"] = ["mbed"] @@ -381,10 +386,12 @@ def _parse_license(raw): class LibraryPropertiesManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.LIBRARY_PROPERTIES + def parse(self, contents): data = self._parse_properties(contents) repository = self._parse_repository(data) - homepage = data.get("url") + homepage = data.get("url") or None if repository and repository["url"] == homepage: homepage = None data.update( @@ -529,6 +536,8 @@ def _parse_export(self): class PlatformJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.PLATFORM_JSON + def parse(self, contents): data = json.loads(contents) if "frameworks" in data: @@ -543,6 +552,8 @@ def _parse_frameworks(raw): class PackageJsonManifestParser(BaseManifestParser): + manifest_type = ManifestFileType.PACKAGE_JSON + def parse(self, contents): data = json.loads(contents) data = self._parse_system(data) diff --git a/platformio/package/pack.py b/platformio/package/pack.py new file mode 100644 index 0000000000..c58b84a36d --- /dev/null +++ b/platformio/package/pack.py @@ -0,0 +1,92 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tarfile +import tempfile + +from platformio import fs +from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory +from platformio.package.manifest.schema import ManifestSchema +from platformio.unpacker import FileUnpacker + + +class PackagePacker(object): + EXCLUDE_DEFAULT = ["._*", ".DS_Store", ".git", ".hg", ".svn", ".pio"] + INCLUDE_DEFAULT = ManifestFileType.items().values() + + def __init__(self, package): + self.package = package + + def pack(self, dst=None): + tmp_dir = tempfile.mkdtemp() + try: + src = self.package + + # if zip/tar.gz -> unpack to tmp dir + if not os.path.isdir(src): + with FileUnpacker(src) as fu: + assert fu.unpack(tmp_dir, silent=True) + src = tmp_dir + + manifest = self.load_manifest(src) + filename = "{name}{system}-{version}.tar.gz".format( + name=manifest["name"], + system="-" + manifest["system"][0] if "system" in manifest else "", + version=manifest["version"], + ) + + if not dst: + dst = os.path.join(os.getcwd(), filename) + elif os.path.isdir(dst): + dst = os.path.join(dst, filename) + + return self._create_tarball( + src, + dst, + include=manifest.get("export", {}).get("include"), + exclude=manifest.get("export", {}).get("exclude"), + ) + finally: + shutil.rmtree(tmp_dir) + + @staticmethod + def load_manifest(src): + mp = ManifestParserFactory.new_from_dir(src) + return ManifestSchema().load_manifest(mp.as_dict()) + + def _create_tarball(self, src, dst, include=None, exclude=None): + # remap root + if ( + include + and len(include) == 1 + and os.path.isdir(os.path.join(src, include[0])) + ): + src = os.path.join(src, include[0]) + include = None + + src_filters = self.compute_src_filters(include, exclude) + with tarfile.open(dst, "w:gz") as tar: + for f in fs.match_src_files(src, src_filters, followlinks=False): + tar.add(os.path.join(src, f), f) + return dst + + def compute_src_filters(self, include, exclude): + result = ["+<%s>" % p for p in include or ["*"]] + result += ["-<%s>" % p for p in exclude or []] + result += ["-<%s>" % p for p in self.EXCLUDE_DEFAULT] + # automatically include manifests + result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT] + return result diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 980b43db39..7fce466dc1 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -73,6 +73,7 @@ def is_bad_link(self, item, base): ).startswith(base) def extract_item(self, item, dest_dir): + dest_dir = self.resolve_path(dest_dir) bad_conds = [ self.is_bad_path(item.name, dest_dir), self.is_link(item) and self.is_bad_link(item, dest_dir), @@ -137,10 +138,13 @@ def __exit__(self, *args): if self._unpacker: self._unpacker.close() - def unpack(self, dest_dir=".", with_progress=True, check_unpacked=True): + def unpack( + self, dest_dir=".", with_progress=True, check_unpacked=True, silent=False + ): assert self._unpacker - if not with_progress: - click.echo("Unpacking...") + if not with_progress or silent: + if not silent: + click.echo("Unpacking...") for item in self._unpacker.get_items(): self._unpacker.extract_item(item, dest_dir) else: diff --git a/tests/test_pkgmanifest.py b/tests/package/test_manifest.py similarity index 100% rename from tests/test_pkgmanifest.py rename to tests/package/test_manifest.py diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py new file mode 100644 index 0000000000..9156521387 --- /dev/null +++ b/tests/package/test_pack.py @@ -0,0 +1,116 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import tarfile + +import pytest + +from platformio import fs +from platformio.package.exception import UnknownManifestError +from platformio.package.pack import PackagePacker + + +def test_base(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join("main.cpp").write("#include ") + p = PackagePacker(str(pkg_dir)) + # test missed manifest + with pytest.raises(UnknownManifestError): + p.pack() + # minimal package + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') + pkg_dir.mkdir("include").join("main.h").write("#ifndef") + with fs.cd(str(pkg_dir)): + p.pack() + with tarfile.open(os.path.join(str(pkg_dir), "foo-1.0.0.tar.gz"), "r:gz") as tar: + assert set(tar.getnames()) == set( + ["include/main.h", "library.json", "main.cpp"] + ) + + +def test_filters(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + src_dir = pkg_dir.mkdir("src") + src_dir.join("main.cpp").write("#include ") + src_dir.mkdir("util").join("helpers.cpp").write("void") + pkg_dir.mkdir("include").join("main.h").write("#ifndef") + test_dir = pkg_dir.mkdir("tests") + test_dir.join("test_1.h").write("") + test_dir.join("test_2.h").write("") + + # test include with remap of root + pkg_dir.join("library.json").write( + json.dumps(dict(name="bar", version="1.2.3", export={"include": "src"})) + ) + p = PackagePacker(str(pkg_dir)) + dst = os.path.join(str(pkg_dir), "tarball.tar.gz") + p.pack(dst) + with tarfile.open(dst, "r:gz") as tar: + assert set(tar.getnames()) == set(["util/helpers.cpp", "main.cpp"]) + + # test include "src" and "include" + pkg_dir.join("library.json").write( + json.dumps( + dict(name="bar", version="1.2.3", export={"include": ["src", "include"]}) + ) + ) + p = PackagePacker(str(pkg_dir)) + dst = os.path.join(str(pkg_dir), "tarball.tar.gz") + p.pack(dst) + with tarfile.open(dst, "r:gz") as tar: + assert set(tar.getnames()) == set( + ["include/main.h", "library.json", "src/main.cpp", "src/util/helpers.cpp"] + ) + + # test include & exclude + pkg_dir.join("library.json").write( + json.dumps( + dict( + name="bar", + version="1.2.3", + export={"include": ["src", "include"], "exclude": ["*/*.h"]}, + ) + ) + ) + p = PackagePacker(str(pkg_dir)) + dst = os.path.join(str(pkg_dir), "tarball.tar.gz") + p.pack(dst) + with tarfile.open(dst, "r:gz") as tar: + assert set(tar.getnames()) == set( + ["library.json", "src/main.cpp", "src/util/helpers.cpp"] + ) + + +def test_symlinks(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + src_dir = pkg_dir.mkdir("src") + src_dir.join("main.cpp").write("#include ") + pkg_dir.mkdir("include").join("main.h").write("#ifndef") + src_dir.join("main.h").mksymlinkto(os.path.join("..", "include", "main.h")) + pkg_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}') + tarball = pkg_dir.join("bar.tar.gz") + with tarfile.open(str(tarball), "w:gz") as tar: + for item in pkg_dir.listdir(): + tar.add(str(item), str(item.relto(pkg_dir))) + + p = PackagePacker(str(tarball)) + assert p.pack(str(pkg_dir)).endswith("bar-2.0.0.tar.gz") + with tarfile.open(os.path.join(str(pkg_dir), "bar-2.0.0.tar.gz"), "r:gz") as tar: + assert set(tar.getnames()) == set( + ["include/main.h", "library.json", "src/main.cpp", "src/main.h"] + ) + m = tar.getmember("src/main.h") + assert m.issym() From ec82fc82a2e7f766383e8f092851012f8f24ddcb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Jan 2020 22:55:56 +0200 Subject: [PATCH 078/142] Support packages with nested folders and with a custom "root" --- platformio/package/exception.py | 6 ++++- platformio/package/pack.py | 29 +++++++++++++++++++++- tests/package/test_pack.py | 44 ++++++++++++++++++++++++++------- tests/test_examples.py | 3 +++ 4 files changed, 71 insertions(+), 11 deletions(-) diff --git a/platformio/package/exception.py b/platformio/package/exception.py index 3927505d6c..7804e5197e 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -15,7 +15,11 @@ from platformio.exception import PlatformioException -class ManifestException(PlatformioException): +class PackageException(PlatformioException): + pass + + +class ManifestException(PackageException): pass diff --git a/platformio/package/pack.py b/platformio/package/pack.py index c58b84a36d..5ed5749a82 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -18,6 +18,7 @@ import tempfile from platformio import fs +from platformio.package.exception import PackageException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema from platformio.unpacker import FileUnpacker @@ -27,8 +28,9 @@ class PackagePacker(object): EXCLUDE_DEFAULT = ["._*", ".DS_Store", ".git", ".hg", ".svn", ".pio"] INCLUDE_DEFAULT = ManifestFileType.items().values() - def __init__(self, package): + def __init__(self, package, manifest_uri=None): self.package = package + self.manifest_uri = manifest_uri def pack(self, dst=None): tmp_dir = tempfile.mkdtemp() @@ -41,6 +43,8 @@ def pack(self, dst=None): assert fu.unpack(tmp_dir, silent=True) src = tmp_dir + src = self.find_source_root(src) + manifest = self.load_manifest(src) filename = "{name}{system}-{version}.tar.gz".format( name=manifest["name"], @@ -67,6 +71,29 @@ def load_manifest(src): mp = ManifestParserFactory.new_from_dir(src) return ManifestSchema().load_manifest(mp.as_dict()) + def find_source_root(self, src): + if self.manifest_uri: + mp = ( + ManifestParserFactory.new_from_file(self.manifest_uri[5:]) + if self.manifest_uri.startswith("file:") + else ManifestParserFactory.new_from_url(self.manifest_uri) + ) + manifest = ManifestSchema().load_manifest(mp.as_dict()) + include = manifest.get("export", {}).get("include", []) + if len(include) == 1: + if not os.path.isdir(os.path.join(src, include[0])): + raise PackageException( + "Non existing `include` directory `%s` in a package" + % include[0] + ) + return os.path.join(src, include[0]) + + for root, _, __ in os.walk(src): + if ManifestFileType.from_dir(root): + return root + + return src + def _create_tarball(self, src, dst, include=None, exclude=None): # remap root if ( diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py index 9156521387..a0af01723c 100644 --- a/tests/package/test_pack.py +++ b/tests/package/test_pack.py @@ -56,9 +56,7 @@ def test_filters(tmpdir_factory): json.dumps(dict(name="bar", version="1.2.3", export={"include": "src"})) ) p = PackagePacker(str(pkg_dir)) - dst = os.path.join(str(pkg_dir), "tarball.tar.gz") - p.pack(dst) - with tarfile.open(dst, "r:gz") as tar: + with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: assert set(tar.getnames()) == set(["util/helpers.cpp", "main.cpp"]) # test include "src" and "include" @@ -68,9 +66,7 @@ def test_filters(tmpdir_factory): ) ) p = PackagePacker(str(pkg_dir)) - dst = os.path.join(str(pkg_dir), "tarball.tar.gz") - p.pack(dst) - with tarfile.open(dst, "r:gz") as tar: + with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: assert set(tar.getnames()) == set( ["include/main.h", "library.json", "src/main.cpp", "src/util/helpers.cpp"] ) @@ -86,9 +82,7 @@ def test_filters(tmpdir_factory): ) ) p = PackagePacker(str(pkg_dir)) - dst = os.path.join(str(pkg_dir), "tarball.tar.gz") - p.pack(dst) - with tarfile.open(dst, "r:gz") as tar: + with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: assert set(tar.getnames()) == set( ["library.json", "src/main.cpp", "src/util/helpers.cpp"] ) @@ -114,3 +108,35 @@ def test_symlinks(tmpdir_factory): ) m = tar.getmember("src/main.h") assert m.issym() + + +def test_source_root(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + root_dir = pkg_dir.mkdir("root") + src_dir = root_dir.mkdir("src") + src_dir.join("main.cpp").write("#include ") + root_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}') + p = PackagePacker(str(pkg_dir)) + with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: + assert set(tar.getnames()) == set(["library.json", "src/main.cpp"]) + + +def test_manifest_uri(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + root_dir = pkg_dir.mkdir("root") + src_dir = root_dir.mkdir("src") + src_dir.join("main.cpp").write("#include ") + root_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') + bar_dir = root_dir.mkdir("library").mkdir("bar") + bar_dir.join("library.json").write('{"name": "bar", "version": "2.0.0"}') + bar_dir.mkdir("include").join("bar.h").write("") + + manifest_path = pkg_dir.join("remote_library.json") + manifest_path.write( + '{"name": "bar", "version": "3.0.0", "export": {"include": "root/library/bar"}}' + ) + + p = PackagePacker(str(pkg_dir), manifest_uri="file:%s" % manifest_path) + p.pack(str(pkg_dir)) + with tarfile.open(os.path.join(str(pkg_dir), "bar-2.0.0.tar.gz"), "r:gz") as tar: + assert set(tar.getnames()) == set(["library.json", "include/bar.h"]) diff --git a/tests/test_examples.py b/tests/test_examples.py index eac514693f..b0d0dfcbf8 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -20,6 +20,7 @@ import pytest from platformio import util +from platformio.compat import PY2 from platformio.managers.platform import PlatformFactory, PlatformManager from platformio.project.config import ProjectConfig @@ -53,6 +54,8 @@ def pytest_generate_tests(metafunc): for root, _, files in walk(examples_dir): if "platformio.ini" not in files or ".skiptest" in files: continue + if "zephyr-" in root and PY2: + continue group = basename(root) if "-" in group: group = group.split("-", 1)[0] From 6020faf970bfd5380b469d007a2a34edea916aea Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 4 Jan 2020 11:38:13 +0200 Subject: [PATCH 079/142] Docs: Add "v" suffix to SparkFun RISC-V board ids --- docs | 2 +- tests/package/test_pack.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs b/docs index 3622e35819..4752b52323 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3622e3581951a4b193f7ce8f3aa5e05f59da29b5 +Subproject commit 4752b523231a11e4cf7c0b8ecf2ff035f775ae4a diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py index a0af01723c..2cc84254bd 100644 --- a/tests/package/test_pack.py +++ b/tests/package/test_pack.py @@ -19,6 +19,7 @@ import pytest from platformio import fs +from platformio.compat import WINDOWS from platformio.package.exception import UnknownManifestError from platformio.package.pack import PackagePacker @@ -89,6 +90,9 @@ def test_filters(tmpdir_factory): def test_symlinks(tmpdir_factory): + # Windows does not support symbolic links + if WINDOWS: + return pkg_dir = tmpdir_factory.mktemp("package") src_dir = pkg_dir.mkdir("src") src_dir.join("main.cpp").write("#include ") From 915b9145f6bd6bd4e2b41d3e92a0950335d816e4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 4 Jan 2020 11:46:25 +0200 Subject: [PATCH 080/142] Filter boards by ID+Name --- platformio/commands/boards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 335d1b3424..7d15df2f04 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -32,7 +32,7 @@ def cli(query, installed, json_output): # pylint: disable=R0912 grpboards = {} for board in _get_boards(installed): - if query and query.lower() not in json.dumps(board).lower(): + if query and not any(query.lower() in board[k] for k in ("id", "name")): continue if board["platform"] not in grpboards: grpboards[board["platform"]] = [] From 178080fd12d573524d492e6c311e2868554610b2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 4 Jan 2020 13:53:08 +0200 Subject: [PATCH 081/142] Improve boards search --- docs | 2 +- platformio/commands/boards.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 4752b52323..f7b734d766 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4752b523231a11e4cf7c0b8ecf2ff035f775ae4a +Subproject commit f7b734d7665095746a36d5332a835d9c4a78a6a0 diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 7d15df2f04..e7e2ecd6bb 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -32,7 +32,10 @@ def cli(query, installed, json_output): # pylint: disable=R0912 grpboards = {} for board in _get_boards(installed): - if query and not any(query.lower() in board[k] for k in ("id", "name")): + if query and not any( + query.lower() in str(board.get(k, "")).lower() + for k in ("id", "name", "mcu", "vendor", "platform", "frameworks") + ): continue if board["platform"] not in grpboards: grpboards[board["platform"]] = [] From 5ab34436ec8d50715e753b0f9389ee40878829e2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 4 Jan 2020 23:48:06 +0200 Subject: [PATCH 082/142] Cleanup package file name when packing --- platformio/package/pack.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 5ed5749a82..c7288f5de8 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import re import shutil import tarfile import tempfile @@ -46,10 +47,14 @@ def pack(self, dst=None): src = self.find_source_root(src) manifest = self.load_manifest(src) - filename = "{name}{system}-{version}.tar.gz".format( - name=manifest["name"], - system="-" + manifest["system"][0] if "system" in manifest else "", - version=manifest["version"], + filename = re.sub( + r"[^\da-zA-Z\-\._]+", + "", + "{name}{system}-{version}.tar.gz".format( + name=manifest["name"], + system="-" + manifest["system"][0] if "system" in manifest else "", + version=manifest["version"], + ), ) if not dst: From 60139035d8d2f1b84722edf9f86a1c1390841639 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 5 Jan 2020 18:29:19 +0200 Subject: [PATCH 083/142] Include hidden files by default in a package --- platformio/package/pack.py | 11 +++++++++-- tests/package/test_pack.py | 5 ++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index c7288f5de8..1e18c55ae6 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -26,7 +26,14 @@ class PackagePacker(object): - EXCLUDE_DEFAULT = ["._*", ".DS_Store", ".git", ".hg", ".svn", ".pio"] + EXCLUDE_DEFAULT = [ + "._*", + ".DS_Store", + ".git", + ".hg", + ".svn", + ".pio", + ] INCLUDE_DEFAULT = ManifestFileType.items().values() def __init__(self, package, manifest_uri=None): @@ -116,7 +123,7 @@ def _create_tarball(self, src, dst, include=None, exclude=None): return dst def compute_src_filters(self, include, exclude): - result = ["+<%s>" % p for p in include or ["*"]] + result = ["+<%s>" % p for p in include or ["*", ".*"]] result += ["-<%s>" % p for p in exclude or []] result += ["-<%s>" % p for p in self.EXCLUDE_DEFAULT] # automatically include manifests diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py index 2cc84254bd..95b435700f 100644 --- a/tests/package/test_pack.py +++ b/tests/package/test_pack.py @@ -26,6 +26,9 @@ def test_base(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join(".git").mkdir().join("file").write("") + pkg_dir.join(".gitignore").write("tests") + pkg_dir.join("._ignored").write("") pkg_dir.join("main.cpp").write("#include ") p = PackagePacker(str(pkg_dir)) # test missed manifest @@ -38,7 +41,7 @@ def test_base(tmpdir_factory): p.pack() with tarfile.open(os.path.join(str(pkg_dir), "foo-1.0.0.tar.gz"), "r:gz") as tar: assert set(tar.getnames()) == set( - ["include/main.h", "library.json", "main.cpp"] + [".gitignore", "include/main.h", "library.json", "main.cpp"] ) From 9c32ff278c2478a0f6c5dd95234f655c864bd09b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 Jan 2020 23:32:55 +0200 Subject: [PATCH 084/142] Sync docs --- docs | 2 +- tests/package/test_manifest.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs b/docs index f7b734d766..d0288c129b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f7b734d7665095746a36d5332a835d9c4a78a6a0 +Subproject commit d0288c129b146d4587f8a9ed880a17ed1d556607 diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index de59e0db4d..0197e66292 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -521,6 +521,7 @@ def test_package_json_schema(): def test_parser_from_dir(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join("package.json").write('{"name": "package.json"}') pkg_dir.join("library.json").write('{"name": "library.json"}') pkg_dir.join("library.properties").write("name=library.properties") From 0f296e7e37551fc54a33c656c6f74c57f19c544f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 Jan 2020 21:28:19 +0200 Subject: [PATCH 085/142] Skip broken example definitions in package manifest --- platformio/package/manifest/parser.py | 4 ++-- tests/package/test_manifest.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 71f63c7939..64d52cbed2 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -186,8 +186,8 @@ def parse_examples(self, data): or not isinstance(examples, list) or not all(isinstance(v, dict) for v in examples) ): - examples = None - if not examples and self.package_dir: + data["examples"] = None + if not data["examples"] and self.package_dir: data["examples"] = self.parse_examples_from_dir(self.package_dir) if "examples" in data and not data["examples"]: del data["examples"] diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 0197e66292..90e2f6c7be 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -39,6 +39,7 @@ def test_library_json_parser(): "build": { "flags": ["-DHELLO"] }, + "examples": ["examples/*/*.pde"], "customField": "Custom Value" } """ From 7e41841a74917cc49e147e9fea965ea6bf51151a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 11 Jan 2020 15:23:36 +0200 Subject: [PATCH 086/142] Allow to call Downloader API in silent mode --- platformio/downloader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/downloader.py b/platformio/downloader.py index 57c712ed24..0a9b981f72 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -74,13 +74,14 @@ def get_size(self): return -1 return int(self._request.headers["content-length"]) - def start(self, with_progress=True): + def start(self, with_progress=True, silent=False): label = "Downloading" itercontent = self._request.iter_content(chunk_size=self.CHUNK_SIZE) f = open(self._destination, "wb") try: if not with_progress or self.get_size() == -1: - click.echo("%s..." % label) + if not silent: + click.echo("%s..." % label) for chunk in itercontent: if chunk: f.write(chunk) From 8d02e8b8f715c5fff65e6783dfada784b32d6267 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 Jan 2020 11:55:58 +0200 Subject: [PATCH 087/142] Docs: Update FAQ for installing Python --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index d0288c129b..fd744480ba 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d0288c129b146d4587f8a9ed880a17ed1d556607 +Subproject commit fd744480ba83741bae8386d5451aa119397c52a9 From 90b80083e8e6a4c6be4ceaf66dd96d1c654bf2ac Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 22 Jan 2020 12:58:16 +0200 Subject: [PATCH 088/142] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index fd744480ba..86845f2eeb 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit fd744480ba83741bae8386d5451aa119397c52a9 +Subproject commit 86845f2eebdc90def4e8fa296a459c74fa4f4953 From 5bdec19f318d14d2b9fc30aed5b8481907a2d40c Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Wed, 22 Jan 2020 20:41:42 +0200 Subject: [PATCH 089/142] Add new debug_build_flags option (#3355) This will allow users to override default debug flags for example in cases when the final binary built in debug mode is too large to be loaded on target --- platformio/builder/tools/piomisc.py | 15 ++++++-- platformio/project/options.py | 10 +++++ tests/test_builder.py | 58 +++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 5ef48d20ef..87b3bf124f 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -314,17 +314,26 @@ def _cleanup_debug_flags(scope): if scope not in env: return unflags = ["-Os", "-g"] - for level in [0, 1, 2]: + for level in [0, 1, 2, 3]: for flag in ("O", "g", "ggdb"): unflags.append("-%s%d" % (flag, level)) env[scope] = [f for f in env.get(scope, []) if f not in unflags] env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"]) - debug_flags = ["-Og", "-g2", "-ggdb2"] for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"): _cleanup_debug_flags(scope) - env.Append(**{scope: debug_flags}) + + debug_flags = env.ParseFlags(env.GetProjectOption("debug_build_flags")) + env.MergeFlags(debug_flags) + optimization_flags = [f for f in debug_flags.get("CCFLAGS", []) if f.startswith( + ("-O", "-g"))] + + if optimization_flags: + env.AppendUnique( + ASFLAGS=optimization_flags, + LINKFLAGS=optimization_flags + ) def ConfigureTestTarget(env): diff --git a/platformio/project/options.py b/platformio/project/options.py index 7d4c5a71b6..f8a50a76ef 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -600,6 +600,16 @@ def ConfigEnvOption(*args, **kwargs): name="debug_tool", description="A name of debugging tool", ), + ConfigEnvOption( + group="debug", + name="debug_build_flags", + description=( + "Custom debug flags/options for preprocessing, compilation, " + "assembly, and linking processes" + ), + multiple=True, + default=["-Og", "-g2", "-ggdb2"], + ), ConfigEnvOption( group="debug", name="debug_init_break", diff --git a/tests/test_builder.py b/tests/test_builder.py index 9dec2e0fba..6e73da0a49 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -112,3 +112,61 @@ def test_build_unflags(clirunner, validate_cliresult, tmpdir): assert "-DTMP_MACRO1" not in build_output assert "-Os" not in build_output assert str(tmpdir) not in build_output + + +def test_debug_default_build_flags(clirunner, validate_cliresult, tmpdir): + tmpdir.join("platformio.ini").write( + """ +[env:native] +platform = native +build_type = debug +""" + ) + + tmpdir.mkdir("src").join("main.c").write( + """ +int main() { +} +""" + ) + + result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) + validate_cliresult(result) + build_output = result.output[result.output.find("Scanning dependencies...") :] + for line in build_output.split("\n"): + if line.startswith("gcc"): + assert all(line.count(flag) == 1 for flag in ("-Og", "-g2", "-ggdb2")) + assert all(line.count("-%s%d" % (flag, level)) == 0 for flag in ( + "O", "g", "ggdb") for level in (0, 1, 3)) + assert "-Os" not in line + + +def test_debug_custom_build_flags(clirunner, validate_cliresult, tmpdir): + custom_debug_build_flags = ("-O3", "-g3", "-ggdb3") + + tmpdir.join("platformio.ini").write( + """ +[env:native] +platform = native +build_type = debug +debug_build_flags = %s + """ + % " ".join(custom_debug_build_flags) + ) + + tmpdir.mkdir("src").join("main.c").write( + """ +int main() { +} +""" + ) + + result = clirunner.invoke(cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) + validate_cliresult(result) + build_output = result.output[result.output.find("Scanning dependencies...") :] + for line in build_output.split("\n"): + if line.startswith("gcc"): + assert all(line.count(f) == 1 for f in custom_debug_build_flags) + assert all(line.count("-%s%d" % (flag, level)) == 0 for flag in ( + "O", "g", "ggdb") for level in (0, 1, 2)) + assert all("-O%s" % optimization not in line for optimization in ("g", "s")) From 17f9d572076c5b97348896dcb69a8b38912f820e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 22 Jan 2020 22:20:24 +0200 Subject: [PATCH 090/142] Control debug flags and optimization level with a new "debug_build_flags" option --- HISTORY.rst | 1 + docs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1ede584dd1..f42ac6be59 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,7 @@ PlatformIO Core 4.0 - Project Manager - Project Configuration UI for `"platformio.ini" `__ +* Control debug flags and optimization level with a new `debug_build_flags `__ option * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Updated SCons tool to 3.1.2 * Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) diff --git a/docs b/docs index 86845f2eeb..1d1246090c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 86845f2eebdc90def4e8fa296a459c74fa4f4953 +Subproject commit 1d1246090c74dc7ea74650827d22af8315c9ef00 From 5ac1e9454fe3cd0d78263a8886f65e08233d14e7 Mon Sep 17 00:00:00 2001 From: Max Prokhorov Date: Thu, 23 Jan 2020 13:56:08 +0300 Subject: [PATCH 091/142] pio-test: pass --verbose to the run command context (#3338) * pio-test: pass --verbose to the run command context * restore old output behavior --- platformio/commands/test/command.py | 4 +++- platformio/commands/test/processor.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index cb1c8117d5..b57b1d5918 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -107,7 +107,8 @@ def cli( # pylint: disable=redefined-builtin raise exception.TestDirNotExists(test_dir) test_names = get_test_names(test_dir) - click.echo("Verbose mode can be enabled via `-v, --verbose` option") + if not verbose: + click.echo("Verbose mode can be enabled via `-v, --verbose` option") click.secho("Collected %d items" % len(test_names), bold=True) results = [] @@ -159,6 +160,7 @@ def cli( # pylint: disable=redefined-builtin monitor_rts=monitor_rts, monitor_dtr=monitor_dtr, verbose=verbose, + silent=not verbose, ), ) result = { diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index ed4f935cbb..7d9704b8f1 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -119,7 +119,8 @@ def build_or_upload(self, target): cmd_run, project_dir=self.options["project_dir"], upload_port=self.options["upload_port"], - silent=not self.options["verbose"], + verbose=self.options["verbose"], + silent=self.options["silent"], environment=[self.env_name], disable_auto_clean="nobuild" in target, target=target, From 46a9c1b6b2267c32bd23b52813769b58401deccf Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 23 Jan 2020 12:57:54 +0200 Subject: [PATCH 092/142] Add initial support for PVS-Studio check tool (#3357) * Add initial support for PVS-Studio check tool * Enable all available PVS-Studio analyzers by default * Add tests for PVS-Studio check tool * Improve handling check tool extra flags that contain colon symbol --- platformio/commands/check/tools/__init__.py | 3 + platformio/commands/check/tools/base.py | 18 +- platformio/commands/check/tools/clangtidy.py | 2 +- platformio/commands/check/tools/cppcheck.py | 4 +- platformio/commands/check/tools/pvsstudio.py | 232 +++++++++++++++++++ platformio/managers/core.py | 1 + platformio/project/options.py | 2 +- tests/commands/test_check.py | 45 +++- 8 files changed, 294 insertions(+), 13 deletions(-) create mode 100644 platformio/commands/check/tools/pvsstudio.py diff --git a/platformio/commands/check/tools/__init__.py b/platformio/commands/check/tools/__init__.py index 4c8a5e72e5..9c4b1b7e51 100644 --- a/platformio/commands/check/tools/__init__.py +++ b/platformio/commands/check/tools/__init__.py @@ -15,6 +15,7 @@ from platformio import exception from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool from platformio.commands.check.tools.cppcheck import CppcheckCheckTool +from platformio.commands.check.tools.pvsstudio import PvsStudioCheckTool class CheckToolFactory(object): @@ -25,6 +26,8 @@ def new(tool, project_dir, config, envname, options): cls = CppcheckCheckTool elif tool == "clangtidy": cls = ClangtidyCheckTool + elif tool == "pvs-studio": + cls = PvsStudioCheckTool else: raise exception.PlatformioException("Unknown check tool `%s`" % tool) return cls(project_dir, config, envname, options) diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index 4c381a0575..a2cf59404d 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -27,10 +27,13 @@ def __init__(self, project_dir, config, envname, options): self.config = config self.envname = envname self.options = options - self.cpp_defines = [] - self.cpp_flags = [] + self.cc_flags = [] + self.cxx_flags = [] self.cpp_includes = [] - + self.cpp_defines = [] + self.toolchain_defines = [] + self.cc_path = None + self.cxx_path = None self._defects = [] self._on_defect_callback = None self._bad_input = False @@ -53,16 +56,19 @@ def _load_cpp_data(self, project_dir, envname): data = load_project_ide_data(project_dir, envname) if not data: return - self.cpp_flags = data.get("cxx_flags", "").split(" ") + self.cc_flags = data.get("cc_flags", "").split(" ") + self.cxx_flags = data.get("cxx_flags", "").split(" ") self.cpp_includes = data.get("includes", []) self.cpp_defines = data.get("defines", []) - self.cpp_defines.extend(self._get_toolchain_defines(data.get("cc_path"))) + self.cc_path = data.get("cc_path") + self.cxx_path = data.get("cxx_path") + self.toolchain_defines = self._get_toolchain_defines(self.cc_path) def get_flags(self, tool): result = [] flags = self.options.get("flags") or [] for flag in flags: - if ":" not in flag: + if ":" not in flag or flag.startswith("-"): result.extend([f for f in flag.split(" ") if f]) elif flag.startswith("%s:" % tool): result.extend([f for f in flag.split(":", 1)[1].split(" ") if f]) diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index b7845f8b41..4efc00a9f2 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -61,7 +61,7 @@ def configure_command(self): cmd.extend(self.get_project_target_files()) cmd.append("--") - cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(["-D%s" % d for d in self.cpp_defines + self.toolchain_defines]) cmd.extend(["-I%s" % inc for inc in self.cpp_includes]) return cmd diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 3d92bc565f..840568bd52 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -112,12 +112,12 @@ def configure_command(self): cmd.append("--language=c++") if not self.is_flag_set("--std", flags): - for f in self.cpp_flags: + for f in self.cxx_flags + self.cc_flags: if "-std" in f: # Standards with GNU extensions are not allowed cmd.append("-" + f.replace("gnu", "c")) - cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.extend(["-D%s" % d for d in self.cpp_defines + self.toolchain_defines]) cmd.extend(flags) cmd.append("--file-list=%s" % self._generate_src_file()) diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py new file mode 100644 index 0000000000..bce4ae1496 --- /dev/null +++ b/platformio/commands/check/tools/pvsstudio.py @@ -0,0 +1,232 @@ +# Copyright (c) 2020-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +from xml.etree.ElementTree import fromstring + +import click + +from platformio import proc, util +from platformio.commands.check.defect import DefectItem +from platformio.commands.check.tools.base import CheckToolBase +from platformio.managers.core import get_core_package_dir + + +class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-attributes + def __init__(self, *args, **kwargs): + self._tmp_dir = tempfile.mkdtemp(prefix="piocheck") + self._tmp_preprocessed_file = self._generate_tmp_file_path() + ".i" + self._tmp_output_file = self._generate_tmp_file_path() + ".pvs" + self._tmp_cfg_file = self._generate_tmp_file_path() + ".cfg" + self._tmp_cmd_file = self._generate_tmp_file_path() + ".cmd" + self.tool_path = os.path.join( + get_core_package_dir("tool-pvs-studio"), + "x64" if "windows" in util.get_systype() else "bin", + "pvs-studio", + ) + super(PvsStudioCheckTool, self).__init__(*args, **kwargs) + + with open(self._tmp_cfg_file, "w") as fp: + fp.write( + "exclude-path = " + + self.config.get_optional_dir("packages").replace("\\", "/") + ) + + with open(self._tmp_cmd_file, "w") as fp: + fp.write( + " ".join( + ['-I"%s"' % inc.replace("\\", "/") for inc in self.cpp_includes] + ) + ) + + def _process_defects(self, defects): + for defect in defects: + if not isinstance(defect, DefectItem): + return + if defect.severity not in self.options["severity"]: + return + self._defects.append(defect) + if self._on_defect_callback: + self._on_defect_callback(defect) + + def _demangle_report(self, output_file): + converter_tool = os.path.join( + get_core_package_dir("tool-pvs-studio"), + "HtmlGenerator" + if "windows" in util.get_systype() + else os.path.join("bin", "plog-converter"), + ) + + cmd = ( + converter_tool, + "-t", + "xml", + output_file, + "-m", + "cwe", + "-m", + "misra", + "-a", + # Enable all possible analyzers and defect levels + "GA:1,2,3;64:1,2,3;OP:1,2,3;CS:1,2,3;MISRA:1,2,3", + "--cerr", + ) + + result = proc.exec_command(cmd) + if result["returncode"] != 0: + click.echo(result["err"]) + self._bad_input = True + + return result["err"] + + def parse_defects(self, output_file): + defects = [] + + report = self._demangle_report(output_file) + if not report: + self._bad_input = True + return [] + + try: + defects_data = fromstring(report) + except: # pylint: disable=bare-except + click.echo("Error: Couldn't decode generated report!") + self._bad_input = True + return [] + + for table in defects_data.iter("PVS-Studio_Analysis_Log"): + message = table.find("Message").text + category = table.find("ErrorType").text + line = table.find("Line").text + file_ = table.find("File").text + defect_id = table.find("ErrorCode").text + cwe = table.find("CWECode") + cwe_id = None + if cwe is not None: + cwe_id = cwe.text.lower().replace("cwe-", "") + misra = table.find("MISRA") + if misra is not None: + message += " [%s]" % misra.text + + severity = DefectItem.SEVERITY_LOW + if category == "error": + severity = DefectItem.SEVERITY_HIGH + elif category == "warning": + severity = DefectItem.SEVERITY_MEDIUM + + defects.append( + DefectItem( + severity, category, message, file_, line, id=defect_id, cwe=cwe_id + ) + ) + + return defects + + def configure_command(self, src_file): # pylint: disable=arguments-differ + if os.path.isfile(self._tmp_output_file): + os.remove(self._tmp_output_file) + + if not os.path.isfile(self._tmp_preprocessed_file): + click.echo( + "Error: Missing preprocessed file '%s'" % (self._tmp_preprocessed_file) + ) + return "" + + cmd = [ + self.tool_path, + "--skip-cl-exe", + "yes", + "--language", + "C" if src_file.endswith(".c") else "C++", + "--preprocessor", + "gcc", + "--cfg", + self._tmp_cfg_file, + "--source-file", + src_file, + "--i-file", + self._tmp_preprocessed_file, + "--output-file", + self._tmp_output_file, + ] + + flags = self.get_flags("pvs-studio") + if not self.is_flag_set("--platform", flags): + cmd.append("--platform=arm") + cmd.extend(flags) + + return cmd + + def _generate_tmp_file_path(self): + # pylint: disable=protected-access + return os.path.join(self._tmp_dir, next(tempfile._get_candidate_names())) + + def _prepare_preprocessed_file(self, src_file): + flags = self.cxx_flags + compiler = self.cxx_path + if src_file.endswith(".c"): + flags = self.cc_flags + compiler = self.cc_path + + cmd = [compiler, src_file, "-E", "-o", self._tmp_preprocessed_file] + cmd.extend([f for f in flags if f]) + cmd.extend(["-D%s" % d for d in self.cpp_defines]) + cmd.append('@"%s"' % self._tmp_cmd_file) + + result = proc.exec_command(" ".join(cmd), shell=True) + if result["returncode"] != 0: + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + click.echo(result["err"]) + self._bad_input = True + + def clean_up(self): + temp_files = ( + self._tmp_output_file, + self._tmp_preprocessed_file, + self._tmp_cfg_file, + self._tmp_cmd_file, + ) + for f in temp_files: + if os.path.isfile(f): + os.remove(f) + + def check(self, on_defect_callback=None): + self._on_defect_callback = on_defect_callback + src_files = [ + f for f in self.get_project_target_files() if not f.endswith((".h", ".hpp")) + ] + + for src_file in src_files: + self._prepare_preprocessed_file(src_file) + cmd = self.configure_command(src_file) + if self.options.get("verbose"): + click.echo(" ".join(cmd)) + if not cmd: + self._bad_input = True + continue + + result = proc.exec_command(cmd) + # pylint: disable=unsupported-membership-test + if result["returncode"] != 0 or "License was not entered" in result["err"]: + self._bad_input = True + click.echo(result["err"]) + continue + + self._process_defects(self.parse_defects(self._tmp_output_file)) + + self.clean_up() + + return self._bad_input diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 6166d8fe63..1d09474b28 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -31,6 +31,7 @@ "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", "tool-cppcheck": "~1.189.0", "tool-clangtidy": "^1.80000.0", + "tool-pvs-studio": "~7.5.0", } PIOPLUS_AUTO_UPDATES_MAX = 100 diff --git a/platformio/project/options.py b/platformio/project/options.py index f8a50a76ef..d38ed566db 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -531,7 +531,7 @@ def ConfigEnvOption(*args, **kwargs): group="check", name="check_tool", description="A list of check tools used for analysis", - type=click.Choice(["cppcheck", "clangtidy"]), + type=click.Choice(["cppcheck", "clangtidy", "pvs-studio"]), multiple=True, default=["cppcheck"], ), diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 64e4602e65..2078acf186 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -239,21 +239,30 @@ def test_check_success_if_no_errors(clirunner, tmpdir): def test_check_individual_flags_passed(clirunner, tmpdir): - config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" - config += "\ncheck_flags = cppcheck: --std=c++11 \n\tclangtidy: --fix-errors" + config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy, pvs-studio" + config += """\ncheck_flags = + cppcheck: --std=c++11 + clangtidy: --fix-errors + pvs-studio: --analysis-mode=4 +""" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) - clang_flags_found = cppcheck_flags_found = False + clang_flags_found = cppcheck_flags_found = pvs_flags_found = False for l in result.output.split("\n"): if "--fix" in l and "clang-tidy" in l and "--std=c++11" not in l: clang_flags_found = True elif "--std=c++11" in l and "cppcheck" in l and "--fix" not in l: cppcheck_flags_found = True + elif ( + "--analysis-mode=4" in l and "pvs-studio" in l.lower() and "--fix" not in l + ): + pvs_flags_found = True assert clang_flags_found assert cppcheck_flags_found + assert pvs_flags_found def test_check_cppcheck_misra_addon(clirunner, check_dir): @@ -344,3 +353,33 @@ def test_check_fails_on_defects_only_on_specified_level(clirunner, tmpdir): assert high_result.exit_code == 0 assert low_result.exit_code != 0 + + +def test_check_pvs_studio_free_license(clirunner, tmpdir): + config = """ +[env:test] +platform = teensy +board = teensy35 +framework = arduino +check_tool = pvs-studio +""" + code = ( + """// This is an open source non-commercial project. Dear PVS-Studio, please check it. +// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com +""" + + TEST_CODE + ) + + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.c").write(code) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high", "-v"] + ) + + errors, warnings, style = count_defects(result.output) + + assert result.exit_code != 0 + assert errors != 0 + assert warnings != 0 + assert style == 0 From fad5d1d74474098d4b1a41b3b03851ec8dc3bf74 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jan 2020 14:06:36 +0200 Subject: [PATCH 093/142] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 1d1246090c..efba65abac 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1d1246090c74dc7ea74650827d22af8315c9ef00 +Subproject commit efba65abac3a301f430dc803be9dc6edd6d6a092 From d4c42bd5464bc8455bf902cd5d7a09a82a0e3e68 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jan 2020 20:51:34 +0200 Subject: [PATCH 094/142] Added support for `PVS-Studio static code analyzer --- HISTORY.rst | 6 +++++- docs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f42ac6be59..6fd1c6a473 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,7 +6,7 @@ Release Notes PlatformIO Core 4.0 ------------------- -4.1.1 (2019-??-??) +4.1.1 (2020-??-??) ~~~~~~~~~~~~~~~~~~ * `PlatformIO Home 3.0 `__: @@ -14,6 +14,10 @@ PlatformIO Core 4.0 - Project Manager - Project Configuration UI for `"platformio.ini" `__ +* `PIO Check `__ – automated code analysis without hassle: + + - Added support for `PVS-Studio `__ static code analyzer + * Control debug flags and optimization level with a new `debug_build_flags `__ option * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Updated SCons tool to 3.1.2 diff --git a/docs b/docs index efba65abac..7b12319095 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit efba65abac3a301f430dc803be9dc6edd6d6a092 +Subproject commit 7b12319095eeb8c06eee06556c4749e1c43838fb From dbeffe426f1e7c3132a65115b4cb8b0692423af4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jan 2020 22:48:22 +0200 Subject: [PATCH 095/142] Update docs for CI --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7b12319095..663e0af38e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7b12319095eeb8c06eee06556c4749e1c43838fb +Subproject commit 663e0af38e2af8688b7c0fd8f29fe3e313172758 From 8fce660afa9d7c58059f1460c7d34cab2ecb17c2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jan 2020 14:47:23 +0200 Subject: [PATCH 096/142] Add notice about VID:PID pairs // Resolve #3349 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 663e0af38e..a430de3311 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 663e0af38e2af8688b7c0fd8f29fe3e313172758 +Subproject commit a430de3311d1a033d2a5a6d6ac1c2b9597c6ffa3 From 2de46f545fa02271bda74164306d8d8544a8dbd9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jan 2020 19:47:20 +0200 Subject: [PATCH 097/142] Formatter --- platformio/builder/tools/piomisc.py | 10 ++++------ tests/test_builder.py | 14 ++++++++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 87b3bf124f..ba019ec301 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -326,14 +326,12 @@ def _cleanup_debug_flags(scope): debug_flags = env.ParseFlags(env.GetProjectOption("debug_build_flags")) env.MergeFlags(debug_flags) - optimization_flags = [f for f in debug_flags.get("CCFLAGS", []) if f.startswith( - ("-O", "-g"))] + optimization_flags = [ + f for f in debug_flags.get("CCFLAGS", []) if f.startswith(("-O", "-g")) + ] if optimization_flags: - env.AppendUnique( - ASFLAGS=optimization_flags, - LINKFLAGS=optimization_flags - ) + env.AppendUnique(ASFLAGS=optimization_flags, LINKFLAGS=optimization_flags) def ConfigureTestTarget(env): diff --git a/tests/test_builder.py b/tests/test_builder.py index 6e73da0a49..f220e50c3f 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -136,8 +136,11 @@ def test_debug_default_build_flags(clirunner, validate_cliresult, tmpdir): for line in build_output.split("\n"): if line.startswith("gcc"): assert all(line.count(flag) == 1 for flag in ("-Og", "-g2", "-ggdb2")) - assert all(line.count("-%s%d" % (flag, level)) == 0 for flag in ( - "O", "g", "ggdb") for level in (0, 1, 3)) + assert all( + line.count("-%s%d" % (flag, level)) == 0 + for flag in ("O", "g", "ggdb") + for level in (0, 1, 3) + ) assert "-Os" not in line @@ -167,6 +170,9 @@ def test_debug_custom_build_flags(clirunner, validate_cliresult, tmpdir): for line in build_output.split("\n"): if line.startswith("gcc"): assert all(line.count(f) == 1 for f in custom_debug_build_flags) - assert all(line.count("-%s%d" % (flag, level)) == 0 for flag in ( - "O", "g", "ggdb") for level in (0, 1, 2)) + assert all( + line.count("-%s%d" % (flag, level)) == 0 + for flag in ("O", "g", "ggdb") + for level in (0, 1, 2) + ) assert all("-O%s" % optimization not in line for optimization in ("g", "s")) From 3b092f28c3a1347a4f719fc89db86a01917aee2c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jan 2020 19:47:47 +0200 Subject: [PATCH 098/142] Added support for "pythonPackages" in `platform.json` --- HISTORY.rst | 1 + docs | 2 +- platformio/managers/platform.py | 50 +++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6fd1c6a473..3b29065083 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ PlatformIO Core 4.0 - Added support for `PVS-Studio `__ static code analyzer * Control debug flags and optimization level with a new `debug_build_flags `__ option +* Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Updated SCons tool to 3.1.2 * Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) diff --git a/docs b/docs index a430de3311..cfc5eed060 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a430de3311d1a033d2a5a6d6ac1c2b9597c6ffa3 +Subproject commit cfc5eed060900a8b5510c23e269b773019e3882f diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 5d61829ee3..983cab5059 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -17,6 +17,7 @@ import base64 import os import re +import subprocess import sys from os.path import basename, dirname, isdir, isfile, join @@ -86,6 +87,8 @@ def install( # don't cleanup packages or install them after update # we check packages for updates in def update() if after_update: + p.install_python_packages() + p.on_installed() return True p.install_packages( @@ -95,6 +98,8 @@ def install( silent=silent, force=force, ) + p.install_python_packages() + p.on_installed() return self.cleanup_packages(list(p.packages)) def uninstall(self, package, requirements=None, after_update=False): @@ -109,6 +114,8 @@ def uninstall(self, package, requirements=None, after_update=False): p = PlatformFactory.newPlatform(pkg_dir) BasePkgManager.uninstall(self, pkg_dir, requirements) + p.uninstall_python_packages() + p.on_uninstalled() # don't cleanup packages or install them after update # we check packages for updates in def update() @@ -594,6 +601,10 @@ def packages(self): packages[name].update({"version": version.strip(), "optional": False}) return packages + @property + def python_packages(self): + return self._manifest.get("pythonPackages") + def get_dir(self): return dirname(self.manifest_path) @@ -699,6 +710,45 @@ def get_lib_storages(self): return [dict(name=name, path=path) for path, name in storages.items()] + def on_installed(self): + pass + + def on_uninstalled(self): + pass + + def install_python_packages(self): + if not self.python_packages: + return None + click.echo( + "Installing Python packages: %s" + % ", ".join(list(self.python_packages.keys())), + ) + args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"] + for name, requirements in self.python_packages.items(): + if any(c in requirements for c in ("<", ">", "=")): + args.append("%s%s" % (name, requirements)) + else: + args.append("%s==%s" % (name, requirements)) + try: + return subprocess.call(args) == 0 + except Exception as e: # pylint: disable=broad-except + click.secho( + "Could not install Python packages -> %s" % e, fg="red", err=True + ) + + def uninstall_python_packages(self): + if not self.python_packages: + return + click.echo("Uninstalling Python packages") + args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"] + args.extend(list(self.python_packages.keys())) + try: + subprocess.call(args) == 0 + except Exception as e: # pylint: disable=broad-except + click.secho( + "Could not install Python packages -> %s" % e, fg="red", err=True + ) + class PlatformBoardConfig(object): def __init__(self, manifest_path): From a62bc3846e2041f61cdd71c281c5bad2c22baafb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jan 2020 19:50:55 +0200 Subject: [PATCH 099/142] Update IPs when checking for Internet --- platformio/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/util.py b/platformio/util.py index 04bb723e6e..640295d252 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -368,7 +368,8 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): PING_INTERNET_IPS = [ "192.30.253.113", # github.com - "78.46.220.20", # dl.platformio.org + "78.46.220.20", # api.platformio.org + "3.124.149.187", # registry.platformio.org ] From 20334772b5e75623bbddffaf03ee5ee61bd217d9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jan 2020 20:20:56 +0200 Subject: [PATCH 100/142] Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option // Resolve #3345 --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/platform.py | 11 +++++++++-- platformio/managers/platform.py | 4 ++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3b29065083..637fd75e00 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ PlatformIO Core 4.0 - Added support for `PVS-Studio `__ static code analyzer * Control debug flags and optimization level with a new `debug_build_flags `__ option +* Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install `__ command (`issue #3345 `_) * Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Updated SCons tool to 3.1.2 diff --git a/docs b/docs index cfc5eed060..7b3bc1494a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit cfc5eed060900a8b5510c23e269b773019e3882f +Subproject commit 7b3bc1494aed26fd2e33940bc44dd81b9231a115 diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index c094b560cd..d4ff49309a 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -299,14 +299,20 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches @click.option("--with-package", multiple=True) @click.option("--without-package", multiple=True) @click.option("--skip-default-package", is_flag=True) +@click.option("--with-all-packages", is_flag=True) @click.option( "-f", "--force", is_flag=True, help="Reinstall/redownload dev/platform and its packages if exist", ) -def platform_install( - platforms, with_package, without_package, skip_default_package, force +def platform_install( # pylint: disable=too-many-arguments + platforms, + with_package, + without_package, + skip_default_package, + with_all_packages, + force, ): pm = PlatformManager() for platform in platforms: @@ -315,6 +321,7 @@ def platform_install( with_packages=with_package, without_packages=without_package, skip_default_package=skip_default_package, + with_all_packages=with_all_packages, force=force, ): click.secho( diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 983cab5059..8f23f1bda4 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -74,6 +74,7 @@ def install( with_packages=None, without_packages=None, skip_default_package=False, + with_all_packages=False, after_update=False, silent=False, force=False, @@ -84,6 +85,9 @@ def install( ) p = PlatformFactory.newPlatform(platform_dir) + if with_all_packages: + with_packages = list(p.packages.keys()) + # don't cleanup packages or install them after update # we check packages for updates in def update() if after_update: From 04ca6621f10047aa6feea1096c099952b101a00b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jan 2020 20:43:05 +0200 Subject: [PATCH 101/142] Verify downloaded package checksum with native Python's hashlib and support all OS --- platformio/downloader.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/platformio/downloader.py b/platformio/downloader.py index 0a9b981f72..ff687db66b 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib from email.utils import parsedate_tz from math import ceil from os.path import getsize, join @@ -27,7 +28,6 @@ FDSizeMismatch, FDUnrecognizedStatusCode, ) -from platformio.proc import exec_command class FileDownloader(object): @@ -103,25 +103,19 @@ def verify(self, sha1=None): _dlsize = getsize(self._destination) if self.get_size() != -1 and _dlsize != self.get_size(): raise FDSizeMismatch(_dlsize, self._fname, self.get_size()) - if not sha1: return None - dlsha1 = None - try: - result = exec_command(["sha1sum", self._destination]) - dlsha1 = result["out"] - except (OSError, ValueError): - try: - result = exec_command(["shasum", "-a", "1", self._destination]) - dlsha1 = result["out"] - except (OSError, ValueError): - pass - if not dlsha1: - return None - dlsha1 = dlsha1[1:41] if dlsha1.startswith("\\") else dlsha1[:40] - if sha1.lower() != dlsha1.lower(): - raise FDSHASumMismatch(dlsha1, self._fname, sha1) + checksum = hashlib.sha1() + with open(self._destination, "rb") as fp: + while True: + chunk = fp.read(1024 * 64) + if not chunk: + break + checksum.update(chunk) + + if sha1.lower() != checksum.hexdigest().lower(): + raise FDSHASumMismatch(checksum.hexdigest(), self._fname, sha1) return True def _preserve_filemtime(self, lmdate): From 253e4b13b5b2cf9e2e05683156b38905232ed53e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jan 2020 21:40:12 +0200 Subject: [PATCH 102/142] Disable buffering file calculating checksum for downloaded file --- platformio/downloader.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/platformio/downloader.py b/platformio/downloader.py index ff687db66b..aebdb20a9a 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -13,10 +13,11 @@ # limitations under the License. import hashlib +import io +import math +import sys from email.utils import parsedate_tz -from math import ceil from os.path import getsize, join -from sys import version_info from time import mktime import click @@ -32,7 +33,7 @@ class FileDownloader(object): - CHUNK_SIZE = 1024 + CHUNK_SIZE = 1024 * 8 def __init__(self, url, dest_dir=None): self._request = None @@ -41,7 +42,7 @@ def __init__(self, url, dest_dir=None): url, stream=True, headers=util.get_request_defheaders(), - verify=version_info >= (2, 7, 9), + verify=sys.version_info >= (2, 7, 9), ) if self._request.status_code != 200: raise FDUnrecognizedStatusCode(self._request.status_code, url) @@ -86,7 +87,7 @@ def start(self, with_progress=True, silent=False): if chunk: f.write(chunk) else: - chunks = int(ceil(self.get_size() / float(self.CHUNK_SIZE))) + chunks = int(math.ceil(self.get_size() / float(self.CHUNK_SIZE))) with click.progressbar(length=chunks, label=label) as pb: for _ in pb: f.write(next(itercontent)) @@ -107,9 +108,9 @@ def verify(self, sha1=None): return None checksum = hashlib.sha1() - with open(self._destination, "rb") as fp: + with io.open(self._destination, "rb", buffering=0) as fp: while True: - chunk = fp.read(1024 * 64) + chunk = fp.read(self.CHUNK_SIZE) if not chunk: break checksum.update(chunk) From decf4823678d9bccdcdb1d1c6c75d3482d1ddf12 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jan 2020 22:05:24 +0200 Subject: [PATCH 103/142] Use io.DEFAULT_BUFFER_SIZE as default chunk size --- platformio/downloader.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/platformio/downloader.py b/platformio/downloader.py index aebdb20a9a..c6ecda807c 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -32,9 +32,6 @@ class FileDownloader(object): - - CHUNK_SIZE = 1024 * 8 - def __init__(self, url, dest_dir=None): self._request = None # make connection @@ -77,7 +74,7 @@ def get_size(self): def start(self, with_progress=True, silent=False): label = "Downloading" - itercontent = self._request.iter_content(chunk_size=self.CHUNK_SIZE) + itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) f = open(self._destination, "wb") try: if not with_progress or self.get_size() == -1: @@ -87,7 +84,7 @@ def start(self, with_progress=True, silent=False): if chunk: f.write(chunk) else: - chunks = int(math.ceil(self.get_size() / float(self.CHUNK_SIZE))) + chunks = int(math.ceil(self.get_size() / float(io.DEFAULT_BUFFER_SIZE))) with click.progressbar(length=chunks, label=label) as pb: for _ in pb: f.write(next(itercontent)) @@ -110,7 +107,7 @@ def verify(self, sha1=None): checksum = hashlib.sha1() with io.open(self._destination, "rb", buffering=0) as fp: while True: - chunk = fp.read(self.CHUNK_SIZE) + chunk = fp.read(io.DEFAULT_BUFFER_SIZE) if not chunk: break checksum.update(chunk) From ba441ca77cec2f9a0de171c9618313ab48e02ee9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jan 2020 22:15:33 +0200 Subject: [PATCH 104/142] Remove double blank lines when saving project config // Resolve #3293 --- platformio/project/config.py | 11 +++++++---- tests/test_projectconf.py | 4 +++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 739429f6a3..ff53dd565b 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -13,6 +13,7 @@ # limitations under the License. import glob +import io import json import os import re @@ -30,7 +31,8 @@ except ImportError: import configparser as ConfigParser -CONFIG_HEADER = """; PlatformIO Project Configuration File +CONFIG_HEADER = """ +; PlatformIO Project Configuration File ; ; Build options: build flags, source filter ; Upload options: custom upload port, speed and extra flags @@ -39,7 +41,6 @@ ; ; Please visit documentation for the other options and examples ; https://docs.platformio.org/page/projectconf.html - """ @@ -449,6 +450,8 @@ def save(self, path=None): if path in self._instances: del self._instances[path] with open(path or self.path, "w") as fp: - fp.write(CONFIG_HEADER) - self._parser.write(fp) + fp.write(CONFIG_HEADER.strip()) + buf = io.StringIO() + self._parser.write(buf) + fp.write("\n\n%s\n" % buf.getvalue().strip()) return True diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index b601ceceb2..c65c5737a2 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -330,9 +330,11 @@ def test_update_and_save(tmpdir_factory): ] config.save() + contents = tmpdir.join("platformio.ini").read() + assert contents[-4:] == "yes\n" lines = [ line.strip() - for line in tmpdir.join("platformio.ini").readlines() + for line in contents.split("\n") if line.strip() and not line.startswith((";", "#")) ] assert lines == [ From e5aa71e4e1ebbbd6e43156f10bacb79a37911bc3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jan 2020 15:47:45 +0200 Subject: [PATCH 105/142] Fix config saving when PY2 is used --- platformio/project/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index ff53dd565b..f277dc946e 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -451,7 +451,7 @@ def save(self, path=None): del self._instances[path] with open(path or self.path, "w") as fp: fp.write(CONFIG_HEADER.strip()) - buf = io.StringIO() + buf = io.BytesIO() if PY2 else io.StringIO() self._parser.write(buf) fp.write("\n\n%s\n" % buf.getvalue().strip()) return True From f9de23b16fee3bcc813aa1206d0b1fb3e1a24ceb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jan 2020 15:48:14 +0200 Subject: [PATCH 106/142] Bump version to 4.1.1b8 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6492ea68ba..243b111f79 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1b7") +VERSION = (4, 1, "1b8") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 880d5bb8b001d39ed339a0853a40f46413fb8085 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jan 2020 20:47:10 +0200 Subject: [PATCH 107/142] Fix project saving --- platformio/project/config.py | 14 ++++++++------ tests/test_projectconf.py | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index f277dc946e..aadfbbb69d 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -13,7 +13,6 @@ # limitations under the License. import glob -import io import json import os import re @@ -449,9 +448,12 @@ def save(self, path=None): path = path or self.path if path in self._instances: del self._instances[path] - with open(path or self.path, "w") as fp: - fp.write(CONFIG_HEADER.strip()) - buf = io.BytesIO() if PY2 else io.StringIO() - self._parser.write(buf) - fp.write("\n\n%s\n" % buf.getvalue().strip()) + with open(path or self.path, "w+") as fp: + fp.write(CONFIG_HEADER.strip() + "\n\n") + self._parser.write(fp) + fp.seek(0) + contents = fp.read() + fp.seek(0) + fp.truncate() + fp.write(contents.strip() + "\n") return True diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index c65c5737a2..2070a43503 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -330,6 +330,7 @@ def test_update_and_save(tmpdir_factory): ] config.save() + print(tmpdir) contents = tmpdir.join("platformio.ini").read() assert contents[-4:] == "yes\n" lines = [ From 13e9306753accc369854ef5282dba561a90fa0b3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jan 2020 20:50:26 +0200 Subject: [PATCH 108/142] Bump version to 4.1.1b9 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 243b111f79..6dccaef9e5 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1b8") +VERSION = (4, 1, "1b9") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From b805822eeaae24b3240df11e7b364a938247cc5e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jan 2020 20:51:20 +0200 Subject: [PATCH 109/142] Remove debug code --- tests/test_projectconf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 2070a43503..c65c5737a2 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -330,7 +330,6 @@ def test_update_and_save(tmpdir_factory): ] config.save() - print(tmpdir) contents = tmpdir.join("platformio.ini").read() assert contents[-4:] == "yes\n" lines = [ From 848e5259198fa0e59c742aaa55e035fde0ed75be Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 26 Jan 2020 00:31:55 +0200 Subject: [PATCH 110/142] Bumo PIO Home to 3.1.0-rc.2 --- platformio/managers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 1d09474b28..4c3ccd2ed7 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.1.0-rc.1,<3.2.0", + "contrib-piohome": ">=3.1.0-rc.2,<3.2.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.1", "tool-unity": "~1.20403.0", From 139171a79fd00a2a7b14cbbb6fb57d6c4f2a9b39 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 29 Jan 2020 18:53:33 +0200 Subject: [PATCH 111/142] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7b3bc1494a..1712ba74a2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7b3bc1494aed26fd2e33940bc44dd81b9231a115 +Subproject commit 1712ba74a29534eed900e7570015cd22c24e0628 From 607e8eb477ddebf1b0876ca6ec6d734f5e16915a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 29 Jan 2020 18:54:30 +0200 Subject: [PATCH 112/142] Improve detecting of active Internet connection // Resolve #3359 --- platformio/util.py | 16 +++++++++------- tests/test_misc.py | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index 640295d252..f9c304f9e2 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -366,10 +366,11 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): ) -PING_INTERNET_IPS = [ - "192.30.253.113", # github.com - "78.46.220.20", # api.platformio.org - "3.124.149.187", # registry.platformio.org +PING_REMOTE_HOSTS = [ + "140.82.118.3", # Github.com + "35.231.145.151", # Gitlab.com + "github.com", + "platformio.org", ] @@ -377,12 +378,13 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): def _internet_on(): timeout = 2 socket.setdefaulttimeout(timeout) - for ip in PING_INTERNET_IPS: + for host in PING_REMOTE_HOSTS: try: if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): - requests.get("http://%s" % ip, allow_redirects=False, timeout=timeout) + requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) + + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80)) return True except: # pylint: disable=bare-except pass diff --git a/tests/test_misc.py b/tests/test_misc.py index 1a1c7c3221..aee9f113e3 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -25,8 +25,8 @@ def test_platformio_cli(): def test_ping_internet_ips(): - for ip in util.PING_INTERNET_IPS: - requests.get("http://%s" % ip, allow_redirects=False, timeout=2) + for host in util.PING_REMOTE_HOSTS: + requests.get("http://%s" % host, allow_redirects=False, timeout=2) def test_api_internet_offline(without_internet, isolated_pio_home): From bc69259dd10c4357e5fc205350305b2c5cbfda12 Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 31 Jan 2020 15:10:45 +0200 Subject: [PATCH 113/142] Update tool-unity package to v2.5.0 // Resolve #3362 --- HISTORY.rst | 1 + platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 637fd75e00..9cdbec4763 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 4.0 * Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Updated SCons tool to 3.1.2 +* Updated Unity tool to 2.5.0 * Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) * Warn about broken library manifest when scanning dependencies (`issue #3268 `_) * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 4c3ccd2ed7..813cac0559 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -27,7 +27,7 @@ "contrib-piohome": ">=3.1.0-rc.2,<3.2.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.1", - "tool-unity": "~1.20403.0", + "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", "tool-cppcheck": "~1.189.0", "tool-clangtidy": "^1.80000.0", From efe8e599fdaa020f451df5ba96c3068893cbc3f1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Feb 2020 00:04:16 +0200 Subject: [PATCH 114/142] Added support for Arduino's library.properties ``depends`` field // Resolve #2781 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/piolib.py | 4 +- platformio/managers/lib.py | 27 +-------- platformio/package/manifest/parser.py | 53 ++++++++++++++++ platformio/package/manifest/schema.py | 27 +++++++++ platformio/util.py | 7 +-- tests/package/test_manifest.py | 87 +++++++++++++++++++++++++-- 8 files changed, 169 insertions(+), 39 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9cdbec4763..dcf46aa030 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,7 @@ PlatformIO Core 4.0 * Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install `__ command (`issue #3345 `_) * Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) +* Added support for Arduino's library.properties ``depends`` field (`issue #2781 `_) * Updated SCons tool to 3.1.2 * Updated Unity tool to 2.5.0 * Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) diff --git a/docs b/docs index 1712ba74a2..a5c3fb32b7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1712ba74a29534eed900e7570015cd22c24e0628 +Subproject commit a5c3fb32b7d89ef87320476769f206572b4b95ef diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 58d965e0e3..3c8746bea0 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -154,9 +154,7 @@ def version(self): @property def dependencies(self): - return LibraryManager.normalize_dependencies( - self._manifest.get("dependencies", []) - ) + return self._manifest.get("dependencies") @property def src_filter(self): diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index e85a1225d5..b006d8afca 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -23,7 +23,7 @@ import semantic_version from platformio import app, exception, util -from platformio.compat import glob_escape, string_types +from platformio.compat import glob_escape from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager from platformio.project.config import ProjectConfig @@ -61,29 +61,6 @@ def get_manifest_path(self, pkg_dir): return None - @staticmethod - def normalize_dependencies(dependencies): - if not dependencies: - return [] - items = [] - if isinstance(dependencies, dict): - if "name" in dependencies: - items.append(dependencies) - else: - for name, version in dependencies.items(): - items.append({"name": name, "version": version}) - elif isinstance(dependencies, list): - items = [d for d in dependencies if "name" in d] - for item in items: - for k in ("frameworks", "platforms"): - if k not in item or isinstance(k, list): - continue - if item[k] == "*": - del item[k] - elif isinstance(item[k], string_types): - item[k] = [i.strip() for i in item[k].split(",") if i.strip()] - return items - def max_satisfying_repo_version(self, versions, requirements=None): def _cmp_dates(datestr1, datestr2): date1 = util.parse_date(datestr1) @@ -312,7 +289,7 @@ def install( # pylint: disable=arguments-differ click.secho("Installing dependencies", fg="yellow") builtin_lib_storages = None - for filters in self.normalize_dependencies(manifest["dependencies"]): + for filters in manifest["dependencies"]: assert "name" in filters # avoid circle dependencies diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 64d52cbed2..2391a84dfb 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -19,6 +19,7 @@ import requests +from platformio import util from platformio.compat import get_class_attributes, string_types from platformio.fs import get_file_contents from platformio.package.exception import ManifestParserError, UnknownManifestError @@ -286,6 +287,8 @@ def parse(self, contents): data["platforms"] = self._parse_platforms(data["platforms"]) or None if "export" in data: data["export"] = self._parse_export(data["export"]) + if "dependencies" in data: + data["dependencies"] = self._parse_dependencies(data["dependencies"]) return data @@ -350,6 +353,26 @@ def _parse_export(raw): result[k] = raw[k] if isinstance(raw[k], list) else [raw[k]] return result + @staticmethod + def _parse_dependencies(raw): + if isinstance(raw, dict): + if "name" in raw: # compatibility with dep as dict + return [raw] + return [dict(name=name, version=version) for name, version in raw.items()] + if isinstance(raw, list): + for i, dependency in enumerate(raw): + assert isinstance(dependency, dict) + for k, v in dependency.items(): + if k not in ("platforms", "frameworks", "authors"): + continue + if "*" in v: + del raw[i][k] + raw[i][k] = util.items_to_list(v) + return raw + raise ManifestParserError( + "Invalid dependencies format, should be list or dictionary" + ) + class ModuleJsonManifestParser(BaseManifestParser): manifest_type = ManifestFileType.MODULE_JSON @@ -408,6 +431,8 @@ def parse(self, contents): if "author" in data: data["authors"] = self._parse_authors(data) del data["author"] + if "depends" in data: + data["dependencies"] = self._parse_dependencies(data["depends"]) return data @staticmethod @@ -534,6 +559,26 @@ def _parse_export(self): result["include"] = [include] return result + @staticmethod + def _parse_dependencies(raw): + result = [] + for item in raw.split(","): + item = item.strip() + if not item: + continue + if item.endswith(")") and "(" in item: + name, version = item.split("(") + result.append( + dict( + name=name.strip(), + version=version[:-1].strip(), + frameworks=["arduino"], + ) + ) + else: + result.append(dict(name=item, frameworks=["arduino"])) + return result + class PlatformJsonManifestParser(BaseManifestParser): manifest_type = ManifestFileType.PLATFORM_JSON @@ -542,6 +587,8 @@ def parse(self, contents): data = json.loads(contents) if "frameworks" in data: data["frameworks"] = self._parse_frameworks(data["frameworks"]) + if "packages" in data: + data["dependencies"] = self._parse_dependencies(data["packages"]) return data @staticmethod @@ -550,6 +597,12 @@ def _parse_frameworks(raw): return None return [name.lower() for name in raw.keys()] + @staticmethod + def _parse_dependencies(raw): + return [ + dict(name=name, version=opts.get("version")) for name, opts in raw.items() + ] + class PackageJsonManifestParser(BaseManifestParser): manifest_type = ManifestFileType.PACKAGE_JSON diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index fe16413b9a..11885f34d1 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -102,6 +102,32 @@ class RepositorySchema(StrictSchema): branch = fields.Str(validate=validate.Length(min=1, max=50)) +class DependencySchema(StrictSchema): + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + version = fields.Str(validate=validate.Length(min=1, max=100)) + authors = StrictListField(fields.Str(validate=validate.Length(min=1, max=50))) + platforms = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + frameworks = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r"^([a-z\d\-_]+|\*)$", error="Only [a-z0-9-_*] chars are allowed" + ), + ] + ) + ) + + class ExportSchema(BaseSchema): include = StrictListField(fields.Str) exclude = StrictListField(fields.Str) @@ -133,6 +159,7 @@ class ManifestSchema(BaseSchema): homepage = fields.Url(validate=validate.Length(min=1, max=255)) license = fields.Str(validate=validate.Length(min=1, max=255)) repository = fields.Nested(RepositorySchema) + dependencies = fields.Nested(DependencySchema, many=True) # library.json export = fields.Nested(ExportSchema) diff --git a/platformio/util.py b/platformio/util.py index f9c304f9e2..d35ce6a20a 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -383,7 +383,6 @@ def _internet_on(): if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80)) return True except: # pylint: disable=bare-except @@ -403,9 +402,9 @@ def pepver_to_semver(pepver): def items_to_list(items): - if not isinstance(items, list): - items = [i.strip() for i in items.split(",")] - return [i.lower() for i in items if i] + if isinstance(items, list): + return items + return [i.strip() for i in items.split(",") if i.strip()] def items_in_list(needle, haystack): diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 90e2f6c7be..8ca16e2c23 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -40,6 +40,11 @@ def test_library_json_parser(): "flags": ["-DHELLO"] }, "examples": ["examples/*/*.pde"], + "dependencies": { + "deps1": "1.2.0", + "deps2": "https://github.com/username/package.git", + "@owner/deps3": "^2.1.3" + }, "customField": "Custom Value" } """ @@ -57,6 +62,11 @@ def test_library_json_parser(): "keywords": ["kw1", "kw2", "kw3"], "homepage": "http://old.url.format", "build": {"flags": ["-DHELLO"]}, + "dependencies": [ + {"name": "deps1", "version": "1.2.0"}, + {"name": "deps2", "version": "https://github.com/username/package.git"}, + {"name": "@owner/deps3", "version": "^2.1.3"}, + ], "customField": "Custom Value", }, ) @@ -68,7 +78,12 @@ def test_library_json_parser(): "platforms": "atmelavr", "export": { "exclude": "audio_samples" - } + }, + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino, espidf"}, + {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} + ] } """ mp = parser.LibraryJsonManifestParser(contents) @@ -79,9 +94,26 @@ def test_library_json_parser(): "frameworks": ["arduino"], "export": {"exclude": ["audio_samples"]}, "platforms": ["atmelavr"], + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + { + "name": "@owner/deps2", + "version": "1.0.0", + "frameworks": ["arduino", "espidf"], + }, + { + "name": "deps3", + "version": "1.0.0", + "platforms": ["ststm32", "sifive"], + }, + ], }, ) + # broken dependencies + with pytest.raises(parser.ManifestParserError): + mp = parser.LibraryJsonManifestParser({"dependencies": ["deps1", "deps2"]}) + def test_module_json_parser(): contents = """ @@ -137,6 +169,7 @@ def test_library_properties_parser(): author=SomeAuthor sentence=This is Arduino library customField=Custom Value +depends=First Library (=2.0.0), Second Library (>=1.2.0), Third """ mp = parser.LibraryPropertiesManifestParser(contents) assert not jsondiff.diff( @@ -154,6 +187,20 @@ def test_library_properties_parser(): "authors": [{"email": "info@author.com", "name": "SomeAuthor"}], "keywords": ["uncategorized"], "customField": "Custom Value", + "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third", + "dependencies": [ + { + "name": "First Library", + "version": "=2.0.0", + "frameworks": ["arduino"], + }, + { + "name": "Second Library", + "version": ">=1.2.0", + "frameworks": ["arduino"], + }, + {"name": "Third", "frameworks": ["arduino"]}, + ], }, ) @@ -244,6 +291,11 @@ def test_library_json_schema(): "base": "examples/JsonHttpClient", "files": ["JsonHttpClient.ino"] } + ], + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino"}, + {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} ] } """ @@ -289,6 +341,15 @@ def test_library_json_schema(): "files": ["JsonHttpClient.ino"], }, ], + "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, + {"name": "@owner/deps2", "version": "1.0.0", "frameworks": ["arduino"]}, + { + "name": "deps3", + "version": "1.0.0", + "platforms": ["ststm32", "sifive"], + }, + ], }, ) @@ -304,6 +365,7 @@ def test_library_properties_schema(): category=Display url=https://github.com/olikraus/u8glib architectures=avr,sam +depends=First Library (=2.0.0), Second Library (>=1.2.0), Third """ raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES @@ -333,6 +395,19 @@ def test_library_properties_schema(): ], "keywords": ["display"], "name": "U8glib", + "dependencies": [ + { + "name": "First Library", + "version": "=2.0.0", + "frameworks": ["arduino"], + }, + { + "name": "Second Library", + "version": ">=1.2.0", + "frameworks": ["arduino"], + }, + {"name": "Third", "frameworks": ["arduino"]}, + ], }, ) @@ -436,11 +511,6 @@ def test_platform_json_schema(): "optional": true, "version": "~4.2.0" }, - "framework-simba": { - "type": "framework", - "optional": true, - "version": ">=7.0.0" - }, "tool-avrdude": { "type": "uploader", "optional": true, @@ -475,6 +545,11 @@ def test_platform_json_schema(): }, "frameworks": sorted(["arduino", "simba"]), "version": "1.15.0", + "dependencies": [ + {"name": "toolchain-atmelavr", "version": "~1.50400.0"}, + {"name": "framework-arduinoavr", "version": "~4.2.0"}, + {"name": "tool-avrdude", "version": "~1.60300.0"}, + ], }, ) From b83121a951039641bd54c912e1ead6727d3a63f9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Feb 2020 14:34:40 +0200 Subject: [PATCH 115/142] Fix package parser test --- tests/package/test_manifest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 8ca16e2c23..74d0aa3ed8 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -48,9 +48,10 @@ def test_library_json_parser(): "customField": "Custom Value" } """ - mp = parser.LibraryJsonManifestParser(contents) + raw_data = parser.LibraryJsonManifestParser(contents).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( - mp.as_dict(), + raw_data, { "name": "TestPackage", "platforms": ["atmelavr", "espressif8266"], @@ -63,12 +64,12 @@ def test_library_json_parser(): "homepage": "http://old.url.format", "build": {"flags": ["-DHELLO"]}, "dependencies": [ + {"name": "@owner/deps3", "version": "^2.1.3"}, {"name": "deps1", "version": "1.2.0"}, {"name": "deps2", "version": "https://github.com/username/package.git"}, - {"name": "@owner/deps3", "version": "^2.1.3"}, ], "customField": "Custom Value", - }, + } ) contents = """ From 365c3eaf4b35c0e23bcc72e8531f8903f844e037 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Feb 2020 15:31:32 +0200 Subject: [PATCH 116/142] Fixed an issue when invalid CLI command does not return non-zero exit code --- HISTORY.rst | 1 + platformio/__main__.py | 5 +++-- tests/package/test_manifest.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index dcf46aa030..14a32bfcce 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -32,6 +32,7 @@ PlatformIO Core 4.0 * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ * Fixed an issue with LDF when header files not found if "libdeps_dir" is within a subdirectory of "lib_extra_dirs" (`issue #3311 `_) * Fixed an issue "Import of non-existent variable 'projenv''" when development platform does not call "env.BuildProgram()" (`issue #3315 `_) +* Fixed an issue when invalid CLI command does not return non-zero exit code 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__main__.py b/platformio/__main__.py index 043befbda0..6679d52ef4 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -100,8 +100,9 @@ def main(argv=None): try: configure() cli() # pylint: disable=no-value-for-parameter - except SystemExit: - pass + except SystemExit as e: + if e.code and str(e.code).isdigit(): + exit_code = int(e.code) except Exception as e: # pylint: disable=broad-except if not isinstance(e, exception.ReturnErrorCode): maintenance.on_platformio_exception(e) diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 74d0aa3ed8..54b16c2fce 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -69,7 +69,7 @@ def test_library_json_parser(): {"name": "deps2", "version": "https://github.com/username/package.git"}, ], "customField": "Custom Value", - } + }, ) contents = """ From 390f1935d682cd942c97f6d7211055987d7e983f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Feb 2020 15:43:39 +0200 Subject: [PATCH 117/142] Fix parsing dependency in a legacy format --- platformio/package/manifest/parser.py | 6 ++-- tests/package/test_manifest.py | 52 ++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 2391a84dfb..e32617fcc7 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -355,9 +355,11 @@ def _parse_export(raw): @staticmethod def _parse_dependencies(raw): + # compatibility with legacy dependency format + if isinstance(raw, dict) and "name" in raw: + raw = [raw] + if isinstance(raw, dict): - if "name" in raw: # compatibility with dep as dict - return [raw] return [dict(name=name, version=version) for name, version in raw.items()] if isinstance(raw, list): for i, dependency in enumerate(raw): diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 54b16c2fce..22f9691d2c 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -87,21 +87,22 @@ def test_library_json_parser(): ] } """ - mp = parser.LibraryJsonManifestParser(contents) + raw_data = parser.LibraryJsonManifestParser(contents).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( - mp.as_dict(), + raw_data, { "keywords": ["sound", "audio", "music", "sd", "card", "playback"], "frameworks": ["arduino"], "export": {"exclude": ["audio_samples"]}, "platforms": ["atmelavr"], "dependencies": [ - {"name": "deps1", "version": "1.0.0"}, { "name": "@owner/deps2", "version": "1.0.0", "frameworks": ["arduino", "espidf"], }, + {"name": "deps1", "version": "1.0.0"}, { "name": "deps3", "version": "1.0.0", @@ -113,7 +114,7 @@ def test_library_json_parser(): # broken dependencies with pytest.raises(parser.ManifestParserError): - mp = parser.LibraryJsonManifestParser({"dependencies": ["deps1", "deps2"]}) + parser.LibraryJsonManifestParser({"dependencies": ["deps1", "deps2"]}) def test_module_json_parser(): @@ -172,9 +173,10 @@ def test_library_properties_parser(): customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third """ - mp = parser.LibraryPropertiesManifestParser(contents) + raw_data = parser.LibraryPropertiesManifestParser(contents).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( - mp.as_dict(), + raw_data, { "name": "TestPackage", "version": "1.2.3", @@ -303,6 +305,7 @@ def test_library_json_schema(): raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_JSON ).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) data = ManifestSchema().load_manifest(raw_data) @@ -343,8 +346,8 @@ def test_library_json_schema(): }, ], "dependencies": [ - {"name": "deps1", "version": "1.0.0"}, {"name": "@owner/deps2", "version": "1.0.0", "frameworks": ["arduino"]}, + {"name": "deps1", "version": "1.0.0"}, { "name": "deps3", "version": "1.0.0", @@ -354,6 +357,36 @@ def test_library_json_schema(): }, ) + # legacy dependencies format + contents = """ +{ + "name": "DallasTemperature", + "version": "3.8.0", + "dependencies": + { + "name": "OneWire", + "authors": "Paul Stoffregen", + "frameworks": "arduino" + } +} +""" + raw_data = parser.LibraryJsonManifestParser(contents).as_dict() + data = ManifestSchema().load_manifest(raw_data) + assert not jsondiff.diff( + data, + { + "name": "DallasTemperature", + "version": "3.8.0", + "dependencies": [ + { + "name": "OneWire", + "authors": ["Paul Stoffregen"], + "frameworks": ["arduino"], + } + ], + }, + ) + def test_library_properties_schema(): contents = """ @@ -371,6 +404,7 @@ def test_library_properties_schema(): raw_data = parser.ManifestParserFactory.new( contents, parser.ManifestFileType.LIBRARY_PROPERTIES ).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) data = ManifestSchema().load_manifest(raw_data) @@ -524,6 +558,8 @@ def test_platform_json_schema(): contents, parser.ManifestFileType.PLATFORM_JSON ).as_dict() raw_data["frameworks"] = sorted(raw_data["frameworks"]) + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + data = ManifestSchema().load_manifest(raw_data) assert not jsondiff.diff( @@ -547,9 +583,9 @@ def test_platform_json_schema(): "frameworks": sorted(["arduino", "simba"]), "version": "1.15.0", "dependencies": [ - {"name": "toolchain-atmelavr", "version": "~1.50400.0"}, {"name": "framework-arduinoavr", "version": "~4.2.0"}, {"name": "tool-avrdude", "version": "~1.60300.0"}, + {"name": "toolchain-atmelavr", "version": "~1.50400.0"}, ], }, ) From ee2e4896d291c3cada1780c2946d9c4d1c2a5ca3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Feb 2020 18:15:44 +0200 Subject: [PATCH 118/142] Fixed an issue when Project Inspector crashes when flash use > 100% // Resolve #3368 --- HISTORY.rst | 1 + platformio/builder/main.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 14a32bfcce..e785be5894 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -33,6 +33,7 @@ PlatformIO Core 4.0 * Fixed an issue with LDF when header files not found if "libdeps_dir" is within a subdirectory of "lib_extra_dirs" (`issue #3311 `_) * Fixed an issue "Import of non-existent variable 'projenv''" when development platform does not call "env.BuildProgram()" (`issue #3315 `_) * Fixed an issue when invalid CLI command does not return non-zero exit code +* Fixed an issue when Project Inspector crashes when flash use > 100% (`issue #3368 `_) 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/main.py b/platformio/builder/main.py index a49017a03c..b162362299 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -161,7 +161,9 @@ ############################################################################## # Checking program size -if env.get("SIZETOOL") and "nobuild" not in COMMAND_LINE_TARGETS: +if env.get("SIZETOOL") and not ( + set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS) +): env.Depends(["upload", "program"], "checkprogsize") # Replace platform's "size" target with our _new_targets = [t for t in DEFAULT_TARGETS if str(t) != "size"] From 09b3df552085806b2090243075e949307052da61 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Feb 2020 22:25:06 +0200 Subject: [PATCH 119/142] Fixed a "UnicodeDecodeError" when listing built-in libraries on macOS with Python 2.7 // Resolve #3370 --- HISTORY.rst | 1 + platformio/commands/home/rpc/handlers/os.py | 8 +++----- platformio/fs.py | 4 ++-- platformio/package/manifest/parser.py | 6 +++--- platformio/project/config.py | 3 ++- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e785be5894..5df3557d07 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -34,6 +34,7 @@ PlatformIO Core 4.0 * Fixed an issue "Import of non-existent variable 'projenv''" when development platform does not call "env.BuildProgram()" (`issue #3315 `_) * Fixed an issue when invalid CLI command does not return non-zero exit code * Fixed an issue when Project Inspector crashes when flash use > 100% (`issue #3368 `_) +* Fixed a "UnicodeDecodeError" when listing built-in libraries on macOS with Python 2.7 (`issue #3370 `_) 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 2d7ce609c6..cefb063057 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -14,7 +14,6 @@ from __future__ import absolute_import -import codecs import glob import os import shutil @@ -66,10 +65,9 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None): def request_content(self, uri, data=None, headers=None, cache_valid=None): if uri.startswith("http"): return self.fetch_content(uri, data, headers, cache_valid) - if not os.path.isfile(uri): - return None - with codecs.open(uri, encoding="utf-8") as fp: - return fp.read() + if os.path.isfile(uri): + return fs.get_file_contents(uri, encoding="utf8") + return None @staticmethod def open_url(url): diff --git a/platformio/fs.py b/platformio/fs.py index 57809ba676..ed0102cd6c 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -49,9 +49,9 @@ def get_source_dir(): return os.path.dirname(curpath) -def get_file_contents(path): +def get_file_contents(path, encoding=None): try: - with open(path) as fp: + with io.open(path, encoding=encoding) as fp: return fp.read() except UnicodeDecodeError: click.secho( diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index e32617fcc7..35a003860b 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -66,7 +66,7 @@ def new_from_file(path, remote_url=False): if not type_from_uri: raise UnknownManifestError("Unknown manifest file type %s" % path) return ManifestParserFactory.new( - get_file_contents(path), type_from_uri, remote_url + get_file_contents(path, encoding="utf8"), type_from_uri, remote_url ) @staticmethod @@ -76,7 +76,7 @@ def new_from_dir(path, remote_url=None): type_from_uri = ManifestFileType.from_uri(remote_url) if remote_url else None if type_from_uri and os.path.isfile(os.path.join(path, type_from_uri)): return ManifestParserFactory.new( - get_file_contents(os.path.join(path, type_from_uri)), + get_file_contents(os.path.join(path, type_from_uri), encoding="utf8"), type_from_uri, remote_url=remote_url, package_dir=path, @@ -88,7 +88,7 @@ def new_from_dir(path, remote_url=None): "Unknown manifest file type in %s directory" % path ) return ManifestParserFactory.new( - get_file_contents(os.path.join(path, type_from_dir)), + get_file_contents(os.path.join(path, type_from_dir), encoding="utf8"), type_from_dir, remote_url=remote_url, package_dir=path, diff --git a/platformio/project/config.py b/platformio/project/config.py index aadfbbb69d..1ee27058ce 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -13,6 +13,7 @@ # limitations under the License. import glob +import io import json import os import re @@ -448,7 +449,7 @@ def save(self, path=None): path = path or self.path if path in self._instances: del self._instances[path] - with open(path or self.path, "w+") as fp: + with io.open(path or self.path, mode="w+", encoding="utf8") as fp: fp.write(CONFIG_HEADER.strip() + "\n\n") self._parser.write(fp) fp.seek(0) From 7716fe1d1c26e0c2a15b7d6871febe88ca32aa21 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Wed, 5 Feb 2020 22:33:05 +0200 Subject: [PATCH 120/142] Autodetect monitor port for boards with specified HWIDs // Resolves #3349 (#3369) * Autodetect serial port for boards with specified hwids * Add new record to history log --- HISTORY.rst | 1 + platformio/commands/device.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 5df3557d07..1f087ab7d6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 4.0 * Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) * Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) * Added support for Arduino's library.properties ``depends`` field (`issue #2781 `_) +* Autodetect monitor port for boards with specified HWIDs (`issue #3349 `_) * Updated SCons tool to 3.1.2 * Updated Unity tool to 2.5.0 * Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 72c7ee8e4a..24e7d3cac9 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -21,6 +21,7 @@ from platformio import exception, fs, util from platformio.compat import dump_json_to_unicode +from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError @@ -189,6 +190,20 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches ports = util.get_serial_ports(filter_hwid=True) if len(ports) == 1: kwargs["port"] = ports[0]["port"] + elif "platform" in project_options and "board" in project_options: + board_hwids = get_board_hwids( + kwargs["project_dir"], + project_options["platform"], + project_options["board"], + ) + for item in ports: + for hwid in board_hwids: + hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") + if hwid_str in item["hwid"]: + kwargs["port"] = item["port"] + break + if kwargs["port"]: + break elif kwargs["port"] and (set(["*", "?", "[", "]"]) & set(kwargs["port"])): for item in util.get_serial_ports(): if fnmatch(item["port"], kwargs["port"]): @@ -254,3 +269,12 @@ def get_project_options(environment=None): else: environment = config.envs()[0] return config.items(env=environment, as_dict=True) + + +def get_board_hwids(project_dir, platform, board): + with fs.cd(project_dir): + return ( + PlatformFactory.newPlatform(platform) + .board_config(board) + .get("build.hwids", []) + ) From 00a9a2c04d3cfb6bffffd6a6cad97b92ae9cd979 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 6 Feb 2020 17:19:48 +0200 Subject: [PATCH 121/142] Generate `compilation database "compile_commands.json" // Resolve #2990 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/main.py | 8 + platformio/builder/tools/compilation_db.py | 206 +++++++++++++++++++++ platformio/builder/tools/piolib.py | 2 +- platformio/builder/tools/platformio.py | 6 +- 6 files changed, 221 insertions(+), 4 deletions(-) create mode 100644 platformio/builder/tools/compilation_db.py diff --git a/HISTORY.rst b/HISTORY.rst index 1f087ab7d6..b6f4354ab5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ PlatformIO Core 4.0 - Added support for `PVS-Studio `__ static code analyzer +* Generate `compilation database "compile_commands.json" `_ (`issue #2990 `_) * Control debug flags and optimization level with a new `debug_build_flags `__ option * Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install `__ command (`issue #3345 `_) * Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) diff --git a/docs b/docs index a5c3fb32b7..ec5a17c390 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a5c3fb32b7d89ef87320476769f206572b4b95ef +Subproject commit ec5a17c390117b91c9f9b22d385ae535f3003964 diff --git a/platformio/builder/main.py b/platformio/builder/main.py index b162362299..86adf23449 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -72,6 +72,7 @@ BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"), BUILD_SRC_DIR=join("$BUILD_DIR", "src"), BUILD_TEST_DIR=join("$BUILD_DIR", "test"), + COMPILATIONDB_PATH=join("$BUILD_DIR", "compile_commands.json"), LIBPATH=["$BUILD_DIR"], PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), @@ -134,6 +135,10 @@ elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): click.echo("Verbose mode can be enabled via `-v, --verbose` option") +# Dynamically load dependent tools +if "compiledb" in COMMAND_LINE_TARGETS: + env.Tool("compilation_db") + if not isdir(env.subst("$BUILD_DIR")): makedirs(env.subst("$BUILD_DIR")) @@ -171,6 +176,9 @@ Default(_new_targets) Default("checkprogsize") +if "compiledb" in COMMAND_LINE_TARGETS: + env.Alias("compiledb", env.CompilationDatabase("$COMPILATIONDB_PATH")) + # Print configured protocols env.AddPreAction( ["upload", "program"], diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py new file mode 100644 index 0000000000..6b77a5a410 --- /dev/null +++ b/platformio/builder/tools/compilation_db.py @@ -0,0 +1,206 @@ +# Copyright (c) 2014-present PlatformIO +# Copyright 2015 MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument, protected-access, unused-variable, import-error +# Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py + +from __future__ import absolute_import + +import itertools +import json + +import SCons + +from platformio.builder.tools.platformio import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT + +# Implements the ability for SCons to emit a compilation database for the MongoDB project. See +# http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation +# database is, and why you might want one. The only user visible entry point here is +# 'env.CompilationDatabase'. This method takes an optional 'target' to name the file that +# should hold the compilation database, otherwise, the file defaults to compile_commands.json, +# which is the name that most clang tools search for by default. + +# TODO: Is there a better way to do this than this global? Right now this exists so that the +# emitter we add can record all of the things it emits, so that the scanner for the top level +# compilation database can access the complete list, and also so that the writer has easy +# access to write all of the files. But it seems clunky. How can the emitter and the scanner +# communicate more gracefully? +__COMPILATION_DB_ENTRIES = [] + + +# We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even +# integrate with the cache, but there doesn't seem to be much call for it. +class __CompilationDbNode(SCons.Node.Python.Value): + def __init__(self, value): + SCons.Node.Python.Value.__init__(self, value) + self.Decider(changed_since_last_build_node) + + +def changed_since_last_build_node(child, target, prev_ni, node): + """ Dummy decider to force always building""" + return True + + +def makeEmitCompilationDbEntry(comstr): + """ + Effectively this creates a lambda function to capture: + * command line + * source + * target + :param comstr: unevaluated command line + :return: an emitter which has captured the above + """ + user_action = SCons.Action.Action(comstr) + + def EmitCompilationDbEntry(target, source, env): + """ + This emitter will be added to each c/c++ object build to capture the info needed + for clang tools + :param target: target node(s) + :param source: source node(s) + :param env: Environment for use building this node + :return: target(s), source(s) + """ + + dbtarget = __CompilationDbNode(source) + + entry = env.__COMPILATIONDB_Entry( + target=dbtarget, + source=[], + __COMPILATIONDB_UTARGET=target, + __COMPILATIONDB_USOURCE=source, + __COMPILATIONDB_UACTION=user_action, + __COMPILATIONDB_ENV=env, + ) + + # TODO: Technically, these next two lines should not be required: it should be fine to + # cache the entries. However, they don't seem to update properly. Since they are quick + # to re-generate disable caching and sidestep this problem. + env.AlwaysBuild(entry) + env.NoCache(entry) + + __COMPILATION_DB_ENTRIES.append(dbtarget) + + return target, source + + return EmitCompilationDbEntry + + +def CompilationDbEntryAction(target, source, env, **kw): + """ + Create a dictionary with evaluated command line, target, source + and store that info as an attribute on the target + (Which has been stored in __COMPILATION_DB_ENTRIES array + :param target: target node(s) + :param source: source node(s) + :param env: Environment for use building this node + :param kw: + :return: None + """ + + command = env["__COMPILATIONDB_UACTION"].strfunction( + target=env["__COMPILATIONDB_UTARGET"], + source=env["__COMPILATIONDB_USOURCE"], + env=env["__COMPILATIONDB_ENV"], + ) + + entry = { + "directory": env.Dir("#").abspath, + "command": command, + "file": str(env["__COMPILATIONDB_USOURCE"][0]), + } + + target[0].write(entry) + + +def WriteCompilationDb(target, source, env): + entries = [] + + for s in __COMPILATION_DB_ENTRIES: + entries.append(s.read()) + + with open(str(target[0]), "w") as target_file: + json.dump( + entries, target_file, sort_keys=True, indent=4, separators=(",", ": ") + ) + + +def ScanCompilationDb(node, env, path): + return __COMPILATION_DB_ENTRIES + + +def generate(env, **kwargs): + + static_obj, shared_obj = SCons.Tool.createObjBuilders(env) + + env["COMPILATIONDB_COMSTR"] = kwargs.get( + "COMPILATIONDB_COMSTR", "Building compilation database $TARGET" + ) + + components_by_suffix = itertools.chain( + itertools.product( + [".%s" % ext for ext in SRC_C_EXT], + [ + (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"), + (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"), + ], + ), + itertools.product( + [".%s" % ext for ext in SRC_CXX_EXT], + [ + (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"), + (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"), + ], + ), + itertools.product( + [".%s" % ext for ext in SRC_ASM_EXT], + [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASCOM")], + ), + ) + + for entry in components_by_suffix: + suffix = entry[0] + builder, base_emitter, command = entry[1] + + # Assumes a dictionary emitter + emitter = builder.emitter[suffix] + builder.emitter[suffix] = SCons.Builder.ListEmitter( + [emitter, makeEmitCompilationDbEntry(command)] + ) + + env["BUILDERS"]["__COMPILATIONDB_Entry"] = SCons.Builder.Builder( + action=SCons.Action.Action(CompilationDbEntryAction, None), + ) + + env["BUILDERS"]["__COMPILATIONDB_Database"] = SCons.Builder.Builder( + action=SCons.Action.Action(WriteCompilationDb, "$COMPILATIONDB_COMSTR"), + target_scanner=SCons.Scanner.Scanner( + function=ScanCompilationDb, node_class=None + ), + ) + + def CompilationDatabase(env, target): + result = env.__COMPILATIONDB_Database(target=target, source=[]) + + env.AlwaysBuild(result) + env.NoCache(result) + + return result + + env.AddMethod(CompilationDatabase, "CompilationDatabase") + + +def exists(env): + return True diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 3c8746bea0..e20226d1aa 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -366,7 +366,7 @@ def _get_found_includes( # pylint: disable=too-many-branches if not fs.path_endswith_ext(_h_path, piotool.SRC_HEADER_EXT): continue _f_part = _h_path[: _h_path.rindex(".")] - for ext in piotool.SRC_C_EXT: + for ext in piotool.SRC_C_EXT + piotool.SRC_CXX_EXT: if not isfile("%s.%s" % (_f_part, ext)): continue _c_path = self.env.File("%s.%s" % (_f_part, ext)) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 5d8222f63e..1132f447e0 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -31,8 +31,10 @@ from platformio.util import pioversion_to_intstr SRC_HEADER_EXT = ["h", "hpp"] -SRC_C_EXT = ["c", "cc", "cpp"] -SRC_BUILD_EXT = SRC_C_EXT + ["S", "spp", "SPP", "sx", "s", "asm", "ASM"] +SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"] +SRC_C_EXT = ["c"] +SRC_CXX_EXT = ["cc", "cpp", "cxx", "c++"] +SRC_BUILD_EXT = SRC_C_EXT + SRC_CXX_EXT + SRC_ASM_EXT SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % os.sep, "-<.svn%s>" % os.sep] From 2ecceb8ed2eaac0036e182adc44f9c5b73dd8b2a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 6 Feb 2020 17:30:55 +0200 Subject: [PATCH 122/142] Generate absolute path for compilation DB item --- platformio/builder/tools/compilation_db.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py index 6b77a5a410..c7a6b945ee 100644 --- a/platformio/builder/tools/compilation_db.py +++ b/platformio/builder/tools/compilation_db.py @@ -20,6 +20,7 @@ import itertools import json +import os import SCons @@ -129,7 +130,9 @@ def WriteCompilationDb(target, source, env): entries = [] for s in __COMPILATION_DB_ENTRIES: - entries.append(s.read()) + item = s.read() + item["file"] = os.path.abspath(item["file"]) + entries.append(item) with open(str(target[0]), "w") as target_file: json.dump( From 81960ce05188fcdabf6ba31dea685ff3797c408c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 6 Feb 2020 17:41:35 +0200 Subject: [PATCH 123/142] Fix test --- platformio/project/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 1ee27058ce..aadfbbb69d 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -13,7 +13,6 @@ # limitations under the License. import glob -import io import json import os import re @@ -449,7 +448,7 @@ def save(self, path=None): path = path or self.path if path in self._instances: del self._instances[path] - with io.open(path or self.path, mode="w+", encoding="utf8") as fp: + with open(path or self.path, "w+") as fp: fp.write(CONFIG_HEADER.strip() + "\n\n") self._parser.write(fp) fp.seek(0) From dfb47a089b3e0aad1ecb3834261896b875ecf2a7 Mon Sep 17 00:00:00 2001 From: Mark Harfouche Date: Thu, 6 Feb 2020 07:48:08 -0800 Subject: [PATCH 124/142] Add license file to sdist using MANIFEST.in (#3325) --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..1aba38f67a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE From d2033aacea388df3845d6d3122679d0bd8e4cf4b Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 6 Feb 2020 23:21:27 +0200 Subject: [PATCH 125/142] Remove the entire folder with temp files used by pvs-studio tool --- platformio/commands/check/tools/pvsstudio.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index bce4ae1496..36ae234106 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -14,6 +14,7 @@ import os import tempfile +import shutil from xml.etree.ElementTree import fromstring import click @@ -193,15 +194,8 @@ def _prepare_preprocessed_file(self, src_file): self._bad_input = True def clean_up(self): - temp_files = ( - self._tmp_output_file, - self._tmp_preprocessed_file, - self._tmp_cfg_file, - self._tmp_cmd_file, - ) - for f in temp_files: - if os.path.isfile(f): - os.remove(f) + if os.path.isdir(self._tmp_dir): + shutil.rmtree(self._tmp_dir) def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback From 73ce3c94e97a534c3b04080261d4593aa5603c28 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 6 Feb 2020 23:32:43 +0200 Subject: [PATCH 126/142] Initial support for `Project Manager // Resolve #3335 --- HISTORY.rst | 7 +- docs | 2 +- platformio/commands/__init__.py | 13 ++++ platformio/commands/ci.py | 9 ++- platformio/commands/{init.py => project.py} | 76 ++++++++++++++++----- platformio/project/exception.py | 2 +- tests/commands/test_init.py | 2 +- 7 files changed, 88 insertions(+), 23 deletions(-) rename platformio/commands/{init.py => project.py} (85%) diff --git a/HISTORY.rst b/HISTORY.rst index b6f4354ab5..e0eb61f6ec 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,7 +6,7 @@ Release Notes PlatformIO Core 4.0 ------------------- -4.1.1 (2020-??-??) +4.2.0 (2020-02-??) ~~~~~~~~~~~~~~~~~~ * `PlatformIO Home 3.0 `__: @@ -18,6 +18,11 @@ PlatformIO Core 4.0 - Added support for `PVS-Studio `__ static code analyzer +* Initial support for `Project Manager `_ CLI: + + - Show computed project configuration with a new `platformio project config `_ command or dump to JSON with ``platformio project config --json-output`` (`issue #3335 `_) + - Moved ``platformio init`` command to `platformio project init `_ + * Generate `compilation database "compile_commands.json" `_ (`issue #2990 `_) * Control debug flags and optimization level with a new `debug_build_flags `__ option * Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install `__ command (`issue #3345 `_) diff --git a/docs b/docs index ec5a17c390..506491d11c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ec5a17c390117b91c9f9b22d385ae535f3003964 +Subproject commit 506491d11c12c8d2c0a81729282e8c54f1877b68 diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index bc018f8c73..f6bac830de 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -63,5 +63,18 @@ def get_command(self, ctx, cmd_name): mod_path = "platformio.commands.%s.command" % cmd_name mod = __import__(mod_path, None, None, ["cli"]) except ImportError: + try: + return self._handle_obsolate_command(cmd_name) + except AttributeError: + pass raise click.UsageError('No such command "%s"' % cmd_name, ctx) return mod.cli + + @staticmethod + def _handle_obsolate_command(name): + # pylint: disable=import-outside-toplevel + if name == "init": + from platformio.commands.project import project_init + + return project_init + raise AttributeError() diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 7c57f46559..9a48f2622f 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -21,8 +21,8 @@ import click from platformio import app, fs -from platformio.commands.init import cli as cmd_init -from platformio.commands.init import validate_boards +from platformio.commands.project import project_init as cmd_project_init +from platformio.commands.project import validate_boards from platformio.commands.run.command import cli as cmd_run from platformio.compat import glob_escape from platformio.exception import CIBuildEnvsEmpty @@ -111,7 +111,10 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches # initialise project ctx.invoke( - cmd_init, project_dir=build_dir, board=board, project_option=project_option + cmd_project_init, + project_dir=build_dir, + board=board, + project_option=project_option, ) # process project diff --git a/platformio/commands/init.py b/platformio/commands/project.py similarity index 85% rename from platformio/commands/init.py rename to platformio/commands/project.py index e37871c950..33d350061b 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/project.py @@ -14,19 +14,61 @@ # pylint: disable=too-many-arguments,too-many-locals, too-many-branches -from os import getcwd, makedirs -from os.path import isdir, isfile, join +import os import click +from tabulate import tabulate from platformio import exception, fs from platformio.commands.platform import platform_install as cli_platform_install from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig +from platformio.project.exception import NotPlatformIOProjectError from platformio.project.helpers import is_platformio_project +@click.group(short_help="Project Manager") +def cli(): + pass + + +@cli.command("config", short_help="Show computed configuration") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path( + exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True + ), +) +@click.option("--json-output", is_flag=True) +def project_config(project_dir, json_output): + if not is_platformio_project(project_dir): + raise NotPlatformIOProjectError(project_dir) + with fs.cd(project_dir): + config = ProjectConfig.get_instance() + if json_output: + return click.echo(config.to_json()) + click.echo( + "Computed project configuration for %s" % click.style(project_dir, fg="cyan") + ) + for section, options in config.as_tuple(): + click.echo() + click.secho(section, fg="cyan") + click.echo("-" * len(section)) + click.echo( + tabulate( + [ + (name, "=", "\n".join(value) if isinstance(value, list) else value) + for name, value in options + ], + tablefmt="plain", + ) + ) + return None + + def validate_boards(ctx, param, value): # pylint: disable=W0613 pm = PlatformManager() for id_ in value: @@ -40,11 +82,11 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 return value -@click.command("init", short_help="Initialize PlatformIO project or update existing") +@cli.command("init", short_help="Initialize a project or update existing") @click.option( "--project-dir", "-d", - default=getcwd, + default=os.getcwd, type=click.Path( exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True ), @@ -55,7 +97,7 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 @click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) @click.pass_context -def cli( +def project_init( ctx, # pylint: disable=R0913 project_dir, board, @@ -65,7 +107,7 @@ def cli( silent, ): if not silent: - if project_dir == getcwd(): + if project_dir == os.getcwd(): click.secho("\nThe current working directory", fg="yellow", nl=False) click.secho(" %s " % project_dir, fg="cyan", nl=False) click.secho("will be used for the project.", fg="yellow") @@ -137,16 +179,16 @@ def init_base_project(project_dir): (config.get_optional_dir("test"), init_test_readme), ] for (path, cb) in dir_to_readme: - if isdir(path): + if os.path.isdir(path): continue - makedirs(path) + os.makedirs(path) if cb: cb(path) def init_include_readme(include_dir): fs.write_file_contents( - join(include_dir, "README"), + os.path.join(include_dir, "README"), """ This directory is intended for project header files. @@ -193,7 +235,7 @@ def init_include_readme(include_dir): def init_lib_readme(lib_dir): # pylint: disable=line-too-long fs.write_file_contents( - join(lib_dir, "README"), + os.path.join(lib_dir, "README"), """ This directory is intended for project specific (private) libraries. PlatformIO will compile them to static libraries and link into executable file. @@ -246,7 +288,7 @@ def init_lib_readme(lib_dir): def init_test_readme(test_dir): fs.write_file_contents( - join(test_dir, "README"), + os.path.join(test_dir, "README"), """ This directory is intended for PIO Unit Testing and project tests. @@ -263,8 +305,8 @@ def init_test_readme(test_dir): def init_ci_conf(project_dir): - conf_path = join(project_dir, ".travis.yml") - if isfile(conf_path): + conf_path = os.path.join(project_dir, ".travis.yml") + if os.path.isfile(conf_path): return fs.write_file_contents( conf_path, @@ -340,8 +382,8 @@ def init_ci_conf(project_dir): def init_cvs_ignore(project_dir): - conf_path = join(project_dir, ".gitignore") - if isfile(conf_path): + conf_path = os.path.join(project_dir, ".gitignore") + if os.path.isfile(conf_path): return fs.write_file_contents(conf_path, ".pio\n") @@ -349,7 +391,9 @@ def init_cvs_ignore(project_dir): def fill_project_envs( ctx, project_dir, board_ids, project_option, env_prefix, force_download ): - config = ProjectConfig(join(project_dir, "platformio.ini"), parse_extra=False) + config = ProjectConfig( + os.path.join(project_dir, "platformio.ini"), parse_extra=False + ) used_boards = [] for section in config.sections(): cond = [section.startswith("env:"), config.has_option(section, "board")] diff --git a/platformio/project/exception.py b/platformio/project/exception.py index c2d7fd09ca..aa45eb07c4 100644 --- a/platformio/project/exception.py +++ b/platformio/project/exception.py @@ -24,7 +24,7 @@ class NotPlatformIOProjectError(ProjectError, UserSideException): MESSAGE = ( "Not a PlatformIO project. `platformio.ini` file has not been " "found in current working directory ({0}). To initialize new project " - "please use `platformio init` command" + "please use `platformio project init` command" ) diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 5e0172a786..fccdee7d92 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -17,7 +17,7 @@ from os.path import getsize, isdir, isfile, join from platformio.commands.boards import cli as cmd_boards -from platformio.commands.init import cli as cmd_init +from platformio.commands.project import project_init as cmd_init from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectEnvsNotAvailableError From 88e40e28fc8f237d007d9b690caf104b9ee12586 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 6 Feb 2020 23:33:27 +0200 Subject: [PATCH 127/142] Bump version to 4.2.0rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6dccaef9e5..da070c90f5 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 1, "1b9") +VERSION = (4, 2, "0rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From fe174e35c81cfe62053afaf002554739befdda7c Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 6 Feb 2020 23:48:13 +0200 Subject: [PATCH 128/142] Suppress warnings from packages dir instead of core dir for CppCheck --- platformio/commands/check/tools/cppcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 840568bd52..4267528ebe 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -123,7 +123,7 @@ def configure_command(self): cmd.append("--file-list=%s" % self._generate_src_file()) cmd.append("--includes-file=%s" % self._generate_inc_file()) - core_dir = self.config.get_optional_dir("core") + core_dir = self.config.get_optional_dir("packages") cmd.append("--suppress=*:%s*" % core_dir) cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) From 0b0b63aa7d4ea8fd6d5c2eafe1127752707dae80 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Fri, 7 Feb 2020 11:26:45 +0200 Subject: [PATCH 129/142] Update templates for Atom, VSCode, CLion (#3371) * Wrap flags with whitespace chars when exporting data for IDEs * Update IDEs templates Take into account compiler flags that can contain whitespace characters (e.g. -iprefix) * Update template for VSCode * Add history record --- HISTORY.rst | 2 + platformio/builder/tools/pioide.py | 11 ++++- platformio/ide/tpls/atom/.gcc-flags.json.tpl | 4 +- .../ide/tpls/clion/CMakeListsPrivate.txt.tpl | 10 +++-- .../vscode/.vscode/c_cpp_properties.json.tpl | 41 +++++++++++++++++-- 5 files changed, 57 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e0eb61f6ec..c8915ebebb 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -42,6 +42,8 @@ PlatformIO Core 4.0 * Fixed an issue when invalid CLI command does not return non-zero exit code * Fixed an issue when Project Inspector crashes when flash use > 100% (`issue #3368 `_) * Fixed a "UnicodeDecodeError" when listing built-in libraries on macOS with Python 2.7 (`issue #3370 `_) +* Fixed an issue with improperly handled compiler flags with space symbols in VSCode template (`issue #3364 `_) + 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 253ce61de1..d36d20128b 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -138,9 +138,16 @@ def _get_svd_path(env): return None +def _escape_build_flag(flags): + return [flag if " " not in flag else '"%s"' % flag for flag in flags] + + def DumpIDEData(env): - LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS" - LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS" + + env["__escape_build_flag"] = _escape_build_flag + + LINTCCOM = "${__escape_build_flag(CFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" + LINTCXXCOM = "${__escape_build_flag(CXXFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" data = { "env_name": env["PIOENV"], diff --git a/platformio/ide/tpls/atom/.gcc-flags.json.tpl b/platformio/ide/tpls/atom/.gcc-flags.json.tpl index d2ddcf78de..361c2f0484 100644 --- a/platformio/ide/tpls/atom/.gcc-flags.json.tpl +++ b/platformio/ide/tpls/atom/.gcc-flags.json.tpl @@ -1,8 +1,8 @@ % _defines = " ".join(["-D%s" % d.replace(" ", "\\\\ ") for d in defines]) { "execPath": "{{ cxx_path }}", - "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", - "gccDefaultCppFlags": "-fsyntax-only {{! cxx_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", + "gccDefaultCFlags": "-fsyntax-only {{! to_unix_path(cc_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", + "gccDefaultCppFlags": "-fsyntax-only {{! to_unix_path(cxx_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}", "gccErrorLimit": 15, "gccIncludePaths": "{{ ','.join(includes) }}", "gccSuppressWarnings": false diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index daf5f2bca0..0c6eda0cda 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -5,7 +5,7 @@ # please create `CMakeListsUser.txt` in the root of project. # The `CMakeListsUser.txt` will not be overwritten by PlatformIO. -%from platformio.project.helpers import (load_project_ide_data) +% from platformio.project.helpers import (load_project_ide_data) % % import re % @@ -22,6 +22,10 @@ % return path % end % +% def _escape(text): +% return to_unix_path(text).replace('"', '\\"') +% end +% % envs = config.envs() % if len(envs) > 1: @@ -37,8 +41,8 @@ set(SVD_PATH "{{ _normalize_path(svd_path) }}") SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}") SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}") -SET(CMAKE_CXX_FLAGS "{{cxx_flags}}") -SET(CMAKE_C_FLAGS "{{cc_flags}}") +SET(CMAKE_CXX_FLAGS "{{ _normalize_path(to_unix_path(cxx_flags)) }}") +SET(CMAKE_C_FLAGS "{{ _normalize_path(to_unix_path(cc_flags)) }}") % STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)") % cc_stds = STD_RE.findall(cc_flags) diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index 5df41c544d..bb94aba28d 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -13,6 +13,35 @@ % return to_unix_path(text).replace('"', '\\"') % end % +% def _escape_required(flag): +% return " " in flag and systype == "windows" +% end +% +% def _split_flags(flags): +% result = [] +% i = 0 +% flags = flags.strip() +% while i < len(flags): +% current_arg = [] +% while i < len(flags) and flags[i] != " ": +% if flags[i] == '"': +% quotes_idx = flags.find('"', i + 1) +% current_arg.extend(flags[i + 1:quotes_idx]) +% i = quotes_idx + 1 +% else: +% current_arg.append(flags[i]) +% i = i + 1 +% end +% end +% arg = "".join(current_arg) +% if arg.strip(): +% result.append(arg.strip()) +% end +% i = i + 1 +% end +% return result +% end +% % cleaned_includes = [] % for include in includes: % if "toolchain-" not in dirname(commonprefix([include, cc_path])) and isdir(include): @@ -55,16 +84,20 @@ % cc_stds = STD_RE.findall(cc_flags) % cxx_stds = STD_RE.findall(cxx_flags) % -% # pass only architecture specific flags -% cc_m_flags = " ".join([f.strip() for f in cc_flags.split(" ") if f.strip().startswith(("-m", "-i", "@"))]) -% % if cc_stds: "cStandard": "c{{ cc_stds[-1] }}", % end % if cxx_stds: "cppStandard": "c++{{ cxx_stds[-1] }}", % end - "compilerPath": "\"{{cc_path}}\" {{! _escape(cc_m_flags) }}" + "compilerPath": "{{ cc_path }}", + "compilerArgs": [ +% for flag in [ '"%s"' % _escape(f) if _escape_required(f) else f for f in _split_flags( +% cc_flags) if f.startswith(("-m", "-i", "@"))]: + "{{ flag }}", +% end + "" + ] } ], "version": 4 From 2fa20b5c52d22b58bde18c2d3fa933a48469778e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 7 Feb 2020 11:28:32 +0200 Subject: [PATCH 130/142] Typo fix --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index c8915ebebb..12c73b126d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,7 +18,7 @@ PlatformIO Core 4.0 - Added support for `PVS-Studio `__ static code analyzer -* Initial support for `Project Manager `_ CLI: +* Initial support for `Project Manager `_ CLI: - Show computed project configuration with a new `platformio project config `_ command or dump to JSON with ``platformio project config --json-output`` (`issue #3335 `_) - Moved ``platformio init`` command to `platformio project init `_ From a78b461d45f412932b55a31f15b8471f0de2733c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 8 Feb 2020 19:10:15 +0200 Subject: [PATCH 131/142] Code formatting --- platformio/builder/tools/pioide.py | 8 ++++++-- platformio/commands/check/tools/pvsstudio.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index d36d20128b..6d54c66348 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -146,8 +146,12 @@ def DumpIDEData(env): env["__escape_build_flag"] = _escape_build_flag - LINTCCOM = "${__escape_build_flag(CFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" - LINTCXXCOM = "${__escape_build_flag(CXXFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" + LINTCCOM = ( + "${__escape_build_flag(CFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" + ) + LINTCXXCOM = ( + "${__escape_build_flag(CXXFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS" + ) data = { "env_name": env["PIOENV"], diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index 36ae234106..7407da63a4 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -13,8 +13,8 @@ # limitations under the License. import os -import tempfile import shutil +import tempfile from xml.etree.ElementTree import fromstring import click From 2763853d8d084051243797d60c54c1ba44d8bfb0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 8 Feb 2020 19:10:48 +0200 Subject: [PATCH 132/142] Fixed an issue when no error is raised if referred parameter (interpolation) is missing in a project configuration file // Resolve #3279 --- HISTORY.rst | 2 +- platformio/project/config.py | 78 ++++++++++++++++++++---------------- tests/test_projectconf.py | 32 ++++++++++----- 3 files changed, 66 insertions(+), 46 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 12c73b126d..b33e3c50de 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -43,7 +43,7 @@ PlatformIO Core 4.0 * Fixed an issue when Project Inspector crashes when flash use > 100% (`issue #3368 `_) * Fixed a "UnicodeDecodeError" when listing built-in libraries on macOS with Python 2.7 (`issue #3370 `_) * Fixed an issue with improperly handled compiler flags with space symbols in VSCode template (`issue #3364 `_) - +* Fixed an issue when no error is raised if referred parameter (interpolation) is missing in a project configuration file (`issue #3279 `_) 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/project/config.py b/platformio/project/config.py index aadfbbb69d..2e063fac05 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -21,7 +21,7 @@ import click from platformio import fs -from platformio.compat import PY2, WINDOWS, hashlib_encode_data +from platformio.compat import PY2, WINDOWS, hashlib_encode_data, string_types from platformio.project import exception from platformio.project.options import ProjectOptions @@ -43,6 +43,9 @@ """ +MISSING = object() + + class ProjectConfigBase(object): INLINE_COMMENT_RE = re.compile(r"\s+;.*$") @@ -242,22 +245,51 @@ def set(self, section, option, value): value = "\n" + value self._parser.set(section, option, value) - def getraw(self, section, option): + def getraw( # pylint: disable=too-many-branches + self, section, option, default=MISSING + ): if not self.expand_interpolations: return self._parser.get(section, option) - value = None - found = False + value = MISSING for sec, opt in self.walk_options(section): if opt == option: value = self._parser.get(sec, option) - found = True break - if not found: - value = self._parser.get(section, option) + option_meta = ProjectOptions.get("%s.%s" % (section.split(":", 1)[0], option)) + if not option_meta: + if value == MISSING: + value = ( + default if default != MISSING else self._parser.get(section, option) + ) + return self._expand_interpolations(value) + + if option_meta.sysenvvar: + envvar_value = os.getenv(option_meta.sysenvvar) + if not envvar_value and option_meta.oldnames: + for oldoption in option_meta.oldnames: + envvar_value = os.getenv("PLATFORMIO_" + oldoption.upper()) + if envvar_value: + break + if envvar_value and option_meta.multiple: + value += ("" if value == MISSING else "\n") + envvar_value + elif envvar_value and value == MISSING: + value = envvar_value + + if value == MISSING: + value = option_meta.default or default + if value == MISSING: + return None + + return self._expand_interpolations(value) - if "${" not in value or "}" not in value: + def _expand_interpolations(self, value): + if ( + not value + or not isinstance(value, string_types) + or not all(["${" in value, "}" in value]) + ): return value return self.VARTPL_RE.sub(self._re_interpolation_handler, value) @@ -267,41 +299,19 @@ def _re_interpolation_handler(self, match): return os.getenv(option) return self.getraw(section, option) - def get(self, section, option, default=None): # pylint: disable=too-many-branches + def get(self, section, option, default=MISSING): value = None try: - value = self.getraw(section, option) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - pass # handle value from system environment + value = self.getraw(section, option, default) except ConfigParser.Error as e: raise exception.InvalidProjectConfError(self.path, str(e)) option_meta = ProjectOptions.get("%s.%s" % (section.split(":", 1)[0], option)) if not option_meta: - return value or default + return value if option_meta.multiple: - value = self.parse_multi_values(value) - - if option_meta.sysenvvar: - envvar_value = os.getenv(option_meta.sysenvvar) - if not envvar_value and option_meta.oldnames: - for oldoption in option_meta.oldnames: - envvar_value = os.getenv("PLATFORMIO_" + oldoption.upper()) - if envvar_value: - break - if envvar_value and option_meta.multiple: - value = value or [] - value.extend(self.parse_multi_values(envvar_value)) - elif envvar_value and not value: - value = envvar_value - - # option is not specified by user - if value is None or ( - option_meta.multiple and value == [] and option_meta.default - ): - return default if default is not None else option_meta.default - + value = self.parse_multi_values(value or []) try: return self.cast_to(value, option_meta.type) except click.BadParameter as e: diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index c65c5737a2..447318753a 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -17,7 +17,7 @@ import pytest from platformio.project.config import ConfigParser, ProjectConfig -from platformio.project.exception import UnknownEnvNamesError +from platformio.project.exception import InvalidProjectConfError, UnknownEnvNamesError BASE_CONFIG = """ [platformio] @@ -34,6 +34,7 @@ Lib1 ; inline comment in multi-line value Lib2 lib_ignore = ${custom.lib_ignore} +custom_builtin_option = ${env.build_type} [strict_ldf] lib_ldf_mode = chain+ @@ -54,7 +55,7 @@ [env:base] build_flags = ${custom.debug_flags} ${custom.extra_flags} -lib_compat_mode = ${strict_ldf.strict} +lib_compat_mode = ${strict_ldf.lib_compat_mode} targets = [env:test_extends] @@ -100,13 +101,10 @@ def config(tmpdir_factory): def test_empty_config(): config = ProjectConfig("/non/existing/platformio.ini") - # unknown section - with pytest.raises(ConfigParser.NoSectionError): - config.getraw("unknown_section", "unknown_option") - + with pytest.raises(InvalidProjectConfError): + config.get("unknown_section", "unknown_option") assert config.sections() == [] - assert config.get("section", "option") is None assert config.get("section", "option", 13) == 13 @@ -159,6 +157,7 @@ def test_options(config): "custom_monitor_speed", "lib_deps", "lib_ignore", + "custom_builtin_option", ] assert config.options(env="test_extends") == [ "extends", @@ -169,6 +168,7 @@ def test_options(config): "custom_monitor_speed", "lib_deps", "lib_ignore", + "custom_builtin_option", ] @@ -180,7 +180,7 @@ def test_has_option(config): def test_sysenv_options(config): - assert config.get("custom", "extra_flags") is None + assert config.getraw("custom", "extra_flags") == "" assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] assert config.get("env:base", "upload_port") is None assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" @@ -205,6 +205,7 @@ def test_sysenv_options(config): "custom_monitor_speed", "lib_deps", "lib_ignore", + "custom_builtin_option", "upload_port", ] @@ -227,6 +228,10 @@ def test_getraw_value(config): with pytest.raises(ConfigParser.NoOptionError): config.getraw("platformio", "monitor_speed") + # default + assert config.getraw("unknown", "option", "default") == "default" + assert config.getraw("env:base", "custom_builtin_option") == "release" + # known assert config.getraw("env:base", "targets") == "" assert config.getraw("env:extra_1", "lib_deps") == "574" @@ -258,17 +263,18 @@ def test_items(config): assert config.items("custom") == [ ("debug_flags", "-D DEBUG=1"), ("lib_flags", "-lc -lm"), - ("extra_flags", None), + ("extra_flags", ""), ("lib_ignore", "LibIgnoreCustom"), ] assert config.items(env="base") == [ ("build_flags", ["-D DEBUG=1"]), - ("lib_compat_mode", "soft"), + ("lib_compat_mode", "strict"), ("targets", []), ("monitor_speed", 9600), ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), + ("custom_builtin_option", "release"), ] assert config.items(env="extra_1") == [ ( @@ -279,6 +285,7 @@ def test_items(config): ("monitor_speed", 9600), ("custom_monitor_speed", "115200"), ("lib_ignore", ["LibIgnoreCustom"]), + ("custom_builtin_option", "release"), ] assert config.items(env="extra_2") == [ ("build_flags", ["-Og"]), @@ -287,6 +294,7 @@ def test_items(config): ("monitor_speed", 9600), ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), + ("custom_builtin_option", "release"), ] assert config.items(env="test_extends") == [ ("extends", ["strict_settings"]), @@ -297,6 +305,7 @@ def test_items(config): ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), + ("custom_builtin_option", "release"), ] @@ -393,6 +402,7 @@ def test_dump(tmpdir_factory): ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["${custom.lib_ignore}"]), + ("custom_builtin_option", "${env.build_type}"), ], ), ("strict_ldf", [("lib_ldf_mode", "chain+"), ("lib_compat_mode", "strict")]), @@ -414,7 +424,7 @@ def test_dump(tmpdir_factory): "env:base", [ ("build_flags", ["${custom.debug_flags} ${custom.extra_flags}"]), - ("lib_compat_mode", "${strict_ldf.strict}"), + ("lib_compat_mode", "${strict_ldf.lib_compat_mode}"), ("targets", []), ], ), From 96f60aab66362733cf801b3c97abfe94501f336a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 8 Feb 2020 20:35:43 +0200 Subject: [PATCH 133/142] Bump version to 4.2.0rc2 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index da070c90f5..c77314ae65 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 2, "0rc1") +VERSION = (4, 2, "0rc2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 813cac0559..648ec9c310 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.1.0-rc.2,<3.2.0", + "contrib-piohome": ">=3.1.0-rc.3,<3.2.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.1", "tool-unity": "~1.20500.0", From 86f2dde6f379892a55db3519a8fc7493d85b4585 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 8 Feb 2020 21:36:32 +0200 Subject: [PATCH 134/142] Do not overwrite a custom items in VSCode's "extensions.json" // Resolve #3374 --- HISTORY.rst | 2 ++ .../tpls/vscode/.vscode/extensions.json.tpl | 27 ++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b33e3c50de..8712a6774a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -34,6 +34,7 @@ PlatformIO Core 4.0 * Updated Unity tool to 2.5.0 * Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) * Warn about broken library manifest when scanning dependencies (`issue #3268 `_) +* Do not overwrite a custom items in VSCode's "extensions.json" (`issue #3374 `_) * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) * Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ @@ -45,6 +46,7 @@ PlatformIO Core 4.0 * Fixed an issue with improperly handled compiler flags with space symbols in VSCode template (`issue #3364 `_) * Fixed an issue when no error is raised if referred parameter (interpolation) is missing in a project configuration file (`issue #3279 `_) + 4.1.0 (2019-11-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl b/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl index 8281e64cc8..62dfebad83 100644 --- a/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl @@ -1,7 +1,22 @@ +% import json +% import os +% import re +% +% recommendations = set(["platformio.platformio-ide"]) +% previous_json = os.path.join(project_dir, ".vscode", "extensions.json") +% if os.path.isfile(previous_json): +% with open(previous_json) as fp: +% contents = re.sub(r"^\s*//.*$", "", fp.read(), flags=re.M).strip() +% if contents: +% recommendations |= set(json.loads(contents).get("recommendations", [])) +% end +% end { - // See http://go.microsoft.com/fwlink/?LinkId=827846 - // for the documentation about the extensions.json format - "recommendations": [ - "platformio.platformio-ide" - ] -} \ No newline at end of file + // See http://go.microsoft.com/fwlink/?LinkId=827846 + // for the documentation about the extensions.json format + "recommendations": [ +% for i, item in enumerate(sorted(recommendations)): + "{{ item }}"{{ ("," if (i + 1) < len(recommendations) else "") }} +% end + ] +} From 86069ab7c6c36620e030b1b2aa4a335272f09420 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 8 Feb 2020 21:37:18 +0200 Subject: [PATCH 135/142] Typo fix --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8712a6774a..8a79182c82 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -34,7 +34,7 @@ PlatformIO Core 4.0 * Updated Unity tool to 2.5.0 * Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) * Warn about broken library manifest when scanning dependencies (`issue #3268 `_) -* Do not overwrite a custom items in VSCode's "extensions.json" (`issue #3374 `_) +* Do not overwrite custom items in VSCode's "extensions.json" (`issue #3374 `_) * Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) * Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ From c0aefe4c62abd0d681259e466a64e7b6c1fefff6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 10 Feb 2020 18:52:31 +0200 Subject: [PATCH 136/142] Add support for Nuclei RISC-V Processor development platform // Resolve #3340 --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 506491d11c..38b2c9df31 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 506491d11c12c8d2c0a81729282e8c54f1877b68 +Subproject commit 38b2c9df319a41d30da4e4d42766caf4d5af9400 diff --git a/examples b/examples index 5f9db822f4..45ef57ba95 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 5f9db822f4dbbf0aa9cf996438dcff291688cfc5 +Subproject commit 45ef57ba95034caecfe387dcb87d1b2978c7dfad From a033e2d1fed2f389b328881fed7a42977c17be14 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 11 Feb 2020 18:37:48 +0200 Subject: [PATCH 137/142] Docs: Update supported upload/debug protocols by Nuclei dev-platform --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 38b2c9df31..8120b0eea0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 38b2c9df319a41d30da4e4d42766caf4d5af9400 +Subproject commit 8120b0eea0416657ca6213359800957ba8e72bba diff --git a/examples b/examples index 45ef57ba95..e1d641126d 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 45ef57ba95034caecfe387dcb87d1b2978c7dfad +Subproject commit e1d641126d5fb8855da3070a029946fec25118b6 From edc2d10574b4b11bfa5ef542855cdb50e8f53f8a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 11 Feb 2020 18:59:35 +0200 Subject: [PATCH 138/142] Update SPDX License List to 3.8 --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 11885f34d1..be49b3ee31 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -242,7 +242,7 @@ def validate_license(self, value): def load_spdx_licenses(): r = requests.get( "https://raw.githubusercontent.com/spdx/license-list-data" - "/v3.7/json/licenses.json" + "/v3.8/json/licenses.json" ) r.raise_for_status() return r.json() From 0bc872eafdf031898ce882704267fe6acd4d7a80 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Feb 2020 13:47:42 +0200 Subject: [PATCH 139/142] Remap command before calling Click --- platformio/__main__.py | 9 +++++++++ platformio/commands/__init__.py | 13 ------------- platformio/telemetry.py | 9 +-------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index 6679d52ef4..4358e4febf 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -91,12 +91,21 @@ def _safe_echo(origin, *args, **kwargs): click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs) +def remap_command_arguments(argv): + if len(argv) > 1 and argv[1] == "init": + return argv[:1] + ["project"] + argv[1:] + return argv + + def main(argv=None): exit_code = 0 prev_sys_argv = sys.argv[:] if argv: assert isinstance(argv, list) sys.argv = argv + else: + sys.argv = remap_command_arguments(sys.argv) + try: configure() cli() # pylint: disable=no-value-for-parameter diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index f6bac830de..bc018f8c73 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -63,18 +63,5 @@ def get_command(self, ctx, cmd_name): mod_path = "platformio.commands.%s.command" % cmd_name mod = __import__(mod_path, None, None, ["cli"]) except ImportError: - try: - return self._handle_obsolate_command(cmd_name) - except AttributeError: - pass raise click.UsageError('No such command "%s"' % cmd_name, ctx) return mod.cli - - @staticmethod - def _handle_obsolate_command(name): - # pylint: disable=import-outside-toplevel - if name == "init": - from platformio.commands.project import project_init - - return project_init - raise AttributeError() diff --git a/platformio/telemetry.py b/platformio/telemetry.py index f765e06d18..ddb12f60dd 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -142,14 +142,7 @@ def _first_arg_from_list(args_, list_): return cmd_path = args[:1] - if args[0] in ( - "platform", - "platforms", - "serialports", - "device", - "settings", - "account", - ): + if args[0] in ("account", "device", "platform", "project", "settings",): cmd_path = args[:2] if args[0] == "lib" and len(args) > 1: lib_subcmds = ( From d0a6861369409270c3168557b733cb5d3532db20 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Feb 2020 15:14:58 +0200 Subject: [PATCH 140/142] Fix "TypeError: TypeError: write() argument 1 must be unicode" when generating project on Windows/Python 2 --- platformio/commands/debug/client.py | 2 +- platformio/ide/projectgenerator.py | 8 ++++---- .../ide/tpls/vscode/.vscode/extensions.json.tpl | 5 +++-- platformio/telemetry.py | 11 ++++++++++- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 2c6d972425..fa468bba62 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -265,7 +265,7 @@ def _handle_error(self, data): telemetry.encode_run_environment(self.env_options), last_erros, ) - telemetry.send_exception("DebugInitError: %s" % err, is_fatal=True) + telemetry.send_exception("DebugInitError: %s" % err) self.transport.loseConnection() def _kill_previous_session(self): diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 6e775099bf..34eb59f54c 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import io +import codecs import os import sys from os.path import basename, isdir, isfile, join, realpath, relpath @@ -129,18 +129,18 @@ def generate(self): dst_dir = join(self.project_dir, tpl_relpath) if not isdir(dst_dir): os.makedirs(dst_dir) - file_name = basename(tpl_path)[:-4] contents = self._render_tpl(tpl_path, tpl_vars) self._merge_contents(join(dst_dir, file_name), contents) @staticmethod def _render_tpl(tpl_path, tpl_vars): - return bottle.template(fs.get_file_contents(tpl_path), **tpl_vars) + with codecs.open(tpl_path, "r", encoding="utf8") as fp: + return bottle.SimpleTemplate(fp.read()).render(**tpl_vars) @staticmethod def _merge_contents(dst_path, contents): if basename(dst_path) == ".gitignore" and isfile(dst_path): return - with io.open(dst_path, "w", encoding="utf8") as fp: + with codecs.open(dst_path, "w", encoding="utf8") as fp: fp.write(contents) diff --git a/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl b/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl index 62dfebad83..1b2dfdd9a8 100644 --- a/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl @@ -5,8 +5,9 @@ % recommendations = set(["platformio.platformio-ide"]) % previous_json = os.path.join(project_dir, ".vscode", "extensions.json") % if os.path.isfile(previous_json): -% with open(previous_json) as fp: -% contents = re.sub(r"^\s*//.*$", "", fp.read(), flags=re.M).strip() +% fp = open(previous_json) +% contents = re.sub(r"^\s*//.*$", "", fp.read(), flags=re.M).strip() +% fp.close() % if contents: % recommendations |= set(json.loads(contents).get("recommendations", [])) % end diff --git a/platformio/telemetry.py b/platformio/telemetry.py index ddb12f60dd..54bd8c4bc5 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -78,6 +78,7 @@ def __init__(self): self._prefill_screen_name() self._prefill_appinfo() + self._prefill_sysargs() self._prefill_custom_data() def __getitem__(self, name): @@ -102,6 +103,15 @@ def _prefill_appinfo(self): dpdata.append("IDE/%s" % os.getenv("PLATFORMIO_IDE")) self["an"] = " ".join(dpdata) + def _prefill_sysargs(self): + args = [] + for arg in sys.argv[1:]: + arg = str(arg).lower() + if "@" in arg or os.path.exists(arg): + arg = "***" + args.append(arg) + self["cd3"] = " ".join(args) + def _prefill_custom_data(self): def _filter_args(items): result = [] @@ -118,7 +128,6 @@ def _filter_args(items): caller_id = str(app.get_session_var("caller_id")) self["cd1"] = util.get_systype() self["cd2"] = "Python/%s %s" % (platform.python_version(), platform.platform()) - # self['cd3'] = " ".join(_filter_args(sys.argv[1:])) self["cd4"] = ( 1 if (not util.is_ci() and (caller_id or not is_container())) else 0 ) From b41262a20e11a3b1cee88d62db49a0acd925d09f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Feb 2020 16:34:38 +0200 Subject: [PATCH 141/142] Fix broken "init" command --- platformio/__main__.py | 9 --------- platformio/commands/__init__.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index 4358e4febf..6679d52ef4 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -91,21 +91,12 @@ def _safe_echo(origin, *args, **kwargs): click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs) -def remap_command_arguments(argv): - if len(argv) > 1 and argv[1] == "init": - return argv[:1] + ["project"] + argv[1:] - return argv - - def main(argv=None): exit_code = 0 prev_sys_argv = sys.argv[:] if argv: assert isinstance(argv, list) sys.argv = argv - else: - sys.argv = remap_command_arguments(sys.argv) - try: configure() cli() # pylint: disable=no-value-for-parameter diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index bc018f8c73..f6bac830de 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -63,5 +63,18 @@ def get_command(self, ctx, cmd_name): mod_path = "platformio.commands.%s.command" % cmd_name mod = __import__(mod_path, None, None, ["cli"]) except ImportError: + try: + return self._handle_obsolate_command(cmd_name) + except AttributeError: + pass raise click.UsageError('No such command "%s"' % cmd_name, ctx) return mod.cli + + @staticmethod + def _handle_obsolate_command(name): + # pylint: disable=import-outside-toplevel + if name == "init": + from platformio.commands.project import project_init + + return project_init + raise AttributeError() From 176cf17f9f70800ca68ef03054f2d6f36af05ebd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Feb 2020 16:42:06 +0200 Subject: [PATCH 142/142] Bump version to 4.2.0 --- HISTORY.rst | 4 ++-- README.rst | 8 ++++++-- docs | 2 +- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8a79182c82..e142359986 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,10 +6,10 @@ Release Notes PlatformIO Core 4.0 ------------------- -4.2.0 (2020-02-??) +4.2.0 (2020-02-12) ~~~~~~~~~~~~~~~~~~ -* `PlatformIO Home 3.0 `__: +* `PlatformIO Home 3.1 `__: - Project Manager - Project Configuration UI for `"platformio.ini" `__ diff --git a/README.rst b/README.rst index 03fd04c60f..992b7c0cff 100644 --- a/README.rst +++ b/README.rst @@ -92,6 +92,7 @@ Development Platforms * `Microchip PIC32 `_ * `Nordic nRF51 `_ * `Nordic nRF52 `_ +* `Nuclei `_ * `NXP LPC `_ * `RISC-V `_ * `RISC-V GAP `_ @@ -109,14 +110,16 @@ Frameworks * `Arduino `_ * `CMSIS `_ -* `Energia `_ * `ESP-IDF `_ * `ESP8266 Non-OS SDK `_ * `ESP8266 RTOS SDK `_ * `Freedom E SDK `_ +* `GigaDevice GD32V SDK `_ * `Kendryte Standalone SDK `_ +* `Kendryte FreeRTOS SDK `_ * `libOpenCM3 `_ -* `mbed `_ +* `Mbed `_ +* `Nuclei SDK `_ * `PULP OS `_ * `Pumbaa `_ * `Shakti SDK `_ @@ -124,6 +127,7 @@ Frameworks * `SPL `_ * `STM32Cube `_ * `WiringPi `_ +* `Zephyr `_ Contributing ------------ diff --git a/docs b/docs index 8120b0eea0..dc25f117fd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8120b0eea0416657ca6213359800957ba8e72bba +Subproject commit dc25f117fd3b3acceed43ebae225e5b4a9f20105 diff --git a/platformio/__init__.py b/platformio/__init__.py index c77314ae65..208873da9a 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 2, "0rc2") +VERSION = (4, 2, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 648ec9c310..c7203eca2d 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": ">=3.1.0-rc.3,<3.2.0", + "contrib-piohome": "~3.1.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.6.1", "tool-unity": "~1.20500.0",