diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index e3bb201f3a..a2239d5bbe 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -26,7 +26,7 @@ jobs: - name: Run on Linux if: startsWith(matrix.os, 'ubuntu') env: - PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,intel_mcs51,aceinna_imu" + PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,intel_mcs51,aceinna_imu" run: | # ChipKIT issue: install 32-bit support for GCC PIC32 sudo apt-get install libc6-i386 @@ -40,7 +40,7 @@ jobs: - name: Run on macOS if: startsWith(matrix.os, 'macos') env: - PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,microchippic32,gd32v,nuclei" + PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,microchippic32,gd32v,nuclei,lattice_ice40" run: | df -h tox -e testexamples @@ -50,7 +50,7 @@ jobs: env: PLATFORMIO_CORE_DIR: C:/pio PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH - PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap" + PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,siwigsm,riscv_gap" run: | tox -e testexamples diff --git a/HISTORY.rst b/HISTORY.rst index c1668e51a3..41b7ab201b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,20 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +5.0.2 (2020-10-30) +~~~~~~~~~~~~~~~~~~ + +- Initialize a new project or update the existing passing working environment name and its options (`issue #3686 `_) +- Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry (`issue #3700 `_) +- Improved "core.call" RPC for PlatformIO Home (`issue #3671 `_) +- Fixed a "PermissionError: [WinError 5]" on Windows when an external repository is used with `lib_deps `__ option (`issue #3664 `_) +- Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 `_) +- Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 `_) +- Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses (`issue #3677 `_) +- Fixed an issue when `pio package publish `__ command removes original archive after submitting to the registry (`issue #3716 `_) +- Fixed an issue when multiple `pio lib install `__ command with the same local library results in duplicates in ``lib_deps`` (`issue #3715 `_) +- Fixed an issue with a "wrong" timestamp in device monitor output using `"time" filter `__ (`issue #3712 `_) + 5.0.1 (2020-09-10) ~~~~~~~~~~~~~~~~~~ diff --git a/Makefile b/Makefile index 57aa76c40b..3ddd1e27da 100644 --- a/Makefile +++ b/Makefile @@ -31,5 +31,8 @@ profile: python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS} snakeviz .tox/.tmp/cprofile.prof +pack: + python setup.py sdist + publish: python setup.py sdist upload diff --git a/README.rst b/README.rst index bdd2858b7f..c137fe436b 100644 --- a/README.rst +++ b/README.rst @@ -16,23 +16,21 @@ PlatformIO .. image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg :target: https://pypi.python.org/pypi/platformio/ :alt: License -.. image:: https://img.shields.io/badge/PlatformIO-Community-orange.svg - :alt: Community Forums - :target: https://community.platformio.org?utm_source=github&utm_medium=core +.. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg + :alt: Community Labs + :target: https://piolabs.com/?utm_source=github&utm_medium=core **Quick Links:** `Web `_ | `PlatformIO IDE `_ | `Project Examples `__ | `Docs `_ | `Donate `_ | -`Contact Us `_ +`Contact Us `_ -**Social:** `Twitter `_ | -`LinkedIn `_ | +**Social:** `LinkedIn `_ | +`Twitter `_ | `Facebook `_ | -`Hackaday `_ | -`Bintray `_ | -`Community `_ +`Community Forums `_ .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png :target: https://platformio.org?utm_source=github&utm_medium=core @@ -51,20 +49,18 @@ Get Started ----------- * `What is PlatformIO? `_ - -Instruments ------------ - * `PlatformIO IDE `_ * `PlatformIO Core (CLI) `_ -* `Library Management `_ * `Project Examples `__ + +Solutions +--------- + +* `Library Management `_ * `Desktop IDEs Integration `_ * `Continuous Integration `_ -* `Advanced Scripting API `_ -Professional ------------- +**Advanced** * `Debugging `_ * `Unit Testing `_ diff --git a/docs b/docs index 9bbb02295a..deae09a880 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9bbb02295a7f5ce325f1ed90a8b549fb81ce3857 +Subproject commit deae09a880822fb8c0b4973c29f75f2343476d6f diff --git a/platformio/__init__.py b/platformio/__init__.py index 145ad0cff4..55813e698c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, 1) +VERSION = (5, 0, 2) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -47,7 +47,7 @@ __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.3.0", + "contrib-piohome": "~3.3.1", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", @@ -57,8 +57,7 @@ } __check_internet_hosts__ = [ - "140.82.118.3", # Github.com - "35.231.145.151", # Gitlab.com + "185.199.110.153", # Github.com "88.198.170.159", # platformio.org "github.com", "platformio.org", diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 1547cf99aa..b1e8ffbd64 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -78,6 +78,7 @@ PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), PYTHONEXE=get_pythonexe_path(), + IDE_EXTRA_DATA={}, ) if not int(ARGUMENTS.get("PIOVERBOSE", 0)): diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index c21b150022..8248c06aaa 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -93,7 +93,9 @@ def _dump_defines(env): defines = [] # global symbols for item in processDefines(env.get("CPPDEFINES", [])): - defines.append(env.subst(item).replace("\\", "")) + item = item.strip() + if item: + defines.append(env.subst(item).replace("\\", "")) # special symbol for Atmel AVR MCU if env["PIOPLATFORM"] == "atmelavr": @@ -164,14 +166,17 @@ def DumpIDEData(env, globalenv): "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")), "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), "prog_path": env.subst("$PROG_PATH"), - "flash_extra_images": [ - {"offset": item[0], "path": env.subst(item[1])} - for item in env.get("FLASH_EXTRA_IMAGES", []) - ], "svd_path": _get_svd_path(env), "compiler_type": env.GetCompilerType(), "targets": globalenv.DumpTargets(), + "extra": dict( + flash_images=[ + {"offset": item[0], "path": env.subst(item[1])} + for item in env.get("FLASH_EXTRA_IMAGES", []) + ] + ), } + data["extra"].update(env.get("IDE_EXTRA_DATA", {})) env_ = env.Clone() # https://github.com/platformio/platformio-atom-ide/issues/34 diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index aac4742621..4e1ca9ca88 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -27,7 +27,7 @@ from SCons.Script import SConscript # pylint: disable=import-error from platformio import __version__, fs -from platformio.compat import string_types +from platformio.compat import MACOS, string_types from platformio.package.version import pepver_to_semver SRC_HEADER_EXT = ["h", "hpp"] @@ -69,7 +69,7 @@ def BuildProgram(env): if ( env.get("LIBS") and env.GetCompilerType() == "gcc" - and env.PioPlatform().is_embedded() + and (env.PioPlatform().is_embedded() or not MACOS) ): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 4d59bcaa62..1e22ca975f 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -133,9 +133,7 @@ def send_request(self, method, path, **kwargs): def fetch_json_data(self, method, path, **kwargs): cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None if not cache_valid: - return self.raise_error_from_response( - self.send_request(method, path, **kwargs) - ) + return self._parse_json_response(self.send_request(method, path, **kwargs)) cache_key = ContentCache.key_from_args( method, path, kwargs.get("params"), kwargs.get("data") ) @@ -144,11 +142,12 @@ def fetch_json_data(self, method, path, **kwargs): if result is not None: return json.loads(result) response = self.send_request(method, path, **kwargs) + data = self._parse_json_response(response) cc.set(cache_key, response.text, cache_valid) - return self.raise_error_from_response(response) + return data @staticmethod - def raise_error_from_response(response, expected_codes=(200, 201, 202)): + def _parse_json_response(response, expected_codes=(200, 201, 202)): if response.status_code in expected_codes: try: return response.json() diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index f8afdfab1b..b0387ce350 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -142,6 +142,6 @@ def get_package(self, type_, owner, name, version=None): cache_valid="1h", ) except HTTPClientError as e: - if e.response.status_code == 404: + if e.response is not None and e.response.status_code == 404: return None raise e diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index fc83405c0e..2ff969329e 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -24,7 +24,10 @@ from platformio import app, exception, fs, proc from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError +from platformio.commands.platform import platform_install as cmd_platform_install from platformio.package.manager.core import inject_contrib_pysite +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project, load_project_ide_data @@ -73,18 +76,29 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro env_options = config.items(env=env_name, as_dict=True) if not set(env_options.keys()) >= set(["platform", "board"]): raise ProjectEnvsNotAvailableError() - debug_options = helpers.validate_debug_options(ctx, env_options) + + try: + platform = PlatformFactory.new(env_options["platform"]) + except UnknownPlatform: + ctx.invoke( + cmd_platform_install, + platforms=[env_options["platform"]], + skip_default_package=True, + ) + platform = PlatformFactory.new(env_options["platform"]) + + debug_options = helpers.configure_initial_debug_options(platform, env_options) assert debug_options if not interface: return helpers.predebug_project(ctx, project_dir, env_name, False, verbose) - configuration = load_project_ide_data(project_dir, env_name) - if not configuration: - raise DebugInvalidOptionsError("Could not load debug configuration") + ide_data = load_project_ide_data(project_dir, env_name) + if not ide_data: + raise DebugInvalidOptionsError("Could not load a build configuration") if "--version" in __unprocessed: - result = proc.exec_command([configuration["gdb_path"], "--version"]) + result = proc.exec_command([ide_data["gdb_path"], "--version"]) if result["returncode"] == 0: return click.echo(result["out"]) raise exception.PlatformioException("\n".join([result["out"], result["err"]])) @@ -99,23 +113,25 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro nl=False, ) - debug_options["load_cmds"] = helpers.configure_esp32_load_cmds( - debug_options, configuration - ) + try: + debug_options = platform.configure_debug_options(debug_options, ide_data) + except NotImplementedError: + # legacy for ESP32 dev-platform <=2.0.0 + debug_options["load_cmds"] = helpers.configure_esp32_load_cmds( + debug_options, ide_data + ) rebuild_prog = False preload = debug_options["load_cmds"] == ["preload"] load_mode = debug_options["load_mode"] if load_mode == "always": - rebuild_prog = preload or not helpers.has_debug_symbols( - configuration["prog_path"] - ) + rebuild_prog = preload or not helpers.has_debug_symbols(ide_data["prog_path"]) elif load_mode == "modified": rebuild_prog = helpers.is_prog_obsolete( - configuration["prog_path"] - ) or not helpers.has_debug_symbols(configuration["prog_path"]) + ide_data["prog_path"] + ) or not helpers.has_debug_symbols(ide_data["prog_path"]) else: - rebuild_prog = not isfile(configuration["prog_path"]) + rebuild_prog = not isfile(ide_data["prog_path"]) if preload or (not rebuild_prog and load_mode != "always"): # don't load firmware through debug server @@ -139,9 +155,9 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro # save SHA sum of newly created prog if load_mode == "modified": - helpers.is_prog_obsolete(configuration["prog_path"]) + helpers.is_prog_obsolete(ide_data["prog_path"]) - if not isfile(configuration["prog_path"]): + if not isfile(ide_data["prog_path"]): raise DebugInvalidOptionsError("Program/firmware is missed") # run debugging client @@ -151,7 +167,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro from platformio.commands.debug.process.client import GDBClient, reactor client = GDBClient(project_dir, __unprocessed, debug_options, env_options) - client.spawn(configuration["gdb_path"], configuration["prog_path"]) + client.spawn(ide_data["gdb_path"], ide_data["prog_path"]) signal.signal(signal.SIGINT, lambda *args, **kwargs: None) reactor.run() diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 657e8c48ea..62ce4f7394 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -23,11 +23,8 @@ from platformio import fs, util from platformio.commands import PlatformioCLI from platformio.commands.debug.exception import DebugInvalidOptionsError -from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.run.command import cli as cmd_run from platformio.compat import is_bytes -from platformio.platform.exception import UnknownPlatform -from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.options import ProjectOptions @@ -89,21 +86,11 @@ def predebug_project(ctx, project_dir, env_name, preload, verbose): time.sleep(5) -def validate_debug_options(cmd_ctx, env_options): +def configure_initial_debug_options(platform, env_options): def _cleanup_cmds(items): items = ProjectConfig.parse_multi_values(items) return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] - try: - platform = PlatformFactory.new(env_options["platform"]) - except UnknownPlatform: - cmd_ctx.invoke( - cmd_platform_install, - platforms=[env_options["platform"]], - skip_default_package=True, - ) - platform = PlatformFactory.new(env_options["platform"]) - board_config = platform.board_config(env_options["board"]) tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) tool_settings = board_config.get("debug", {}).get("tools", {}).get(tool_name, {}) @@ -195,13 +182,16 @@ def _cleanup_cmds(items): def configure_esp32_load_cmds(debug_options, configuration): + """ + DEPRECATED: Moved to ESP32 dev-platform + See platform.py::configure_debug_options + """ + flash_images = configuration.get("extra", {}).get("flash_images") ignore_conds = [ debug_options["load_cmds"] != ["load"], "xtensa-esp32" not in configuration.get("cc_path", ""), - not configuration.get("flash_extra_images"), - not all( - [isfile(item["path"]) for item in configuration.get("flash_extra_images")] - ), + not flash_images, + not all([isfile(item["path"]) for item in flash_images]), ] if any(ignore_conds): return debug_options["load_cmds"] @@ -210,7 +200,7 @@ def configure_esp32_load_cmds(debug_options, configuration): 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( path=fs.to_unix_path(item["path"]), offset=item["offset"] ) - for item in configuration.get("flash_extra_images") + for item in flash_images ] mon_cmds.append( 'monitor program_esp32 "{%s.bin}" 0x10000 verify' diff --git a/platformio/commands/device/filters/time.py b/platformio/commands/device/filters/time.py index 203e6aa9fc..0c2d888410 100644 --- a/platformio/commands/device/filters/time.py +++ b/platformio/commands/device/filters/time.py @@ -22,13 +22,16 @@ class Timestamp(DeviceMonitorFilter): def __init__(self, *args, **kwargs): super(Timestamp, self).__init__(*args, **kwargs) - self._first_text_received = False + self._line_started = False def rx(self, text): - if self._first_text_received and "\n" not in text: + if self._line_started and "\n" not in text: return text timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] - if not self._first_text_received: - self._first_text_received = True - return "%s > %s" % (timestamp, text) + if not self._line_started: + self._line_started = True + text = "%s > %s" % (timestamp, text) + if text.endswith("\n"): + self._line_started = False + return text[:-1].replace("\n", "\n%s > " % timestamp) + "\n" return text.replace("\n", "\n%s > " % timestamp) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 00adf5b6b1..7a16f9c64a 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -27,7 +27,13 @@ from platformio import __main__, __version__, fs from platformio.commands.home import helpers -from platformio.compat import PY2, get_filesystem_encoding, is_bytes, string_types +from platformio.compat import ( + PY2, + get_filesystem_encoding, + get_locale_encoding, + is_bytes, + string_types, +) try: from thread import get_ident as thread_get_ident @@ -95,10 +101,11 @@ def _call_generator(args, options=None): else: args[i] = str(arg) + options = options or {} to_json = "--json-output" in args try: - if args and args[0] == "remote": + if options.get("force_subprocess"): result = yield PIOCoreRPC._call_subprocess(args, options) defer.returnValue(PIOCoreRPC._process_result(result, to_json)) else: @@ -117,7 +124,7 @@ def _call_generator(args, options=None): @staticmethod def _call_inline(args, options): PIOCoreRPC.setup_multithreading_std_streams() - cwd = (options or {}).get("cwd") or os.getcwd() + cwd = options.get("cwd") or os.getcwd() def _thread_task(): with fs.cd(cwd): @@ -143,13 +150,15 @@ def _call_subprocess(args, options): @staticmethod def _process_result(result, to_json=False): out, err, code = result + if out and is_bytes(out): + out = out.decode(get_locale_encoding()) + if err and is_bytes(err): + err = err.decode(get_locale_encoding()) text = ("%s\n\n%s" % (out, err)).strip() if code != 0: raise Exception(text) if not to_json: return text - if is_bytes(out): - out = out.decode() try: return json.loads(out) except ValueError as e: diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 2db966b699..eb9cd23721 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -198,7 +198,9 @@ def init(self, board, framework, project_dir): and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call(args, options={"cwd": project_dir}) + d = PIOCoreRPC.call( + args, options={"cwd": project_dir, "force_subprocess": True} + ) d.addCallback(self._generate_project_main, project_dir, framework) return d @@ -291,7 +293,9 @@ def import_arduino(self, board, use_arduino_libs, arduino_project_dir): and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call(args, options={"cwd": project_dir}) + d = PIOCoreRPC.call( + args, options={"cwd": project_dir, "force_subprocess": True} + ) d.addCallback(self._finalize_arduino_import, project_dir, arduino_project_dir) return d @@ -324,6 +328,8 @@ def import_pio(project_dir): and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() ): args.extend(["--ide", state["storage"]["coreCaller"]]) - d = PIOCoreRPC.call(args, options={"cwd": new_project_dir}) + d = PIOCoreRPC.call( + args, options={"cwd": new_project_dir, "force_subprocess": True} + ) d.addCallback(lambda _: new_project_dir) return d diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py index 732604a3f9..c720b10d59 100644 --- a/platformio/commands/lib/helpers.py +++ b/platformio/commands/lib/helpers.py @@ -81,15 +81,22 @@ def save_project_libdeps(project_dir, specs, environments=None, action="add"): if environments and env not in environments: continue config.expand_interpolations = False - lib_deps = [] + candidates = [] try: - lib_deps = ignore_deps_by_specs(config.get("env:" + env, "lib_deps"), specs) + candidates = ignore_deps_by_specs( + config.get("env:" + env, "lib_deps"), specs + ) except InvalidProjectConfError: pass if action == "add": - lib_deps.extend(spec.as_dependency() for spec in specs) - if lib_deps: - config.set("env:" + env, "lib_deps", lib_deps) + candidates.extend(spec.as_dependency() for spec in specs) + if candidates: + result = [] + for item in candidates: + item = item.strip() + if item and item not in result: + result.append(item) + config.set("env:" + env, "lib_deps", result) elif config.has_option("env:" + env, "lib_deps"): config.remove_option("env:" + env, "lib_deps") config.save() diff --git a/platformio/commands/package.py b/platformio/commands/package.py index f62362ff57..bda365706a 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -13,11 +13,14 @@ # limitations under the License. import os +import tempfile from datetime import datetime import click +from platformio import fs from platformio.clients.registry import RegistryClient +from platformio.compat import ensure_python3 from platformio.package.meta import PackageSpec, PackageType from platformio.package.pack import PackagePacker @@ -77,13 +80,16 @@ def package_pack(package, output): help="Notify by email when package is processed", ) def package_publish(package, owner, released_at, private, notify): - p = PackagePacker(package) - archive_path = p.pack() - response = RegistryClient().publish_package( - archive_path, owner, released_at, private, notify - ) - os.remove(archive_path) - click.secho(response.get("message"), fg="green") + assert ensure_python3() + with tempfile.TemporaryDirectory() as tmp_dir: # pylint: disable=no-member + with fs.cd(tmp_dir): + p = PackagePacker(package) + archive_path = p.pack() + response = RegistryClient().publish_package( + archive_path, owner, released_at, private, notify + ) + os.remove(archive_path) + click.secho(response.get("message"), fg="green") @cli.command("unpublish", short_help="Remove a pushed package from the registry") diff --git a/platformio/commands/project.py b/platformio/commands/project.py index f861b5b199..592f3a483c 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -174,8 +174,10 @@ def project_init( if is_new_project: init_base_project(project_dir) - if board: - fill_project_envs( + if environment: + update_project_env(project_dir, environment, project_option) + elif board: + update_board_envs( ctx, project_dir, board, project_option, env_prefix, ide is not None ) @@ -358,7 +360,7 @@ def init_cvs_ignore(project_dir): fp.write(".pio\n") -def fill_project_envs( +def update_board_envs( ctx, project_dir, board_ids, project_option, env_prefix, force_download ): config = ProjectConfig( @@ -417,6 +419,26 @@ def _install_dependent_platforms(ctx, platforms): ) +def update_project_env(project_dir, environment, project_option): + if not project_option: + return + config = ProjectConfig( + os.path.join(project_dir, "platformio.ini"), parse_extra=False + ) + + section = "env:%s" % environment + if not config.has_section(section): + config.add_section(section) + + for item in project_option: + if "=" not in item: + continue + _name, _value = item.split("=", 1) + config.set(section, _name.strip(), _value.strip()) + + config.save() + + def get_best_envname(config, board_ids=None): envname = None default_envs = config.default_envs() diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index cafbbd1f30..1a2c8ee2f4 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -23,12 +23,12 @@ import click -from platformio import exception, fs, proc +from platformio import fs, proc from platformio.commands.device import helpers as device_helpers from platformio.commands.device.command import device_monitor as cmd_device_monitor from platformio.commands.run.command import cli as cmd_run from platformio.commands.test.command import cli as cmd_test -from platformio.compat import PY2 +from platformio.compat import ensure_python3 from platformio.package.manager.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError @@ -37,13 +37,7 @@ @click.option("-a", "--agent", multiple=True) @click.pass_context def cli(ctx, agent): - if PY2: - raise exception.UserSideException( - "PlatformIO Remote Development requires Python 3.5 or above. \n" - "Please install the latest Python 3 and reinstall PlatformIO Core using " - "installation script:\n" - "https://docs.platformio.org/page/core/installation.html" - ) + assert ensure_python3() ctx.obj = agent inject_contrib_pysite(verify_openssl=True) diff --git a/platformio/compat.py b/platformio/compat.py index 59362d01b3..1ef2883b22 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -23,9 +23,12 @@ import re import sys +from platformio.exception import UserSideException + PY2 = sys.version_info[0] == 2 CYGWIN = sys.platform.startswith("cygwin") WINDOWS = sys.platform.startswith("win") +MACOS = sys.platform.startswith("darwin") def get_filesystem_encoding(): @@ -58,6 +61,17 @@ def ci_strings_are_equal(a, b): return a.strip().lower() == b.strip().lower() +def ensure_python3(raise_exception=True): + if not raise_exception or not PY2: + return not PY2 + raise UserSideException( + "Python 3.5 or later is required for this operation. \n" + "Please install the latest Python 3 and reinstall PlatformIO Core using " + "installation script:\n" + "https://docs.platformio.org/page/core/installation.html" + ) + + if PY2: import imp @@ -84,7 +98,7 @@ def dump_json_to_unicode(obj): if isinstance(obj, unicode): return obj return json.dumps( - obj, encoding=get_filesystem_encoding(), ensure_ascii=False, sort_keys=True + obj, encoding=get_filesystem_encoding(), ensure_ascii=False ).encode("utf8") _magic_check = re.compile("([*?[])") @@ -132,7 +146,7 @@ def hashlib_encode_data(data): def dump_json_to_unicode(obj): if isinstance(obj, string_types): return obj - return json.dumps(obj, ensure_ascii=False, sort_keys=True) + return json.dumps(obj) def glob_recursive(pathname): return glob.glob(pathname, recursive=True) diff --git a/platformio/ide/tpls/vim/.ccls.tpl b/platformio/ide/tpls/vim/.ccls.tpl index b6d7d55aa4..39e29f8735 100644 --- a/platformio/ide/tpls/vim/.ccls.tpl +++ b/platformio/ide/tpls/vim/.ccls.tpl @@ -18,5 +18,5 @@ clang % end % for define in defines: --D{{ define }} +-D{{ !define }} % end diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 9d82d6fefd..6cdccf0948 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -152,7 +152,10 @@ def install_from_url(self, url, spec, checksum=None, silent=False): return self._install_tmp_pkg(pkg_item) finally: if os.path.isdir(tmp_dir): - fs.rmtree(tmp_dir) + try: + shutil.rmtree(tmp_dir) + except: # pylint: disable=bare-except + pass def _install_tmp_pkg(self, tmp_pkg): assert isinstance(tmp_pkg, PackageItem) @@ -213,10 +216,10 @@ def _cleanup_dir(path): # move existing into the new place pkg_dir = os.path.join(self.package_dir, target_dirname) _cleanup_dir(pkg_dir) - shutil.move(dst_pkg.path, pkg_dir) + shutil.copytree(dst_pkg.path, pkg_dir, symlinks=True) # move new source to the destination location _cleanup_dir(dst_pkg.path) - shutil.move(tmp_pkg.path, dst_pkg.path) + shutil.copytree(tmp_pkg.path, dst_pkg.path, symlinks=True) return PackageItem(dst_pkg.path) if action == "detach-new": @@ -233,10 +236,10 @@ def _cleanup_dir(path): ) pkg_dir = os.path.join(self.package_dir, target_dirname) _cleanup_dir(pkg_dir) - shutil.move(tmp_pkg.path, pkg_dir) + shutil.copytree(tmp_pkg.path, pkg_dir, symlinks=True) return PackageItem(pkg_dir) # otherwise, overwrite existing _cleanup_dir(dst_pkg.path) - shutil.move(tmp_pkg.path, dst_pkg.path) + shutil.copytree(tmp_pkg.path, dst_pkg.path, symlinks=True) return PackageItem(dst_pkg.path) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index dcfe03f0c6..3664a72e2b 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -51,7 +51,7 @@ class BasePackageManager( # pylint: disable=too-many-public-methods def __init__(self, pkg_type, package_dir): self.pkg_type = pkg_type - self.package_dir = self.ensure_dir_exists(package_dir) + self.package_dir = package_dir self._MEMORY_CACHE = {} self._lockfile = None @@ -62,7 +62,9 @@ def __init__(self, pkg_type, package_dir): def lock(self): if self._lockfile: return + self.ensure_dir_exists(os.path.dirname(self.package_dir)) self._lockfile = LockFile(self.package_dir) + self.ensure_dir_exists(self.package_dir) self._lockfile.acquire() def unlock(self): @@ -91,10 +93,7 @@ def is_system_compatible(value): @staticmethod def ensure_dir_exists(path): if not os.path.isdir(path): - try: - os.makedirs(path) - except: # pylint: disable=bare-except - pass + os.makedirs(path) assert os.path.isdir(path) return path @@ -193,6 +192,9 @@ def build_metadata(self, pkg_dir, spec, vcs_revision=None): return metadata def get_installed(self): + if not os.path.isdir(self.package_dir): + return [] + cache_key = "get_installed" if self.memcache_get(cache_key): return self.memcache_get(cache_key) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index fd1cd856b3..a324beb466 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -14,12 +14,14 @@ import json import os +import shutil import subprocess import sys from datetime import date from platformio import __core_packages__, exception, fs, util from platformio.compat import PY2 +from platformio.package.exception import UnknownPackageError from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import get_pythonexe_path @@ -74,9 +76,17 @@ def inject_contrib_pysite(verify_openssl=False): # pylint: disable=import-outside-toplevel from site import addsitedir - contrib_pysite_dir = get_core_package_dir("contrib-pysite") + try: + contrib_pysite_dir = get_core_package_dir("contrib-pysite") + except UnknownPackageError: + pm = ToolPackageManager() + contrib_pysite_dir = build_contrib_pysite_package( + os.path.join(pm.package_dir, "contrib-pysite") + ) + if contrib_pysite_dir in sys.path: return True + addsitedir(contrib_pysite_dir) sys.path.insert(0, contrib_pysite_dir) @@ -87,34 +97,31 @@ def inject_contrib_pysite(verify_openssl=False): # pylint: disable=import-error,unused-import,unused-variable from OpenSSL import SSL except: # pylint: disable=bare-except - build_contrib_pysite_deps(get_core_package_dir("contrib-pysite")) + build_contrib_pysite_package(contrib_pysite_dir) return True -def build_contrib_pysite_deps(target_dir): +def build_contrib_pysite_package(target_dir, with_metadata=True): + systype = util.get_systype() if os.path.isdir(target_dir): fs.rmtree(target_dir) os.makedirs(target_dir) # build dependencies - pythonexe = get_pythonexe_path() + args = [ + get_pythonexe_path(), + "-m", + "pip", + "install", + "--no-compile", + "-t", + target_dir, + ] + if "linux" in systype: + args.extend(["--no-binary", ":all:"]) for dep in get_contrib_pysite_deps(): - subprocess.check_call( - [ - pythonexe, - "-m", - "pip", - "install", - # "--no-cache-dir", - "--no-compile", - "--no-binary", - ":all:", - "-t", - target_dir, - dep, - ] - ) + subprocess.check_call(args + [dep]) # build manifests with open(os.path.join(target_dir, "package.json"), "w") as fp: @@ -127,18 +134,55 @@ def build_contrib_pysite_deps(target_dir): sys.version_info.minor, date.today().strftime("%y%m%d"), ), - system=util.get_systype(), + system=list( + set([systype, "linux_armv6l", "linux_armv7l", "linux_armv8l"]) + ) + if systype.startswith("linux_arm") + else systype, + description="Extra Python package for PlatformIO Core", + keywords=["platformio", "platformio-core"], + homepage="https://docs.platformio.org/page/core/index.html", + repository={ + "type": "git", + "url": "https://github.com/platformio/platformio-core", + }, ), fp, ) - pm = ToolPackageManager() - pkg = PackageItem(target_dir) - pkg.metadata = pm.build_metadata( - target_dir, PackageSpec(owner="platformio", name="contrib-pysite") - ) - pkg.dump_meta() - return True + # generate package metadata + if with_metadata: + pm = ToolPackageManager() + pkg = PackageItem(target_dir) + pkg.metadata = pm.build_metadata( + target_dir, PackageSpec(owner="platformio", name="contrib-pysite") + ) + pkg.dump_meta() + + # remove unused files + shutil.rmtree(os.path.join(target_dir, "autobahn", "xbr", "contracts")) + for root, dirs, files in os.walk(target_dir): + for t in ("_test", "test", "tests"): + if t in dirs: + shutil.rmtree(os.path.join(root, t)) + for name in files: + if name.endswith((".chm", ".pyc")): + os.remove(os.path.join(root, name)) + + # apply patches + with open( + os.path.join(target_dir, "autobahn", "twisted", "__init__.py"), "r+" + ) as fp: + contents = fp.read() + contents = contents.replace( + "from autobahn.twisted.wamp import ApplicationSession", + "# from autobahn.twisted.wamp import ApplicationSession", + ) + fp.seek(0) + fp.truncate() + fp.write(contents) + + return target_dir def get_contrib_pysite_deps(): @@ -148,7 +192,7 @@ def get_contrib_pysite_deps(): twisted_version = "19.10.0" if PY2 else "20.3.0" result = [ "twisted == %s" % twisted_version, - "autobahn == %s" % ("19.11.2" if PY2 else "20.4.3"), + "autobahn == %s" % ("19.11.2" if PY2 else "20.7.1"), "json-rpc == 1.13.0", ] @@ -169,8 +213,8 @@ def get_contrib_pysite_deps(): result.append("pypiwin32 == 223") # workaround for twisted wheels twisted_wheel = ( - "https://download.lfd.uci.edu/pythonlibs/g5apjq5m/Twisted-" - "%s-cp%s-cp%sm-win%s.whl" + "https://download.lfd.uci.edu/pythonlibs/x2tqcw5k/Twisted-" + "%s-cp%s-cp%s-win%s.whl" % ( twisted_version, py_version, diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index b492bb7957..4dd4476de0 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -119,12 +119,13 @@ def new_from_archive(path): assert path.endswith("tar.gz") with tarfile.open(path, mode="r:gz") as tf: for t in sorted(ManifestFileType.items().values()): - try: - return ManifestParserFactory.new( - tf.extractfile(t).read().decode(), t - ) - except KeyError: - pass + for member in (t, "./" + t): + try: + return ManifestParserFactory.new( + tf.extractfile(member).read().decode(), t + ) + except KeyError: + pass raise UnknownManifestError("Unknown manifest file type in %s archive" % path) @staticmethod diff --git a/platformio/package/pack.py b/platformio/package/pack.py index d88957fc5a..ebdca496a4 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -20,6 +20,7 @@ import tempfile from platformio import fs +from platformio.compat import ensure_python3 from platformio.package.exception import PackageException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema @@ -28,20 +29,70 @@ class PackagePacker(object): + INCLUDE_DEFAULT = ManifestFileType.items().values() EXCLUDE_DEFAULT = [ + # PlatformIO internal files + PackageItem.METAFILE_NAME, + ".pio/", + "**/.pio/", + # Hidden files "._*", "__*", ".DS_Store", + ".vscode", + ".cache", + "**/.cache", + # VCS ".git/", ".hg/", ".svn/", - ".pio/", - "**/.pio/", - PackageItem.METAFILE_NAME, + # Tests + "tests?", + # Docs + "doc", + "docs", + "mkdocs", + "**/*.[pP][dD][fF]", + "**/*.[dD][oO][cC]?", + "**/*.[pP][pP][tT]?", + "**/*.[dD][oO][xX]", + "**/*.[hH][tT][mM]?", + "**/*.[tT][eE][xX]", + "**/*.[jJ][sS]", + "**/*.[cC][sS][sS]", + # Binary files + "**/*.[jJ][pP][gG]", + "**/*.[jJ][pP][eE][gG]", + "**/*.[pP][nN][gG]", + "**/*.[gG][iI][fF]", + "**/*.[zZ][iI][pP]", + "**/*.[gG][zZ]", + "**/*.3[gG][pP]", + "**/*.[mM][oO][vV]", + "**/*.[mM][pP][34]", + "**/*.[pP][sS][dD]", + "**/*.[wW][aA][wW]", + ] + EXCLUDE_LIBRARY_EXTRA = [ + "assets", + "extra", + "resources", + "html", + "media", + "doxygen", + "**/build/", + "**/*.flat", + "**/*.[jJ][aA][rR]", + "**/*.[eE][xX][eE]", + "**/*.[bB][iI][nN]", + "**/*.[hH][eE][xX]", + "**/*.[dD][bB]", + "**/*.[dD][aA][tT]", + "**/*.[dD][lL][lL]", ] - INCLUDE_DEFAULT = ManifestFileType.items().values() def __init__(self, package, manifest_uri=None): + assert ensure_python3() self.package = package self.manifest_uri = manifest_uri @@ -130,16 +181,28 @@ def _create_tarball(self, src, dst, manifest): json.dump(manifest_updated, fp, indent=2, ensure_ascii=False) include = None - src_filters = self.compute_src_filters(include, exclude) + src_filters = self.compute_src_filters(src, include, exclude) with tarfile.open(dst, "w:gz") as tar: for f in fs.match_src_files(src, src_filters, followlinks=False): tar.add(os.path.join(src, f), f) return dst - def compute_src_filters(self, include, exclude): + def compute_src_filters(self, src, include, exclude): + exclude_default = self.EXCLUDE_DEFAULT[:] + # extend with library extra filters + if any( + os.path.isfile(os.path.join(src, name)) + for name in ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ) + ): + exclude_default.extend(self.EXCLUDE_LIBRARY_EXTRA) + result = ["+<%s>" % p for p in include or ["*", ".*"]] result += ["-<%s>" % p for p in exclude or []] - result += ["-<%s>" % p for p in self.EXCLUDE_DEFAULT] + result += ["-<%s>" % p for p in exclude_default] # automatically include manifests result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT] return result diff --git a/platformio/platform/base.py b/platformio/platform/base.py index b214da5f54..81d9b085ef 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -203,6 +203,9 @@ def configure_default_packages(self, options, targets): elif "nobuild" in targets and opts.get("type") != "framework": self.packages[name]["optional"] = True + def configure_debug_options(self, initial_debug_options, ide_data): + raise NotImplementedError + def get_lib_storages(self): storages = {} for opts in (self.frameworks or {}).values(): diff --git a/platformio/project/config.py b/platformio/project/config.py index 2d841b396e..786f080aab 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -358,12 +358,6 @@ def validate(self, envs=None, silent=False): click.secho("Warning! %s" % warning, fg="yellow") return True - def remove_option(self, section, option): - return self._parser.remove_option(section, option) - - def remove_section(self, section): - return self._parser.remove_section(section) - class ProjectConfigDirsMixin(object): def _get_core_dir(self, exists=False): diff --git a/platformio/util.py b/platformio/util.py index 6da4708b83..e777d10a02 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -19,7 +19,6 @@ import os import platform import re -import sys import time from functools import wraps from glob import glob @@ -167,12 +166,9 @@ def get_mdns_services(): try: import zeroconf except ImportError: - from site import addsitedir - from platformio.package.manager.core import get_core_package_dir + from platformio.package.manager.core import inject_contrib_pysite - contrib_pysite_dir = get_core_package_dir("contrib-pysite") - addsitedir(contrib_pysite_dir) - sys.path.insert(0, contrib_pysite_dir) + inject_contrib_pysite() import zeroconf # pylint: disable=import-outside-toplevel class mDNSListener(object): diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 3698aa0bc7..881160ed20 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -1066,6 +1066,8 @@ def update_project_examples(): # Frameworks frameworks = [] for framework in API_FRAMEWORKS: + if framework["name"] not in framework_examples_md_lines: + continue readme_dir = join(project_examples_dir, "frameworks", framework["name"]) if not isdir(readme_dir): os.makedirs(readme_dir) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 8b948a95e4..920aee049f 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -21,6 +21,7 @@ import semantic_version from platformio import fs, util +from platformio.compat import PY2 from platformio.package.exception import ( MissingPackageManifestError, UnknownPackageError, @@ -144,6 +145,7 @@ def test_build_metadata(isolated_pio_core, tmpdir_factory): assert metadata.version.build[1] == vcs_revision +@pytest.mark.skipif(PY2, reason="Requires Python 3.5 or higher") def test_install_from_url(isolated_pio_core, tmpdir_factory): tmp_dir = tmpdir_factory.mktemp("tmp") storage_dir = tmpdir_factory.mktemp("storage") diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py index a964f5cdc0..92db31c4c9 100644 --- a/tests/package/test_pack.py +++ b/tests/package/test_pack.py @@ -19,10 +19,12 @@ import pytest from platformio import fs -from platformio.compat import WINDOWS +from platformio.compat import PY2, WINDOWS from platformio.package.exception import UnknownManifestError from platformio.package.pack import PackagePacker +pytestmark = pytest.mark.skipif(PY2, reason="Requires Python 3.5 or higher") + def test_base(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") diff --git a/tests/test_examples.py b/tests/test_examples.py index 994eb8c014..9e2b7cc079 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import random from glob import glob -from os import listdir, walk -from os.path import basename, dirname, getsize, isdir, isfile, join, normpath import pytest @@ -32,24 +31,26 @@ def pytest_generate_tests(metafunc): examples_dirs = [] # repo examples - examples_dirs.append(normpath(join(dirname(__file__), "..", "examples"))) + examples_dirs.append( + os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "examples")) + ) # dev/platforms for pkg in PlatformPackageManager().get_installed(): p = PlatformFactory.new(pkg) - examples_dir = join(p.get_dir(), "examples") - assert isdir(examples_dir) - examples_dirs.append(examples_dir) + examples_dir = os.path.join(p.get_dir(), "examples") + if os.path.isdir(examples_dir): + examples_dirs.append(examples_dir) project_dirs = [] for examples_dir in examples_dirs: candidates = {} - for root, _, files in walk(examples_dir): + for root, _, files in os.walk(examples_dir): if "platformio.ini" not in files or ".skiptest" in files: continue if "zephyr-" in root and PY2: continue - group = basename(root) + group = os.path.basename(root) if "-" in group: group = group.split("-", 1)[0] if group not in candidates: @@ -67,7 +68,7 @@ def test_run(pioproject_dir): with fs.cd(pioproject_dir): config = ProjectConfig() build_dir = config.get_optional_dir("build") - if isdir(build_dir): + if os.path.isdir(build_dir): fs.rmtree(build_dir) env_names = config.envs() @@ -77,18 +78,18 @@ def test_run(pioproject_dir): if result["returncode"] != 0: pytest.fail(str(result)) - assert isdir(build_dir) + assert os.path.isdir(build_dir) # check .elf file - for item in listdir(build_dir): - if not isdir(item): + for item in os.listdir(build_dir): + if not os.path.isdir(item): continue - assert isfile(join(build_dir, item, "firmware.elf")) + assert os.path.isfile(os.path.join(build_dir, item, "firmware.elf")) # check .hex or .bin files firmwares = [] for ext in ("bin", "hex"): - firmwares += glob(join(build_dir, item, "firmware*.%s" % ext)) + firmwares += glob(os.path.join(build_dir, item, "firmware*.%s" % ext)) if not firmwares: pytest.fail("Missed firmware file") for firmware in firmwares: - assert getsize(firmware) > 0 + assert os.path.getsize(firmware) > 0 diff --git a/tox.ini b/tox.ini index 0faae46b5e..bb41a67b04 100644 --- a/tox.ini +++ b/tox.ini @@ -13,13 +13,13 @@ # limitations under the License. [tox] -envlist = py27,py37,py38 +envlist = py27,py37,py38,py39 [testenv] passenv = * usedevelop = True deps = - py36,py37,py38: black + py36,py37,py38,py39: black isort<5 pylint pytest