From 0eb8895959c90cda07f014294d854cf76913c0d7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 14:25:28 +0300 Subject: [PATCH 001/223] =?UTF-8?q?Add=20support=20for=20=E2=80=9Cglobstar?= =?UTF-8?q?/**=E2=80=9D=20(recursive)=20pattern?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 5 +++++ platformio/commands/check/tools/base.py | 5 ++--- platformio/commands/ci.py | 10 ++++------ platformio/commands/home/rpc/handlers/os.py | 7 ++++--- platformio/compat.py | 7 +++++++ platformio/fs.py | 5 ++--- platformio/project/config.py | 11 ++++++++--- 7 files changed, 32 insertions(+), 18 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ab9119fe24..250e680ac8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,11 @@ Release Notes PlatformIO Core 4 ----------------- +4.3.5 (2020-??-??) +~~~~~~~~~~~~~~~~~~ + +* Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. + 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index d6f5d4f124..d873810d42 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob import os from tempfile import NamedTemporaryFile import click -from platformio import fs, proc +from platformio import compat, fs, proc from platformio.commands.check.defect import DefectItem from platformio.project.helpers import load_project_ide_data @@ -183,7 +182,7 @@ def _add_file(path): result["c++"].append(os.path.realpath(path)) for pattern in patterns: - for item in glob.glob(pattern): + for item in compat.glob_recursive(pattern): if not os.path.isdir(item): _add_file(item) for root, _, files in os.walk(item, followlinks=True): diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 9a48f2622f..f68b2bb763 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from glob import glob from os import getenv, makedirs, remove from os.path import basename, isdir, isfile, join, realpath from shutil import copyfile, copytree @@ -20,11 +19,10 @@ import click -from platformio import app, fs +from platformio import app, compat, fs from platformio.commands.project import project_init as cmd_project_init from platformio.commands.project import validate_boards from platformio.commands.run.command import cli as cmd_run -from platformio.compat import glob_escape from platformio.exception import CIBuildEnvsEmpty from platformio.project.config import ProjectConfig @@ -36,7 +34,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument if p.startswith("~"): value[i] = fs.expanduser(p) value[i] = realpath(value[i]) - if not glob(value[i]): + if not compat.glob_recursive(value[i]): invalid_path = p break try: @@ -98,7 +96,7 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches continue contents = [] for p in patterns: - contents += glob(p) + contents += compat.glob_recursive(p) _copy_contents(join(build_dir, dir_name), contents) if project_conf and isfile(project_conf): @@ -159,7 +157,7 @@ def _copy_contents(dst_dir, contents): def _exclude_contents(dst_dir, patterns): contents = [] for p in patterns: - contents += glob(join(glob_escape(dst_dir), p)) + contents += compat.glob_recursive(join(compat.glob_escape(dst_dir), p)) for path in contents: path = realpath(path) if isdir(path): diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 2997e8aaba..3b4bd4e1ae 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -14,7 +14,6 @@ from __future__ import absolute_import -import glob import io import os import shutil @@ -25,7 +24,7 @@ from platformio import app, fs, util from platformio.commands.home import helpers -from platformio.compat import PY2, get_filesystem_encoding +from platformio.compat import PY2, get_filesystem_encoding, glob_recursive class OSRPC(object): @@ -115,7 +114,9 @@ def glob(pathnames, root=None): pathnames = [pathnames] result = set() for pathname in pathnames: - result |= set(glob.glob(os.path.join(root, pathname) if root else pathname)) + result |= set( + glob_recursive(os.path.join(root, pathname) if root else pathname) + ) return list(result) @staticmethod diff --git a/platformio/compat.py b/platformio/compat.py index c812e98d00..7cfe47f18a 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -15,6 +15,7 @@ # pylint: disable=unused-import, no-name-in-module, import-error, # pylint: disable=no-member, undefined-variable +import glob import inspect import json import locale @@ -81,6 +82,9 @@ def dump_json_to_unicode(obj): _magic_check = re.compile("([*?[])") _magic_check_bytes = re.compile(b"([*?[])") + def glob_recursive(pathname): + return glob.glob(pathname) + def glob_escape(pathname): """Escape all special characters.""" # https://github.com/python/cpython/blob/master/Lib/glob.py#L161 @@ -122,6 +126,9 @@ def dump_json_to_unicode(obj): return obj return json.dumps(obj, ensure_ascii=False, sort_keys=True) + def glob_recursive(pathname): + return glob.glob(pathname, recursive=True) + def load_python_module(name, pathname): spec = importlib.util.spec_from_file_location(name, pathname) module = importlib.util.module_from_spec(spec) diff --git a/platformio/fs.py b/platformio/fs.py index 575a14e57f..5122c882f3 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -18,12 +18,11 @@ import shutil import stat import sys -from glob import glob import click from platformio import exception -from platformio.compat import WINDOWS, glob_escape +from platformio.compat import WINDOWS, glob_escape, glob_recursive class cd(object): @@ -135,7 +134,7 @@ def _append_build_item(items, item, src_dir): src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter): items = set() - for item in glob(os.path.join(glob_escape(src_dir), pattern)): + for item in glob_recursive(os.path.join(glob_escape(src_dir), pattern)): if os.path.isdir(item): for root, _, files in os.walk(item, followlinks=followlinks): for f in files: diff --git a/platformio/project/config.py b/platformio/project/config.py index 23d089bfdf..786f080aab 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob import json import os import re @@ -21,7 +20,13 @@ import click from platformio import fs -from platformio.compat import PY2, WINDOWS, hashlib_encode_data, string_types +from platformio.compat import ( + PY2, + WINDOWS, + glob_recursive, + hashlib_encode_data, + string_types, +) from platformio.project import exception from platformio.project.options import ProjectOptions @@ -117,7 +122,7 @@ def read(self, path, parse_extra=True): for pattern in self.get("platformio", "extra_configs", []): if pattern.startswith("~"): pattern = fs.expanduser(pattern) - for item in glob.glob(pattern): + for item in glob_recursive(pattern): self.read(item) def _maintain_renaimed_options(self): From 38699cca8fe97a47a5b36d248e43586075432caa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 14:26:42 +0300 Subject: [PATCH 002/223] Bump version to 4.3.5a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 1e2e3fd173..92359835be 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, 4) +VERSION = (4, 3, "5a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 58470e89114b529030e4a531ec4d98d5c5888678 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 14:30:43 +0300 Subject: [PATCH 003/223] PY2 lint fix --- platformio/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/compat.py b/platformio/compat.py index 7cfe47f18a..7f749fc96a 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -13,7 +13,7 @@ # limitations under the License. # pylint: disable=unused-import, no-name-in-module, import-error, -# pylint: disable=no-member, undefined-variable +# pylint: disable=no-member, undefined-variable, unexpected-keyword-arg import glob import inspect From 49cc5d606b8dc9ba2c8c3941830672eaee703039 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 21:58:58 +0300 Subject: [PATCH 004/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 683415246b..b26f9eb483 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 683415246be491a91c2f8e63fa46b0e6ab55f91b +Subproject commit b26f9eb4834bd5de1cc28c9d5a6cd99b4332ebef From 19cdc7d34ad2be76b46790ebb5b9c206e7045c80 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 22:01:32 +0300 Subject: [PATCH 005/223] Initial support for package publishing in to the registry --- platformio/__init__.py | 4 +- platformio/clients/__init__.py | 13 ++++++ platformio/clients/registry.py | 41 ++++++++++++++++++ platformio/clients/rest.py | 62 +++++++++++++++++++++++++++ platformio/commands/account/client.py | 6 +-- platformio/commands/package.py | 59 +++++++++++++++++++++++++ platformio/package/manifest/parser.py | 2 +- platformio/package/pack.py | 37 ++++++++++++++++ 8 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 platformio/clients/__init__.py create mode 100644 platformio/clients/registry.py create mode 100644 platformio/clients/rest.py create mode 100644 platformio/commands/package.py diff --git a/platformio/__init__.py b/platformio/__init__.py index 92359835be..60621751c8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -34,5 +34,7 @@ __copyright__ = "Copyright 2014-present PlatformIO" __apiurl__ = "https://api.platformio.org" -__pioaccount_api__ = "https://api.accounts.platformio.org" + +__accounts_api__ = "https://api.accounts.platformio.org" +__registry_api__ = "https://api.registry.platformio.org" __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" diff --git a/platformio/clients/__init__.py b/platformio/clients/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/clients/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py new file mode 100644 index 0000000000..7ab3a3c403 --- /dev/null +++ b/platformio/clients/registry.py @@ -0,0 +1,41 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import __registry_api__ +from platformio.clients.rest import RESTClient +from platformio.commands.account.client import AccountClient +from platformio.package.pack import PackageType + + +class RegistryClient(RESTClient): + def __init__(self): + super(RegistryClient, self).__init__(base_url=__registry_api__) + + def publish_package( + self, archive_path, owner=None, released_at=None, private=False + ): + client = AccountClient() + if not owner: + owner = client.get_account_info(offline=True).get("profile").get("username") + with open(archive_path, "rb") as fp: + response = self.send_request( + "post", + "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), + params={"private": 1 if private else 0, "released_at": released_at}, + headers={ + "Authorization": "Bearer %s" % client.fetch_authentication_token() + }, + data=fp, + ) + return response diff --git a/platformio/clients/rest.py b/platformio/clients/rest.py new file mode 100644 index 0000000000..4921e2ccb8 --- /dev/null +++ b/platformio/clients/rest.py @@ -0,0 +1,62 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests.adapters +from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error + +from platformio import app, util +from platformio.exception import PlatformioException + + +class RESTClientError(PlatformioException): + pass + + +class RESTClient(object): + def __init__(self, base_url): + if base_url.endswith("/"): + base_url = base_url[:-1] + self.base_url = base_url + self._session = requests.Session() + self._session.headers.update({"User-Agent": app.get_user_agent()}) + retry = Retry( + total=5, + backoff_factor=1, + method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], + status_forcelist=[500, 502, 503, 504], + ) + adapter = requests.adapters.HTTPAdapter(max_retries=retry) + self._session.mount(base_url, adapter) + + def send_request(self, method, path, **kwargs): + # check internet before and resolve issue with 60 seconds timeout + util.internet_on(raise_exception=True) + try: + response = getattr(self._session, method)(self.base_url + path, **kwargs) + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: + raise RESTClientError(e) + return self.raise_error_from_response(response) + + @staticmethod + def raise_error_from_response(response, expected_codes=(200, 201, 202)): + if response.status_code in expected_codes: + try: + return response.json() + except ValueError: + pass + try: + message = response.json()["message"] + except (KeyError, ValueError): + message = response.text + raise RESTClientError(message) diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py index fb679dc03d..e49f30bbe4 100644 --- a/platformio/commands/account/client.py +++ b/platformio/commands/account/client.py @@ -20,7 +20,7 @@ import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error -from platformio import __pioaccount_api__, app +from platformio import __accounts_api__, app from platformio.commands.account import exception from platformio.exception import InternetIsOffline @@ -30,7 +30,7 @@ class AccountClient(object): SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__( - self, api_base_url=__pioaccount_api__, retries=3, + self, api_base_url=__accounts_api__, retries=3, ): if api_base_url.endswith("/"): api_base_url = api_base_url[:-1] @@ -184,7 +184,7 @@ def update_profile(self, profile, current_password): ) return response - def get_account_info(self, offline): + def get_account_info(self, offline=False): account = app.get_state_item("account") if not account: raise exception.AccountNotAuthorized() diff --git a/platformio/commands/package.py b/platformio/commands/package.py new file mode 100644 index 0000000000..a2b6c38337 --- /dev/null +++ b/platformio/commands/package.py @@ -0,0 +1,59 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from datetime import datetime + +import click + +from platformio.clients.registry import RegistryClient +from platformio.package.pack import PackagePacker + + +def validate_datetime(ctx, param, value): # pylint: disable=unused-argument + try: + datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + except ValueError as e: + raise click.BadParameter(e) + return value + + +@click.group("package", short_help="Package Manager") +def cli(): + pass + + +@cli.command( + "publish", short_help="Publish a package to the PlatformIO Universal Registry" +) +@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +@click.option( + "--owner", + help="PIO Account username (could be organization username). " + "Default is set to a username of the authorized PIO Account", +) +@click.option( + "--released-at", + callback=validate_datetime, + help="Custom release date and time in the next format (UTC): 2014-06-13 17:08:52", +) +@click.option("--private", is_flag=True, help="Restricted access (not a public)") +def package_publish(package, owner, released_at, private): + p = PackagePacker(package) + archive_path = p.pack() + response = RegistryClient().publish_package( + archive_path, owner, released_at, private + ) + os.remove(archive_path) + click.secho(response.get("message"), fg="green") diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index e8ec592969..bf01772159 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -40,7 +40,7 @@ class ManifestFileType(object): @classmethod def items(cls): - return get_object_members(ManifestFileType) + return get_object_members(cls) @classmethod def from_uri(cls, uri): diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 1e18c55ae6..ecf14a4f5d 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -19,12 +19,49 @@ import tempfile from platformio import fs +from platformio.compat import get_object_members from platformio.package.exception import PackageException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema from platformio.unpacker import FileUnpacker +class PackageType(object): + LIBRARY = "library" + PLATFORM = "platform" + TOOL = "tool" + + @classmethod + def items(cls): + return get_object_members(cls) + + @classmethod + def get_manifest_map(cls): + return { + cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), + cls.LIBRARY: ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ), + cls.TOOL: (ManifestFileType.PACKAGE_JSON,), + } + + @classmethod + def from_archive(cls, path): + assert path.endswith("tar.gz") + manifest_map = cls.get_manifest_map() + with tarfile.open(path, mode="r|gz") as tf: + for t in sorted(cls.items().values()): + try: + for manifest in manifest_map[t]: + if tf.getmember(manifest): + return t + except KeyError: + pass + return None + + class PackagePacker(object): EXCLUDE_DEFAULT = [ "._*", From 8346b9822d236dd1a7614bdd6f144d83c1dfd30f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 May 2020 22:17:55 +0300 Subject: [PATCH 006/223] Implement "package pack" command --- platformio/clients/registry.py | 8 +++++--- platformio/commands/package.py | 8 ++++++++ platformio/commands/platform.py | 11 ----------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 7ab3a3c403..03e5b1784a 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -25,16 +25,18 @@ def __init__(self): def publish_package( self, archive_path, owner=None, released_at=None, private=False ): - client = AccountClient() + account = AccountClient() if not owner: - owner = client.get_account_info(offline=True).get("profile").get("username") + owner = ( + account.get_account_info(offline=True).get("profile").get("username") + ) with open(archive_path, "rb") as fp: response = self.send_request( "post", "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), params={"private": 1 if private else 0, "released_at": released_at}, headers={ - "Authorization": "Bearer %s" % client.fetch_authentication_token() + "Authorization": "Bearer %s" % account.fetch_authentication_token() }, data=fp, ) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index a2b6c38337..261523e270 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -34,6 +34,14 @@ def cli(): pass +@cli.command("pack", short_help="Create a tarball from a package") +@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +def package_pack(package): + p = PackagePacker(package) + tarball_path = p.pack() + click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green") + + @cli.command( "publish", short_help="Publish a package to the PlatformIO Universal Registry" ) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index d4ff49309a..c4a9ca5d1a 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -20,7 +20,6 @@ from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformFactory, PlatformManager -from platformio.package.pack import PackagePacker @click.group(short_help="Platform Manager") @@ -411,13 +410,3 @@ def platform_update( # pylint: disable=too-many-locals click.echo() return True - - -@cli.command( - "pack", short_help="Create a tarball from development platform/tool package" -) -@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") -def platform_pack(package): - p = PackagePacker(package) - tarball_path = p.pack() - click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green") From deb12972fb900d41e42eb1a3e6978c09a034ca11 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 May 2020 01:10:35 +0300 Subject: [PATCH 007/223] Implement "package unpublish" CLI --- platformio/clients/registry.py | 19 +++++++++++++++ platformio/commands/package.py | 26 ++++++++++++++++----- platformio/package/spec.py | 42 ++++++++++++++++++++++++++++++++++ tests/package/test_spec.py | 27 ++++++++++++++++++++++ 4 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 platformio/package/spec.py create mode 100644 tests/package/test_spec.py diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 03e5b1784a..6e9ea3fa82 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -41,3 +41,22 @@ def publish_package( data=fp, ) return response + + def unpublish_package(self, name, owner=None, version=None, undo=False): + account = AccountClient() + if not owner: + owner = ( + account.get_account_info(offline=True).get("profile").get("username") + ) + path = "/v3/package/%s/%s" % (owner, name) + if version: + path = path + "/version/" + version + response = self.send_request( + "delete", + path, + params={"undo": 1 if undo else 0}, + headers={ + "Authorization": "Bearer %s" % account.fetch_authentication_token() + }, + ) + return response diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 261523e270..bf42b35063 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -19,6 +19,7 @@ from platformio.clients.registry import RegistryClient from platformio.package.pack import PackagePacker +from platformio.package.spec import PackageSpec def validate_datetime(ctx, param, value): # pylint: disable=unused-argument @@ -38,17 +39,15 @@ def cli(): @click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") def package_pack(package): p = PackagePacker(package) - tarball_path = p.pack() - click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green") + archive_path = p.pack() + click.secho('Wrote a tarball to "%s"' % archive_path, fg="green") -@cli.command( - "publish", short_help="Publish a package to the PlatformIO Universal Registry" -) +@cli.command("publish", short_help="Publish a package to the registry") @click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") @click.option( "--owner", - help="PIO Account username (could be organization username). " + help="PIO Account username (can be organization username). " "Default is set to a username of the authorized PIO Account", ) @click.option( @@ -65,3 +64,18 @@ def package_publish(package, owner, released_at, private): ) os.remove(archive_path) click.secho(response.get("message"), fg="green") + + +@cli.command("unpublish", short_help="Remove a pushed package from the registry") +@click.argument("package", required=True, metavar="[<@organization>/][@]") +@click.option( + "--undo", + is_flag=True, + help="Undo a remove, putting a version back into the registry", +) +def package_unpublish(package, undo): + spec = PackageSpec(package) + response = RegistryClient().unpublish_package( + owner=spec.organization, name=spec.name, version=spec.version, undo=undo + ) + click.secho(response.get("message"), fg="green") diff --git a/platformio/package/spec.py b/platformio/package/spec.py new file mode 100644 index 0000000000..0de8227ac3 --- /dev/null +++ b/platformio/package/spec.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class PackageSpec(object): + def __init__(self, raw=None, organization=None, name=None, version=None): + if raw is not None: + organization, name, version = self.parse(raw) + + self.organization = organization + self.name = name + self.version = version + + @staticmethod + def parse(raw): + organization = None + name = None + version = None + raw = raw.strip() + if raw.startswith("@") and "/" in raw: + tokens = raw[1:].split("/", 1) + organization = tokens[0].strip() + raw = tokens[1] + if "@" in raw: + name, version = raw.split("@", 1) + name = name.strip() + version = version.strip() + else: + name = raw.strip() + + return organization, name, version diff --git a/tests/package/test_spec.py b/tests/package/test_spec.py new file mode 100644 index 0000000000..1886a8367b --- /dev/null +++ b/tests/package/test_spec.py @@ -0,0 +1,27 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.spec import PackageSpec + + +def test_parser(): + inputs = [ + ("foo", (None, "foo", None)), + ("@org/foo", ("org", "foo", None)), + ("@org/foo @ 1.2.3", ("org", "foo", "1.2.3")), + ("bar @ 1.2.3", (None, "bar", "1.2.3")), + ("cat@^1.2", (None, "cat", "^1.2")), + ] + for raw, result in inputs: + assert PackageSpec.parse(raw) == result From 0c301b2f5dcadaea4f53c4114895d91ef40bd287 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 May 2020 01:14:07 +0300 Subject: [PATCH 008/223] Fix order of arguments --- platformio/commands/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index bf42b35063..93423cc76a 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -76,6 +76,6 @@ def package_publish(package, owner, released_at, private): def package_unpublish(package, undo): spec = PackageSpec(package) response = RegistryClient().unpublish_package( - owner=spec.organization, name=spec.name, version=spec.version, undo=undo + name=spec.name, owner=spec.organization, version=spec.version, undo=undo ) click.secho(response.get("message"), fg="green") From e706a2cfe23d2fb66f00f5937cf97ff9ff63970a Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 27 May 2020 13:39:58 +0300 Subject: [PATCH 009/223] Refactor pio account client. Resolve #3525 (#3529) --- .../account/client.py => clients/account.py} | 116 ++++++------------ platformio/clients/registry.py | 2 +- .../{account/command.py => account.py} | 5 +- platformio/commands/account/__init__.py | 13 -- platformio/commands/account/exception.py | 30 ----- .../commands/home/rpc/handlers/account.py | 2 +- platformio/commands/remote/factory/client.py | 2 +- tests/commands/test_account.py | 2 +- 8 files changed, 45 insertions(+), 127 deletions(-) rename platformio/{commands/account/client.py => clients/account.py} (65%) rename platformio/commands/{account/command.py => account.py} (98%) delete mode 100644 platformio/commands/account/__init__.py delete mode 100644 platformio/commands/account/exception.py diff --git a/platformio/commands/account/client.py b/platformio/clients/account.py similarity index 65% rename from platformio/commands/account/client.py rename to platformio/clients/account.py index e49f30bbe4..078e2b8afb 100644 --- a/platformio/commands/account/client.py +++ b/platformio/clients/account.py @@ -12,47 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=unused-argument - import os import time -import requests.adapters -from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error - from platformio import __accounts_api__, app -from platformio.commands.account import exception -from platformio.exception import InternetIsOffline +from platformio.clients.rest import RESTClient +from platformio.exception import PlatformioException + + +class AccountError(PlatformioException): + + MESSAGE = "{0}" + + +class AccountNotAuthorized(AccountError): + + MESSAGE = "You are not authorized! Please log in to PIO Account." + + +class AccountAlreadyAuthorized(AccountError): + + MESSAGE = "You are already authorized with {0} account." -class AccountClient(object): +class AccountClient(RESTClient): SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 - def __init__( - self, api_base_url=__accounts_api__, retries=3, - ): - if api_base_url.endswith("/"): - api_base_url = api_base_url[:-1] - self.api_base_url = api_base_url - self._session = requests.Session() - self._session.headers.update({"User-Agent": app.get_user_agent()}) - retry = Retry( - total=retries, - read=retries, - connect=retries, - backoff_factor=2, - method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], - ) - adapter = requests.adapters.HTTPAdapter(max_retries=retry) - self._session.mount(api_base_url, adapter) + def __init__(self): + super(AccountClient, self).__init__(base_url=__accounts_api__) @staticmethod def get_refresh_token(): try: return app.get_state_item("account").get("auth").get("refresh_token") except: # pylint:disable=bare-except - raise exception.AccountNotAuthorized() + raise AccountNotAuthorized() @staticmethod def delete_local_session(): @@ -72,14 +67,12 @@ def login(self, username, password): except: # pylint:disable=bare-except pass else: - raise exception.AccountAlreadyAuthorized( + raise AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) result = self.send_request( - "post", - self.api_base_url + "/v1/login", - data={"username": username, "password": password}, + "post", "/v1/login", data={"username": username, "password": password}, ) app.set_state_item("account", result) return result @@ -90,13 +83,13 @@ def login_with_code(self, client_id, code, redirect_uri): except: # pylint:disable=bare-except pass else: - raise exception.AccountAlreadyAuthorized( + raise AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) result = self.send_request( "post", - self.api_base_url + "/v1/login/code", + "/v1/login/code", data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, ) app.set_state_item("account", result) @@ -107,11 +100,9 @@ def logout(self): self.delete_local_session() try: self.send_request( - "post", - self.api_base_url + "/v1/logout", - data={"refresh_token": refresh_token}, + "post", "/v1/logout", data={"refresh_token": refresh_token}, ) - except exception.AccountError: + except AccountError: pass return True @@ -119,7 +110,7 @@ def change_password(self, old_password, new_password): token = self.fetch_authentication_token() self.send_request( "post", - self.api_base_url + "/v1/password", + "/v1/password", headers={"Authorization": "Bearer %s" % token}, data={"old_password": old_password, "new_password": new_password}, ) @@ -133,13 +124,13 @@ def registration( except: # pylint:disable=bare-except pass else: - raise exception.AccountAlreadyAuthorized( + raise AccountAlreadyAuthorized( app.get_state_item("account", {}).get("email", "") ) return self.send_request( "post", - self.api_base_url + "/v1/registration", + "/v1/registration", data={ "username": username, "email": email, @@ -153,23 +144,19 @@ def auth_token(self, password, regenerate): token = self.fetch_authentication_token() result = self.send_request( "post", - self.api_base_url + "/v1/token", + "/v1/token", headers={"Authorization": "Bearer %s" % token}, data={"password": password, "regenerate": 1 if regenerate else 0}, ) return result.get("auth_token") def forgot_password(self, username): - return self.send_request( - "post", self.api_base_url + "/v1/forgot", data={"username": username}, - ) + return self.send_request("post", "/v1/forgot", data={"username": username},) def get_profile(self): token = self.fetch_authentication_token() return self.send_request( - "get", - self.api_base_url + "/v1/profile", - headers={"Authorization": "Bearer %s" % token}, + "get", "/v1/profile", headers={"Authorization": "Bearer %s" % token}, ) def update_profile(self, profile, current_password): @@ -178,7 +165,7 @@ def update_profile(self, profile, current_password): self.delete_local_state("summary") response = self.send_request( "put", - self.api_base_url + "/v1/profile", + "/v1/profile", headers={"Authorization": "Bearer %s" % token}, data=profile, ) @@ -187,7 +174,7 @@ def update_profile(self, profile, current_password): def get_account_info(self, offline=False): account = app.get_state_item("account") if not account: - raise exception.AccountNotAuthorized() + raise AccountNotAuthorized() if ( account.get("summary") and account["summary"].get("expire_at", 0) > time.time() @@ -202,9 +189,7 @@ def get_account_info(self, offline=False): } token = self.fetch_authentication_token() result = self.send_request( - "get", - self.api_base_url + "/v1/summary", - headers={"Authorization": "Bearer %s" % token}, + "get", "/v1/summary", headers={"Authorization": "Bearer %s" % token}, ) account["summary"] = dict( profile=result.get("profile"), @@ -227,36 +212,13 @@ def fetch_authentication_token(self): try: result = self.send_request( "post", - self.api_base_url + "/v1/login", + "/v1/login", headers={ "Authorization": "Bearer %s" % auth.get("refresh_token") }, ) app.set_state_item("account", result) return result.get("auth").get("access_token") - except exception.AccountError: + except AccountError: self.delete_local_session() - raise exception.AccountNotAuthorized() - - def send_request(self, method, url, headers=None, data=None): - try: - response = getattr(self._session, method)( - url, headers=headers or {}, data=data or {} - ) - except requests.exceptions.ConnectionError: - raise InternetIsOffline() - return self.raise_error_from_response(response) - - def raise_error_from_response(self, response, expected_codes=(200, 201, 202)): - if response.status_code in expected_codes: - try: - return response.json() - except ValueError: - pass - try: - message = response.json()["message"] - except (KeyError, ValueError): - message = response.text - if "Authorization session has been expired" in message: - self.delete_local_session() - raise exception.AccountError(message) + raise AccountNotAuthorized() diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 6e9ea3fa82..8c4d41c61d 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -13,8 +13,8 @@ # limitations under the License. from platformio import __registry_api__ +from platformio.clients.account import AccountClient from platformio.clients.rest import RESTClient -from platformio.commands.account.client import AccountClient from platformio.package.pack import PackageType diff --git a/platformio/commands/account/command.py b/platformio/commands/account.py similarity index 98% rename from platformio/commands/account/command.py rename to platformio/commands/account.py index 0177d00a24..78c5aa9eeb 100644 --- a/platformio/commands/account/command.py +++ b/platformio/commands/account.py @@ -21,8 +21,7 @@ import click from tabulate import tabulate -from platformio.commands.account import exception -from platformio.commands.account.client import AccountClient +from platformio.clients.account import AccountClient, AccountNotAuthorized @click.group("account", short_help="Manage PIO Account") @@ -167,7 +166,7 @@ def account_update(current_password, **kwargs): return None try: client.logout() - except exception.AccountNotAuthorized: + except AccountNotAuthorized: pass if email_changed: return click.secho( diff --git a/platformio/commands/account/__init__.py b/platformio/commands/account/__init__.py deleted file mode 100644 index b051490361..0000000000 --- a/platformio/commands/account/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/platformio/commands/account/exception.py b/platformio/commands/account/exception.py deleted file mode 100644 index a1a0059e27..0000000000 --- a/platformio/commands/account/exception.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from platformio.exception import PlatformioException - - -class AccountError(PlatformioException): - - MESSAGE = "{0}" - - -class AccountNotAuthorized(AccountError): - - MESSAGE = "You are not authorized! Please log in to PIO Account." - - -class AccountAlreadyAuthorized(AccountError): - - MESSAGE = "You are already authorized with {0} account." diff --git a/platformio/commands/home/rpc/handlers/account.py b/platformio/commands/home/rpc/handlers/account.py index 911006bc41..d28379f83f 100644 --- a/platformio/commands/home/rpc/handlers/account.py +++ b/platformio/commands/home/rpc/handlers/account.py @@ -14,7 +14,7 @@ import jsonrpc # pylint: disable=import-error -from platformio.commands.account.client import AccountClient +from platformio.clients.account import AccountClient class AccountRPC(object): diff --git a/platformio/commands/remote/factory/client.py b/platformio/commands/remote/factory/client.py index 26abe08017..2b47ab01a6 100644 --- a/platformio/commands/remote/factory/client.py +++ b/platformio/commands/remote/factory/client.py @@ -17,7 +17,7 @@ from twisted.spread import pb # pylint: disable=import-error from platformio.app import get_host_id -from platformio.commands.account.client import AccountClient +from platformio.clients.account import AccountClient class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory): diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py index ef7ffbad66..5b160f0c93 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account.py @@ -18,7 +18,7 @@ import pytest -from platformio.commands.account.command import cli as cmd_account +from platformio.commands.account import cli as cmd_account pytestmark = pytest.mark.skipif( not ( From c06859aa9fc7dfe900dcbb56a87445cb372b96ff Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 May 2020 14:30:27 +0300 Subject: [PATCH 010/223] Add package type to unpublish command --- platformio/clients/registry.py | 8 ++++--- platformio/commands/package.py | 20 +++++++++++++---- platformio/package/pack.py | 37 ------------------------------ platformio/package/spec.py | 41 ++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 44 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 8c4d41c61d..f68c4fb4cf 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -15,7 +15,7 @@ from platformio import __registry_api__ from platformio.clients.account import AccountClient from platformio.clients.rest import RESTClient -from platformio.package.pack import PackageType +from platformio.package.spec import PackageType class RegistryClient(RESTClient): @@ -42,13 +42,15 @@ def publish_package( ) return response - def unpublish_package(self, name, owner=None, version=None, undo=False): + def unpublish_package( # pylint: disable=redefined-builtin,too-many-arguments + self, type, name, owner=None, version=None, undo=False + ): account = AccountClient() if not owner: owner = ( account.get_account_info(offline=True).get("profile").get("username") ) - path = "/v3/package/%s/%s" % (owner, name) + path = "/v3/package/%s/%s/%s" % (owner, type, name) if version: path = path + "/version/" + version response = self.send_request( diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 93423cc76a..04e78e3020 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -19,7 +19,7 @@ from platformio.clients.registry import RegistryClient from platformio.package.pack import PackagePacker -from platformio.package.spec import PackageSpec +from platformio.package.spec import PackageSpec, PackageType def validate_datetime(ctx, param, value): # pylint: disable=unused-argument @@ -67,15 +67,27 @@ def package_publish(package, owner, released_at, private): @cli.command("unpublish", short_help="Remove a pushed package from the registry") -@click.argument("package", required=True, metavar="[<@organization>/][@]") +@click.argument( + "package", required=True, metavar="[<@organization>/][@]" +) +@click.option( + "--type", + type=click.Choice(list(PackageType.items().values())), + default="library", + help="Package type, default is set to `library`", +) @click.option( "--undo", is_flag=True, help="Undo a remove, putting a version back into the registry", ) -def package_unpublish(package, undo): +def package_unpublish(package, type, undo): # pylint: disable=redefined-builtin spec = PackageSpec(package) response = RegistryClient().unpublish_package( - name=spec.name, owner=spec.organization, version=spec.version, undo=undo + type=type, + name=spec.name, + owner=spec.organization, + version=spec.version, + undo=undo, ) click.secho(response.get("message"), fg="green") diff --git a/platformio/package/pack.py b/platformio/package/pack.py index ecf14a4f5d..1e18c55ae6 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -19,49 +19,12 @@ import tempfile from platformio import fs -from platformio.compat import get_object_members from platformio.package.exception import PackageException from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory from platformio.package.manifest.schema import ManifestSchema from platformio.unpacker import FileUnpacker -class PackageType(object): - LIBRARY = "library" - PLATFORM = "platform" - TOOL = "tool" - - @classmethod - def items(cls): - return get_object_members(cls) - - @classmethod - def get_manifest_map(cls): - return { - cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), - cls.LIBRARY: ( - ManifestFileType.LIBRARY_JSON, - ManifestFileType.LIBRARY_PROPERTIES, - ManifestFileType.MODULE_JSON, - ), - cls.TOOL: (ManifestFileType.PACKAGE_JSON,), - } - - @classmethod - def from_archive(cls, path): - assert path.endswith("tar.gz") - manifest_map = cls.get_manifest_map() - with tarfile.open(path, mode="r|gz") as tf: - for t in sorted(cls.items().values()): - try: - for manifest in manifest_map[t]: - if tf.getmember(manifest): - return t - except KeyError: - pass - return None - - class PackagePacker(object): EXCLUDE_DEFAULT = [ "._*", diff --git a/platformio/package/spec.py b/platformio/package/spec.py index 0de8227ac3..e729dae2ab 100644 --- a/platformio/package/spec.py +++ b/platformio/package/spec.py @@ -12,6 +12,47 @@ # See the License for the specific language governing permissions and # limitations under the License. +import tarfile + +from platformio.compat import get_object_members +from platformio.package.manifest.parser import ManifestFileType + + +class PackageType(object): + LIBRARY = "library" + PLATFORM = "platform" + TOOL = "tool" + + @classmethod + def items(cls): + return get_object_members(cls) + + @classmethod + def get_manifest_map(cls): + return { + cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), + cls.LIBRARY: ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ), + cls.TOOL: (ManifestFileType.PACKAGE_JSON,), + } + + @classmethod + def from_archive(cls, path): + assert path.endswith("tar.gz") + manifest_map = cls.get_manifest_map() + with tarfile.open(path, mode="r|gz") as tf: + for t in sorted(cls.items().values()): + try: + for manifest in manifest_map[t]: + if tf.getmember(manifest): + return t + except KeyError: + pass + return None + class PackageSpec(object): def __init__(self, raw=None, organization=None, name=None, version=None): From d38f5aca5c927db071dd253cc007a1204945d497 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 May 2020 16:20:02 +0300 Subject: [PATCH 011/223] Fix metavar for package CLI --- platformio/commands/package.py | 4 ++-- platformio/telemetry.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 04e78e3020..ac5c58ee0c 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -36,7 +36,7 @@ def cli(): @cli.command("pack", short_help="Create a tarball from a package") -@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +@click.argument("package", required=True, metavar="") def package_pack(package): p = PackagePacker(package) archive_path = p.pack() @@ -44,7 +44,7 @@ def package_pack(package): @cli.command("publish", short_help="Publish a package to the registry") -@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") +@click.argument("package", required=True, metavar="") @click.option( "--owner", help="PIO Account username (can be organization username). " diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 83c53c7f1d..7435bdabbd 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -146,7 +146,15 @@ def _first_arg_from_list(args_, list_): return cmd_path = args[:1] - if args[0] in ("account", "device", "platform", "project", "settings",): + if args[0] in ( + "account", + "device", + "platform", + "package", + "project", + "settings", + "system", + ): cmd_path = args[:2] if args[0] == "lib" and len(args) > 1: lib_subcmds = ( From c1965b607b67679f464ecc6dc39fb3b985ce71af Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 May 2020 17:27:05 +0300 Subject: [PATCH 012/223] Add binary stream to package publishing request --- platformio/clients/registry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index f68c4fb4cf..ba52c83bdc 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -36,7 +36,8 @@ def publish_package( "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), params={"private": 1 if private else 0, "released_at": released_at}, headers={ - "Authorization": "Bearer %s" % account.fetch_authentication_token() + "Authorization": "Bearer %s" % account.fetch_authentication_token(), + "Content-Type": "application/octet-stream", }, data=fp, ) From 8e72c48319cd30a25edcfa1d4b902c12722ee43e Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Wed, 27 May 2020 22:30:16 +0300 Subject: [PATCH 013/223] fix datetime validation in package publish command --- platformio/commands/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index ac5c58ee0c..b42d34b7e3 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -23,6 +23,8 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument + if not value: + return value try: datetime.strptime(value, "%Y-%m-%d %H:%M:%S") except ValueError as e: From 25a421402b43101722a59fd9a13c4a5c2ac20317 Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Thu, 28 May 2020 12:49:32 +0300 Subject: [PATCH 014/223] fix package type detector --- platformio/package/spec.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio/package/spec.py b/platformio/package/spec.py index e729dae2ab..0535d4ba4b 100644 --- a/platformio/package/spec.py +++ b/platformio/package/spec.py @@ -45,12 +45,12 @@ def from_archive(cls, path): manifest_map = cls.get_manifest_map() with tarfile.open(path, mode="r|gz") as tf: for t in sorted(cls.items().values()): - try: - for manifest in manifest_map[t]: + for manifest in manifest_map[t]: + try: if tf.getmember(manifest): return t - except KeyError: - pass + except KeyError: + pass return None From 49960b257dec1d1c4f49c17a03f39fac64b20d5f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 May 2020 16:07:02 +0300 Subject: [PATCH 015/223] Implement fs.calculate_file_hashsum --- platformio/downloader.py | 17 ++++------------- platformio/fs.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/platformio/downloader.py b/platformio/downloader.py index 21f5477bd8..ccbc5b361e 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib import io import math import sys @@ -23,7 +22,7 @@ import click import requests -from platformio import app, util +from platformio import app, fs, util from platformio.exception import ( FDSHASumMismatch, FDSizeMismatch, @@ -103,17 +102,9 @@ def verify(self, sha1=None): raise FDSizeMismatch(_dlsize, self._fname, self.get_size()) if not sha1: return None - - checksum = hashlib.sha1() - with io.open(self._destination, "rb", buffering=0) as fp: - while True: - chunk = fp.read(io.DEFAULT_BUFFER_SIZE) - if not chunk: - break - checksum.update(chunk) - - if sha1.lower() != checksum.hexdigest().lower(): - raise FDSHASumMismatch(checksum.hexdigest(), self._fname, sha1) + checksum = fs.calculate_file_hashsum("sha1", self._destination) + if sha1.lower() != checksum.lower(): + raise FDSHASumMismatch(checksum, self._fname, sha1) return True def _preserve_filemtime(self, lmdate): diff --git a/platformio/fs.py b/platformio/fs.py index 5122c882f3..7a592746ec 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib +import io import json import os import re @@ -72,6 +74,17 @@ def format_filesize(filesize): return "%d%sB" % ((base * filesize / unit), suffix) +def calculate_file_hashsum(algorithm, path): + h = hashlib.new(algorithm) + with io.open(path, "rb", buffering=0) as fp: + while True: + chunk = fp.read(io.DEFAULT_BUFFER_SIZE) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel From 37e795d539c045c4b13f9ec2db9d012f64ccfb0b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 May 2020 16:07:20 +0300 Subject: [PATCH 016/223] Send package checksum when publishing --- platformio/clients/registry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index ba52c83bdc..bbb0725c91 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio import __registry_api__ +from platformio import __registry_api__, fs from platformio.clients.account import AccountClient from platformio.clients.rest import RESTClient from platformio.package.spec import PackageType @@ -38,6 +38,7 @@ def publish_package( headers={ "Authorization": "Bearer %s" % account.fetch_authentication_token(), "Content-Type": "application/octet-stream", + "X-PIO-SHA256": fs.calculate_file_hashsum("sha256", archive_path), }, data=fp, ) From ae58cc74bdc5a960e36299f7a02a8342cb6acb67 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 May 2020 16:08:37 +0300 Subject: [PATCH 017/223] Rename checksum header to X-PIO-Content-SHA256 --- platformio/clients/registry.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index bbb0725c91..123b323916 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -38,7 +38,9 @@ def publish_package( headers={ "Authorization": "Bearer %s" % account.fetch_authentication_token(), "Content-Type": "application/octet-stream", - "X-PIO-SHA256": fs.calculate_file_hashsum("sha256", archive_path), + "X-PIO-Content-SHA256": fs.calculate_file_hashsum( + "sha256", archive_path + ), }, data=fp, ) From 26ba6e4756e2d3efea54ad1d5e6f1c14e9bda091 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 28 May 2020 17:06:36 +0300 Subject: [PATCH 018/223] Add new option to package publishing CLI which allows to disable email notiication --- platformio/clients/registry.py | 12 +++++++++--- platformio/commands/package.py | 9 +++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 123b323916..f90282d8cc 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -17,13 +17,15 @@ from platformio.clients.rest import RESTClient from platformio.package.spec import PackageType +# pylint: disable=too-many-arguments + class RegistryClient(RESTClient): def __init__(self): super(RegistryClient, self).__init__(base_url=__registry_api__) def publish_package( - self, archive_path, owner=None, released_at=None, private=False + self, archive_path, owner=None, released_at=None, private=False, notify=True ): account = AccountClient() if not owner: @@ -34,7 +36,11 @@ def publish_package( response = self.send_request( "post", "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), - params={"private": 1 if private else 0, "released_at": released_at}, + params={ + "private": 1 if private else 0, + "notify": 1 if notify else 0, + "released_at": released_at, + }, headers={ "Authorization": "Bearer %s" % account.fetch_authentication_token(), "Content-Type": "application/octet-stream", @@ -46,7 +52,7 @@ def publish_package( ) return response - def unpublish_package( # pylint: disable=redefined-builtin,too-many-arguments + def unpublish_package( # pylint: disable=redefined-builtin self, type, name, owner=None, version=None, undo=False ): account = AccountClient() diff --git a/platformio/commands/package.py b/platformio/commands/package.py index b42d34b7e3..e5eb7db66b 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -58,11 +58,16 @@ def package_pack(package): help="Custom release date and time in the next format (UTC): 2014-06-13 17:08:52", ) @click.option("--private", is_flag=True, help="Restricted access (not a public)") -def package_publish(package, owner, released_at, private): +@click.option( + "--notify/--no-notify", + default=True, + help="Notify by email when package is processed", +) +def package_publish(package, owner, released_at, private, notify): p = PackagePacker(package) archive_path = p.pack() response = RegistryClient().publish_package( - archive_path, owner, released_at, private + archive_path, owner, released_at, private, notify ) os.remove(archive_path) click.secho(response.get("message"), fg="green") From 9a1d2970cc02c3365079fe1f78e82c8fc20b7f3d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 30 May 2020 01:10:04 +0300 Subject: [PATCH 019/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b26f9eb483..dd9c31e97e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b26f9eb4834bd5de1cc28c9d5a6cd99b4332ebef +Subproject commit dd9c31e97efdd2d37b3386aa7324697ed3b3f6a1 From 9064fcbc775a61fc5fc1de43731d366d53237916 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 14:33:03 +0300 Subject: [PATCH 020/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index dd9c31e97e..88a52b6700 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit dd9c31e97efdd2d37b3386aa7324697ed3b3f6a1 +Subproject commit 88a52b6700255801b9bb53d2aab2f36c8072c73a From fe52f60389da9d116d2ad0402dff7078aafa5df8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 14:33:53 +0300 Subject: [PATCH 021/223] Bypass PermissionError when cleaning the cache --- platformio/maintenance.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index d2e7ea1cac..5f27573600 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -139,7 +139,10 @@ def after_upgrade(ctx): return else: click.secho("Please wait while upgrading PlatformIO...", fg="yellow") - app.clean_cache() + try: + app.clean_cache() + except PermissionError: + pass # Update PlatformIO's Core packages update_core_packages(silent=True) From 8c586dc360afca5201a54675a5e39a975d86d37e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 17:16:59 +0300 Subject: [PATCH 022/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 88a52b6700..5071271dff 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 88a52b6700255801b9bb53d2aab2f36c8072c73a +Subproject commit 5071271dff8a7162489fc43fb6d98966d2593c09 From 140fff9c23bc6749ba6e1791f716107e6975244c Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 3 Jun 2020 17:41:30 +0300 Subject: [PATCH 023/223] CLI to manage organizations. Resolve #3532 (#3540) * CLI to manage organizations. Resolve #3532 * fix tests * fix test * add org owner test * fix org test * fix invalid username/orgname error text * refactor auth request in clients * fix * fix send auth request * fix regexp * remove duplicated code. minor fixes. * Remove space Co-authored-by: Ivan Kravets --- platformio/clients/account.py | 70 ++++++++---- platformio/clients/registry.py | 20 ++-- platformio/commands/account.py | 12 ++- platformio/commands/org.py | 128 ++++++++++++++++++++++ tests/commands/test_account.py | 14 ++- tests/commands/test_orgs.py | 191 +++++++++++++++++++++++++++++++++ 6 files changed, 394 insertions(+), 41 deletions(-) create mode 100644 platformio/commands/org.py create mode 100644 tests/commands/test_orgs.py diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 078e2b8afb..9e0e65812a 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -35,7 +35,7 @@ class AccountAlreadyAuthorized(AccountError): MESSAGE = "You are already authorized with {0} account." -class AccountClient(RESTClient): +class AccountClient(RESTClient): # pylint:disable=too-many-public-methods SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 @@ -61,6 +61,14 @@ def delete_local_state(key): del account[key] app.set_state_item("account", account) + def send_auth_request(self, *args, **kwargs): + headers = kwargs.get("headers", {}) + if "Authorization" not in headers: + token = self.fetch_authentication_token() + headers["Authorization"] = "Bearer %s" % token + kwargs["headers"] = headers + return self.send_request(*args, **kwargs) + def login(self, username, password): try: self.fetch_authentication_token() @@ -107,11 +115,9 @@ def logout(self): return True def change_password(self, old_password, new_password): - token = self.fetch_authentication_token() - self.send_request( + self.send_auth_request( "post", "/v1/password", - headers={"Authorization": "Bearer %s" % token}, data={"old_password": old_password, "new_password": new_password}, ) return True @@ -141,11 +147,9 @@ def registration( ) def auth_token(self, password, regenerate): - token = self.fetch_authentication_token() - result = self.send_request( + result = self.send_auth_request( "post", "/v1/token", - headers={"Authorization": "Bearer %s" % token}, data={"password": password, "regenerate": 1 if regenerate else 0}, ) return result.get("auth_token") @@ -154,21 +158,12 @@ def forgot_password(self, username): return self.send_request("post", "/v1/forgot", data={"username": username},) def get_profile(self): - token = self.fetch_authentication_token() - return self.send_request( - "get", "/v1/profile", headers={"Authorization": "Bearer %s" % token}, - ) + return self.send_auth_request("get", "/v1/profile",) def update_profile(self, profile, current_password): - token = self.fetch_authentication_token() profile["current_password"] = current_password self.delete_local_state("summary") - response = self.send_request( - "put", - "/v1/profile", - headers={"Authorization": "Bearer %s" % token}, - data=profile, - ) + response = self.send_auth_request("put", "/v1/profile", data=profile,) return response def get_account_info(self, offline=False): @@ -187,10 +182,7 @@ def get_account_info(self, offline=False): "username": account.get("username"), } } - token = self.fetch_authentication_token() - result = self.send_request( - "get", "/v1/summary", headers={"Authorization": "Bearer %s" % token}, - ) + result = self.send_auth_request("get", "/v1/summary",) account["summary"] = dict( profile=result.get("profile"), packages=result.get("packages"), @@ -201,6 +193,40 @@ def get_account_info(self, offline=False): app.set_state_item("account", account) return result + def create_org(self, orgname, email, display_name): + response = self.send_auth_request( + "post", + "/v1/orgs", + data={"orgname": orgname, "email": email, "displayname": display_name}, + ) + return response + + def list_orgs(self): + response = self.send_auth_request("get", "/v1/orgs",) + return response + + def update_org(self, orgname, data): + response = self.send_auth_request( + "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v} + ) + return response + + def add_org_owner(self, orgname, username): + response = self.send_auth_request( + "post", "/v1/orgs/%s/owners" % orgname, data={"username": username}, + ) + return response + + def list_org_owners(self, orgname): + response = self.send_auth_request("get", "/v1/orgs/%s/owners" % orgname,) + return response + + def remove_org_owner(self, orgname, username): + response = self.send_auth_request( + "delete", "/v1/orgs/%s/owners" % orgname, data={"username": username}, + ) + return response + def fetch_authentication_token(self): if "PLATFORMIO_AUTH_TOKEN" in os.environ: return os.environ["PLATFORMIO_AUTH_TOKEN"] diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index f90282d8cc..5936d2e994 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -24,6 +24,14 @@ class RegistryClient(RESTClient): def __init__(self): super(RegistryClient, self).__init__(base_url=__registry_api__) + def send_auth_request(self, *args, **kwargs): + headers = kwargs.get("headers", {}) + if "Authorization" not in headers: + token = AccountClient().fetch_authentication_token() + headers["Authorization"] = "Bearer %s" % token + kwargs["headers"] = headers + return self.send_request(*args, **kwargs) + def publish_package( self, archive_path, owner=None, released_at=None, private=False, notify=True ): @@ -33,7 +41,7 @@ def publish_package( account.get_account_info(offline=True).get("profile").get("username") ) with open(archive_path, "rb") as fp: - response = self.send_request( + response = self.send_auth_request( "post", "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), params={ @@ -42,7 +50,6 @@ def publish_package( "released_at": released_at, }, headers={ - "Authorization": "Bearer %s" % account.fetch_authentication_token(), "Content-Type": "application/octet-stream", "X-PIO-Content-SHA256": fs.calculate_file_hashsum( "sha256", archive_path @@ -63,12 +70,7 @@ def unpublish_package( # pylint: disable=redefined-builtin path = "/v3/package/%s/%s/%s" % (owner, type, name) if version: path = path + "/version/" + version - response = self.send_request( - "delete", - path, - params={"undo": 1 if undo else 0}, - headers={ - "Authorization": "Bearer %s" % account.fetch_authentication_token() - }, + response = self.send_auth_request( + "delete", path, params={"undo": 1 if undo else 0}, ) return response diff --git a/platformio/commands/account.py b/platformio/commands/account.py index 78c5aa9eeb..39cca5bdbf 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -29,13 +29,15 @@ def cli(): pass -def validate_username(value): +def validate_username(value, field="username"): value = str(value).strip() - if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){3,38}$", value, flags=re.I): + if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I): raise click.BadParameter( - "Invalid username format. " - "Username must contain at least 4 characters including single hyphens," - " and cannot begin or end with a hyphen" + "Invalid %s format. " + "%s may only contain alphanumeric characters " + "or single hyphens, cannot begin or end with a hyphen, " + "and must not be longer than 38 characters." + % (field.lower(), field.capitalize()) ) return value diff --git a/platformio/commands/org.py b/platformio/commands/org.py new file mode 100644 index 0000000000..2658492355 --- /dev/null +++ b/platformio/commands/org.py @@ -0,0 +1,128 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json + +import click +from tabulate import tabulate + +from platformio.clients.account import AccountClient +from platformio.commands.account import validate_email, validate_username + + +@click.group("org", short_help="Manage Organizations") +def cli(): + pass + + +def validate_orgname(value): + return validate_username(value, "Organization name") + + +@cli.command("create", short_help="Create a new organization") +@click.argument( + "orgname", callback=lambda _, __, value: validate_orgname(value), +) +@click.option( + "--email", callback=lambda _, __, value: validate_email(value) if value else value +) +@click.option("--display-name",) +def org_create(orgname, email, display_name): + client = AccountClient() + client.create_org(orgname, email, display_name) + return click.secho("An organization has been successfully created.", fg="green",) + + +@cli.command("list", short_help="List organizations") +@click.option("--json-output", is_flag=True) +def org_list(json_output): + client = AccountClient() + orgs = client.list_orgs() + if json_output: + return click.echo(json.dumps(orgs)) + click.echo() + click.secho("Organizations", fg="cyan") + click.echo("=" * len("Organizations")) + for org in orgs: + click.echo() + click.secho(org.get("orgname"), bold=True) + click.echo("-" * len(org.get("orgname"))) + data = [] + if org.get("displayname"): + data.append(("Display Name:", org.get("displayname"))) + if org.get("email"): + data.append(("Email:", org.get("email"))) + data.append( + ( + "Owners:", + ", ".join((owner.get("username") for owner in org.get("owners"))), + ) + ) + click.echo(tabulate(data, tablefmt="plain")) + return click.echo() + + +@cli.command("update", short_help="Update organization") +@click.argument("orgname") +@click.option("--new-orgname") +@click.option("--email") +@click.option("--display-name",) +def org_update(orgname, **kwargs): + client = AccountClient() + org = next( + (org for org in client.list_orgs() if org.get("orgname") == orgname), None + ) + if not org: + return click.ClickException("Organization '%s' not found" % orgname) + del org["owners"] + new_org = org.copy() + if not any(kwargs.values()): + for field in org: + new_org[field] = click.prompt( + field.replace("_", " ").capitalize(), default=org[field] + ) + if field == "email": + validate_email(new_org[field]) + if field == "orgname": + validate_orgname(new_org[field]) + else: + new_org.update( + {key.replace("new_", ""): value for key, value in kwargs.items() if value} + ) + client.update_org(orgname, new_org) + return click.secho("An organization has been successfully updated.", fg="green",) + + +@cli.command("add", short_help="Add a new owner to organization") +@click.argument("orgname",) +@click.argument("username",) +def org_add_owner(orgname, username): + client = AccountClient() + client.add_org_owner(orgname, username) + return click.secho( + "A new owner has been successfully added to organization.", fg="green", + ) + + +@cli.command("remove", short_help="Remove an owner from organization") +@click.argument("orgname",) +@click.argument("username",) +def org_remove_owner(orgname, username): + client = AccountClient() + client.remove_org_owner(orgname, username) + return click.secho( + "An owner has been successfully removed from organization.", fg="green", + ) diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py index 5b160f0c93..1be778eb71 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account.py @@ -145,8 +145,12 @@ def test_account_password_change_with_invalid_old_password( ) assert result.exit_code > 0 assert result.exception - assert "Invalid user password" in str(result.exception) - + assert ( + "Invalid request data for new_password -> " + "'Password must contain at least 8 " + "characters including a number and a lowercase letter'" + in str(result.exception) + ) finally: clirunner.invoke(cmd_account, ["logout"]) @@ -174,9 +178,9 @@ def test_account_password_change_with_invalid_new_password_format( assert result.exit_code > 0 assert result.exception assert ( - "Invalid password format. Password must contain at" - " least 8 characters including a number and a lowercase letter" - in str(result.exception) + "Invalid request data for new_password -> " + "'Password must contain at least 8 characters" + " including a number and a lowercase letter'" in str(result.exception) ) finally: diff --git a/tests/commands/test_orgs.py b/tests/commands/test_orgs.py new file mode 100644 index 0000000000..b49f1f0018 --- /dev/null +++ b/tests/commands/test_orgs.py @@ -0,0 +1,191 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +import pytest + +from platformio.commands.account import cli as cmd_account +from platformio.commands.org import cli as cmd_org + +pytestmark = pytest.mark.skipif( + not ( + os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") + and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") + ), + reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", +) + + +@pytest.fixture(scope="session") +def credentials(): + return { + "login": os.environ["PLATFORMIO_TEST_ACCOUNT_LOGIN"], + "password": os.environ["PLATFORMIO_TEST_ACCOUNT_PASSWORD"], + } + + +def test_org_add(clirunner, credentials, validate_cliresult, isolated_pio_home): + try: + result = clirunner.invoke( + cmd_account, + ["login", "-u", credentials["login"], "-p", credentials["password"]], + ) + validate_cliresult(result) + assert "Successfully logged in!" in result.output + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + if len(json_result) < 3: + for i in range(3 - len(json_result)): + result = clirunner.invoke( + cmd_org, + [ + "create", + "%s-%s" % (i, credentials["login"]), + "--email", + "test@test.com", + "--display-name", + "TEST ORG %s" % i, + ], + ) + validate_cliresult(result) + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + finally: + clirunner.invoke(cmd_account, ["logout"]) + + +def test_org_list(clirunner, credentials, validate_cliresult, isolated_pio_home): + try: + result = clirunner.invoke( + cmd_account, + ["login", "-u", credentials["login"], "-p", credentials["password"]], + ) + validate_cliresult(result) + assert "Successfully logged in!" in result.output + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + check = False + for org in json_result: + assert "orgname" in org + assert "displayname" in org + assert "email" in org + assert "owners" in org + for owner in org.get("owners"): + assert "username" in owner + check = owner["username"] == credentials["login"] if not check else True + assert "firstname" in owner + assert "lastname" in owner + assert check + finally: + clirunner.invoke(cmd_account, ["logout"]) + + +def test_org_update(clirunner, credentials, validate_cliresult, isolated_pio_home): + try: + result = clirunner.invoke( + cmd_account, + ["login", "-u", credentials["login"], "-p", credentials["password"]], + ) + validate_cliresult(result) + assert "Successfully logged in!" in result.output + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + org = json_result[0] + assert "orgname" in org + assert "displayname" in org + assert "email" in org + assert "owners" in org + + old_orgname = org["orgname"] + if len(old_orgname) > 10: + new_orgname = "neworg" + org["orgname"][6:] + + result = clirunner.invoke( + cmd_org, ["update", old_orgname, "--new-orgname", new_orgname], + ) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_org, ["update", new_orgname, "--new-orgname", old_orgname], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + assert json.loads(result.output.strip()) == json_result + finally: + clirunner.invoke(cmd_account, ["logout"]) + + +def test_org_owner(clirunner, credentials, validate_cliresult, isolated_pio_home): + try: + result = clirunner.invoke( + cmd_account, + ["login", "-u", credentials["login"], "-p", credentials["password"]], + ) + validate_cliresult(result) + assert "Successfully logged in!" in result.output + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + org = json_result[0] + assert "orgname" in org + assert "displayname" in org + assert "email" in org + assert "owners" in org + + result = clirunner.invoke(cmd_org, ["add", org["orgname"], "platformio"],) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + check = False + for item in json_result: + if item["orgname"] != org["orgname"]: + continue + for owner in item.get("owners"): + check = owner["username"] == "platformio" if not check else True + assert check + + result = clirunner.invoke(cmd_org, ["remove", org["orgname"], "platformio"],) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) == 3 + check = False + for item in json_result: + if item["orgname"] != org["orgname"]: + continue + for owner in item.get("owners"): + check = owner["username"] == "platformio" if not check else True + assert not check + + finally: + clirunner.invoke(cmd_account, ["logout"]) From f7dceb782cee937d3ecb12268f259932815a95b7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 21:24:01 +0300 Subject: [PATCH 024/223] Fix PY2.7 when PermissionError is not avialable --- platformio/maintenance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 5f27573600..16712872bb 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -141,7 +141,7 @@ def after_upgrade(ctx): click.secho("Please wait while upgrading PlatformIO...", fg="yellow") try: app.clean_cache() - except PermissionError: + except: # pylint: disable=bare-except pass # Update PlatformIO's Core packages From cbcd3f7c4d085e9a23c9d191b03bb21a7d735dfd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 21:40:03 +0300 Subject: [PATCH 025/223] Fix cmd.org test --- tests/commands/test_orgs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/commands/test_orgs.py b/tests/commands/test_orgs.py index b49f1f0018..33f5291b38 100644 --- a/tests/commands/test_orgs.py +++ b/tests/commands/test_orgs.py @@ -157,7 +157,7 @@ def test_org_owner(clirunner, credentials, validate_cliresult, isolated_pio_home assert "email" in org assert "owners" in org - result = clirunner.invoke(cmd_org, ["add", org["orgname"], "platformio"],) + result = clirunner.invoke(cmd_org, ["add", org["orgname"], "ivankravets"],) validate_cliresult(result) result = clirunner.invoke(cmd_org, ["list", "--json-output"],) @@ -169,10 +169,10 @@ def test_org_owner(clirunner, credentials, validate_cliresult, isolated_pio_home if item["orgname"] != org["orgname"]: continue for owner in item.get("owners"): - check = owner["username"] == "platformio" if not check else True + check = owner["username"] == "ivankravets" if not check else True assert check - result = clirunner.invoke(cmd_org, ["remove", org["orgname"], "platformio"],) + result = clirunner.invoke(cmd_org, ["remove", org["orgname"], "ivankravets"],) validate_cliresult(result) result = clirunner.invoke(cmd_org, ["list", "--json-output"],) @@ -184,7 +184,7 @@ def test_org_owner(clirunner, credentials, validate_cliresult, isolated_pio_home if item["orgname"] != org["orgname"]: continue for owner in item.get("owners"): - check = owner["username"] == "platformio" if not check else True + check = owner["username"] == "ivankravets" if not check else True assert not check finally: From 6c97cc61928af4f2433ce2f6a7d6ff1fdd9ec9c2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 22:22:13 +0300 Subject: [PATCH 026/223] Cosmetic changes to Org CLI --- platformio/commands/org.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/platformio/commands/org.py b/platformio/commands/org.py index 2658492355..5cf0b13b49 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -53,12 +53,9 @@ def org_list(json_output): orgs = client.list_orgs() if json_output: return click.echo(json.dumps(orgs)) - click.echo() - click.secho("Organizations", fg="cyan") - click.echo("=" * len("Organizations")) for org in orgs: click.echo() - click.secho(org.get("orgname"), bold=True) + click.secho(org.get("orgname"), fg="cyan") click.echo("-" * len(org.get("orgname"))) data = [] if org.get("displayname"): @@ -113,7 +110,7 @@ def org_add_owner(orgname, username): client = AccountClient() client.add_org_owner(orgname, username) return click.secho( - "A new owner has been successfully added to organization.", fg="green", + "A new owner has been successfully added to the organization.", fg="green", ) @@ -124,5 +121,5 @@ def org_remove_owner(orgname, username): client = AccountClient() client.remove_org_owner(orgname, username) return click.secho( - "An owner has been successfully removed from organization.", fg="green", + "An owner has been successfully removed from the organization.", fg="green", ) From 87b5fbd237651ec7e7da35d040cde8d69a9803d1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 22:34:37 +0300 Subject: [PATCH 027/223] More cosmetic changes to Org CLI --- platformio/commands/account.py | 2 +- platformio/commands/org.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/platformio/commands/account.py b/platformio/commands/account.py index 39cca5bdbf..67603224a9 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -34,7 +34,7 @@ def validate_username(value, field="username"): if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I): raise click.BadParameter( "Invalid %s format. " - "%s may only contain alphanumeric characters " + "%s must only contain alphanumeric characters " "or single hyphens, cannot begin or end with a hyphen, " "and must not be longer than 38 characters." % (field.lower(), field.capitalize()) diff --git a/platformio/commands/org.py b/platformio/commands/org.py index 5cf0b13b49..d5f7a2f9af 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -43,7 +43,9 @@ def validate_orgname(value): def org_create(orgname, email, display_name): client = AccountClient() client.create_org(orgname, email, display_name) - return click.secho("An organization has been successfully created.", fg="green",) + return click.secho( + "The organization %s has been successfully created." % orgname, fg="green", + ) @cli.command("list", short_help="List organizations") @@ -100,7 +102,9 @@ def org_update(orgname, **kwargs): {key.replace("new_", ""): value for key, value in kwargs.items() if value} ) client.update_org(orgname, new_org) - return click.secho("An organization has been successfully updated.", fg="green",) + return click.secho( + "The organization %s has been successfully updated." % orgname, fg="green", + ) @cli.command("add", short_help="Add a new owner to organization") @@ -110,7 +114,9 @@ def org_add_owner(orgname, username): client = AccountClient() client.add_org_owner(orgname, username) return click.secho( - "A new owner has been successfully added to the organization.", fg="green", + "The new owner %s has been successfully added to the %s organization." + % (username, orgname), + fg="green", ) @@ -121,5 +127,7 @@ def org_remove_owner(orgname, username): client = AccountClient() client.remove_org_owner(orgname, username) return click.secho( - "An owner has been successfully removed from the organization.", fg="green", + "The %s owner has been successfully removed from the %s organization." + % (username, orgname), + fg="green", ) From d7f4eb59558da803a228f23b1d620d70b96b3b61 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jun 2020 22:40:37 +0300 Subject: [PATCH 028/223] Minor grammar fix --- platformio/commands/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/account.py b/platformio/commands/account.py index 67603224a9..c254dbee46 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -34,7 +34,7 @@ def validate_username(value, field="username"): if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I): raise click.BadParameter( "Invalid %s format. " - "%s must only contain alphanumeric characters " + "%s must contain only alphanumeric characters " "or single hyphens, cannot begin or end with a hyphen, " "and must not be longer than 38 characters." % (field.lower(), field.capitalize()) From 3c1b08daab2eba6f8a546e34f299a535fc29a510 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jun 2020 13:57:56 +0300 Subject: [PATCH 029/223] Ignore empty PLATFORMIO_AUTH_TOKEN --- platformio/clients/account.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 9e0e65812a..084110da8f 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -228,8 +228,8 @@ def remove_org_owner(self, orgname, username): return response def fetch_authentication_token(self): - if "PLATFORMIO_AUTH_TOKEN" in os.environ: - return os.environ["PLATFORMIO_AUTH_TOKEN"] + if os.environ.get("PLATFORMIO_AUTH_TOKEN"): + return os.environ.get("PLATFORMIO_AUTH_TOKEN") auth = app.get_state_item("account", {}).get("auth", {}) if auth.get("access_token") and auth.get("access_token_expire"): if auth.get("access_token_expire") > time.time(): From 0c4c113b0a1d3e5e09c0c1079f685272f8195478 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jun 2020 14:09:42 +0300 Subject: [PATCH 030/223] Fix account shpw command when PLATFORMIO_AUTH_TOKEN is used --- platformio/clients/account.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 084110da8f..545595c1b2 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -167,15 +167,13 @@ def update_profile(self, profile, current_password): return response def get_account_info(self, offline=False): - account = app.get_state_item("account") - if not account: - raise AccountNotAuthorized() + account = app.get_state_item("account") or {} if ( account.get("summary") and account["summary"].get("expire_at", 0) > time.time() ): return account["summary"] - if offline: + if offline and account.get("email"): return { "profile": { "email": account.get("email"), From 42df3c9c3fca2106718ac56f084c6fce4f94e959 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jun 2020 15:27:46 +0300 Subject: [PATCH 031/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 5071271dff..b993a3e8cc 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5071271dff8a7162489fc43fb6d98966d2593c09 +Subproject commit b993a3e8ccbb0fe0c5103f667e4b26d7b1c87502 From 94cb8082856331ae6b9ec0b5b45009d5b4284c4c Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Thu, 4 Jun 2020 19:31:30 +0300 Subject: [PATCH 032/223] CLI to manage teams. Resolve #3533 (#3547) * CLI to manage teams.Minor fixes. Resolve #3533 * fix teams tests * disable org and team tests * minor fixes. fix error texts * fix split compatibility --- platformio/clients/account.py | 70 +++++++++--- platformio/commands/org.py | 12 +- platformio/commands/team.py | 201 ++++++++++++++++++++++++++++++++++ tests/commands/test_orgs.py | 106 ++++++------------ tests/commands/test_teams.py | 158 ++++++++++++++++++++++++++ 5 files changed, 453 insertions(+), 94 deletions(-) create mode 100644 platformio/commands/team.py create mode 100644 tests/commands/test_teams.py diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 545595c1b2..9534777d71 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -115,12 +115,11 @@ def logout(self): return True def change_password(self, old_password, new_password): - self.send_auth_request( + return self.send_auth_request( "post", "/v1/password", data={"old_password": old_password, "new_password": new_password}, ) - return True def registration( self, username, email, password, firstname, lastname @@ -147,12 +146,11 @@ def registration( ) def auth_token(self, password, regenerate): - result = self.send_auth_request( + return self.send_auth_request( "post", "/v1/token", data={"password": password, "regenerate": 1 if regenerate else 0}, - ) - return result.get("auth_token") + ).get("auth_token") def forgot_password(self, username): return self.send_request("post", "/v1/forgot", data={"username": username},) @@ -192,38 +190,76 @@ def get_account_info(self, offline=False): return result def create_org(self, orgname, email, display_name): - response = self.send_auth_request( + return self.send_auth_request( "post", "/v1/orgs", data={"orgname": orgname, "email": email, "displayname": display_name}, ) - return response + + def get_org(self, orgname): + return self.send_auth_request("get", "/v1/orgs/%s" % orgname) def list_orgs(self): - response = self.send_auth_request("get", "/v1/orgs",) - return response + return self.send_auth_request("get", "/v1/orgs",) def update_org(self, orgname, data): - response = self.send_auth_request( + return self.send_auth_request( "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v} ) - return response def add_org_owner(self, orgname, username): - response = self.send_auth_request( + return self.send_auth_request( "post", "/v1/orgs/%s/owners" % orgname, data={"username": username}, ) - return response def list_org_owners(self, orgname): - response = self.send_auth_request("get", "/v1/orgs/%s/owners" % orgname,) - return response + return self.send_auth_request("get", "/v1/orgs/%s/owners" % orgname,) def remove_org_owner(self, orgname, username): - response = self.send_auth_request( + return self.send_auth_request( "delete", "/v1/orgs/%s/owners" % orgname, data={"username": username}, ) - return response + + def create_team(self, orgname, teamname, description): + return self.send_auth_request( + "post", + "/v1/orgs/%s/teams" % orgname, + data={"name": teamname, "description": description}, + ) + + def destroy_team(self, orgname, teamname): + return self.send_auth_request( + "delete", "/v1/orgs/%s/teams/%s" % (orgname, teamname), + ) + + def get_team(self, orgname, teamname): + return self.send_auth_request( + "get", "/v1/orgs/%s/teams/%s" % (orgname, teamname), + ) + + def list_teams(self, orgname): + return self.send_auth_request("get", "/v1/orgs/%s/teams" % orgname,) + + def update_team(self, orgname, teamname, data): + return self.send_auth_request( + "put", + "/v1/orgs/%s/teams/%s" % (orgname, teamname), + data={k: v for k, v in data.items() if v}, + ) + + def add_team_member(self, orgname, teamname, username): + return self.send_auth_request( + "post", + "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), + data={"username": username}, + ) + + def remove_team_member(self, orgname, teamname, username): + return self.send_auth_request( + "delete", + "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), + data={"username": username}, + ) def fetch_authentication_token(self): if os.environ.get("PLATFORMIO_AUTH_TOKEN"): diff --git a/platformio/commands/org.py b/platformio/commands/org.py index d5f7a2f9af..7d62120f90 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -55,6 +55,8 @@ def org_list(json_output): orgs = client.list_orgs() if json_output: return click.echo(json.dumps(orgs)) + if not orgs: + return click.echo("You do not have any organizations") for org in orgs: click.echo() click.secho(org.get("orgname"), fg="cyan") @@ -76,16 +78,14 @@ def org_list(json_output): @cli.command("update", short_help="Update organization") @click.argument("orgname") -@click.option("--new-orgname") +@click.option( + "--new-orgname", callback=lambda _, __, value: validate_orgname(value), +) @click.option("--email") @click.option("--display-name",) def org_update(orgname, **kwargs): client = AccountClient() - org = next( - (org for org in client.list_orgs() if org.get("orgname") == orgname), None - ) - if not org: - return click.ClickException("Organization '%s' not found" % orgname) + org = client.get_org(orgname) del org["owners"] new_org = org.copy() if not any(kwargs.values()): diff --git a/platformio/commands/team.py b/platformio/commands/team.py new file mode 100644 index 0000000000..5461cabd4e --- /dev/null +++ b/platformio/commands/team.py @@ -0,0 +1,201 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json +import re + +import click +from tabulate import tabulate + +from platformio.clients.account import AccountClient + + +def validate_orgname_teamname(value, teamname_validate=False): + if ":" not in value: + raise click.BadParameter( + "Please specify organization and team name in the next" + " format - orgname:teamname. For example, mycompany:DreamTeam" + ) + teamname = str(value.strip().split(":", 1)[1]) + if teamname_validate: + validate_teamname(teamname) + return value + + +def validate_teamname(value): + if not value: + return value + value = str(value).strip() + if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I): + raise click.BadParameter( + "Invalid team name format. " + "Team name must only contain alphanumeric characters, " + "single hyphens, underscores, spaces. It can not " + "begin or end with a hyphen or a underscore and must" + " not be longer than 20 characters." + ) + return value + + +@click.group("team", short_help="Manage Teams") +def cli(): + pass + + +@cli.command("create", short_help="Create a new team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname( + value, teamname_validate=True + ), +) +@click.option("--description",) +def team_create(orgname_teamname, description): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.create_team(orgname, teamname, description) + return click.secho( + "The team %s has been successfully created." % teamname, fg="green", + ) + + +@cli.command("list", short_help="List teams") +@click.argument("orgname", required=False) +@click.option("--json-output", is_flag=True) +def team_list(orgname, json_output): + client = AccountClient() + data = {} + if not orgname: + for item in client.list_orgs(): + teams = client.list_teams(item.get("orgname")) + data[item.get("orgname")] = teams + else: + teams = client.list_teams(orgname) + data[orgname] = teams + if json_output: + return click.echo(json.dumps(data[orgname] if orgname else data)) + if not any(data.values()): + return click.secho("You do not have any teams.", fg="yellow") + for org_name in data: + for team in data[org_name]: + click.echo() + click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan") + click.echo("-" * len("%s:%s" % (org_name, team.get("name")))) + table_data = [] + if team.get("description"): + table_data.append(("Description:", team.get("description"))) + table_data.append( + ( + "Members:", + ", ".join( + (member.get("username") for member in team.get("members")) + ) + if team.get("members") + else "-", + ) + ) + click.echo(tabulate(table_data, tablefmt="plain")) + return click.echo() + + +@cli.command("update", short_help="Update team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.option( + "--name", callback=lambda _, __, value: validate_teamname(value), +) +@click.option("--description",) +def team_update(orgname_teamname, **kwargs): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + team = client.get_team(orgname, teamname) + del team["id"] + del team["members"] + new_team = team.copy() + if not any(kwargs.values()): + for field in team: + new_team[field] = click.prompt( + field.replace("_", " ").capitalize(), default=team[field] + ) + if field == "name": + validate_teamname(new_team[field]) + else: + new_team.update({key: value for key, value in kwargs.items() if value}) + client.update_team(orgname, teamname, new_team) + return click.secho( + "The team %s has been successfully updated." % teamname, fg="green", + ) + + +@cli.command("destroy", short_help="Destroy a team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +def team_destroy(orgname_teamname): + orgname, teamname = orgname_teamname.split(":", 1) + click.confirm( + click.style( + "Are you sure you want to destroy the %s team?" % teamname, fg="yellow" + ), + abort=True, + ) + client = AccountClient() + client.destroy_team(orgname, teamname) + return click.secho( + "The team %s has been successfully destroyed." % teamname, fg="green", + ) + + +@cli.command("add", short_help="Add a new member to team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.argument("username",) +def team_add_member(orgname_teamname, username): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.add_team_member(orgname, teamname, username) + return click.secho( + "The new member %s has been successfully added to the %s team." + % (username, teamname), + fg="green", + ) + + +@cli.command("remove", short_help="Remove a member from team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.argument("username",) +def org_remove_owner(orgname_teamname, username): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.remove_team_member(orgname, teamname, username) + return click.secho( + "The %s member has been successfully removed from the %s team." + % (username, teamname), + fg="green", + ) diff --git a/tests/commands/test_orgs.py b/tests/commands/test_orgs.py index 33f5291b38..4650caaf66 100644 --- a/tests/commands/test_orgs.py +++ b/tests/commands/test_orgs.py @@ -37,7 +37,7 @@ def credentials(): } -def test_org_add(clirunner, credentials, validate_cliresult, isolated_pio_home): +def test_orgs(clirunner, credentials, validate_cliresult, isolated_pio_home): try: result = clirunner.invoke( cmd_account, @@ -66,26 +66,11 @@ def test_org_add(clirunner, credentials, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_org, ["list", "--json-output"],) validate_cliresult(result) json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_org_list(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - assert "Successfully logged in!" in result.output - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 + assert len(json_result) >= 3 check = False for org in json_result: assert "orgname" in org + orgname = org["orgname"] assert "displayname" in org assert "email" in org assert "owners" in org @@ -95,10 +80,41 @@ def test_org_list(clirunner, credentials, validate_cliresult, isolated_pio_home) assert "firstname" in owner assert "lastname" in owner assert check + + result = clirunner.invoke(cmd_org, ["add", orgname, "ivankravets"],) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) >= 3 + check = False + for item in json_result: + if item["orgname"] != orgname: + continue + for owner in item.get("owners"): + check = owner["username"] == "ivankravets" if not check else True + assert check + + result = clirunner.invoke(cmd_org, ["remove", orgname, "ivankravets"],) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) >= 3 + check = False + for item in json_result: + if item["orgname"] != orgname: + continue + for owner in item.get("owners"): + check = owner["username"] == "ivankravets" if not check else True + assert not check finally: clirunner.invoke(cmd_account, ["logout"]) +@pytest.mark.skip def test_org_update(clirunner, credentials, validate_cliresult, isolated_pio_home): try: result = clirunner.invoke( @@ -111,7 +127,7 @@ def test_org_update(clirunner, credentials, validate_cliresult, isolated_pio_hom result = clirunner.invoke(cmd_org, ["list", "--json-output"],) validate_cliresult(result) json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 + assert len(json_result) >= 3 org = json_result[0] assert "orgname" in org assert "displayname" in org @@ -137,55 +153,3 @@ def test_org_update(clirunner, credentials, validate_cliresult, isolated_pio_hom assert json.loads(result.output.strip()) == json_result finally: clirunner.invoke(cmd_account, ["logout"]) - - -def test_org_owner(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - assert "Successfully logged in!" in result.output - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 - org = json_result[0] - assert "orgname" in org - assert "displayname" in org - assert "email" in org - assert "owners" in org - - result = clirunner.invoke(cmd_org, ["add", org["orgname"], "ivankravets"],) - validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 - check = False - for item in json_result: - if item["orgname"] != org["orgname"]: - continue - for owner in item.get("owners"): - check = owner["username"] == "ivankravets" if not check else True - assert check - - result = clirunner.invoke(cmd_org, ["remove", org["orgname"], "ivankravets"],) - validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) == 3 - check = False - for item in json_result: - if item["orgname"] != org["orgname"]: - continue - for owner in item.get("owners"): - check = owner["username"] == "ivankravets" if not check else True - assert not check - - finally: - clirunner.invoke(cmd_account, ["logout"]) diff --git a/tests/commands/test_teams.py b/tests/commands/test_teams.py new file mode 100644 index 0000000000..13e30ce016 --- /dev/null +++ b/tests/commands/test_teams.py @@ -0,0 +1,158 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import time + +import pytest + +from platformio.commands.account import cli as cmd_account +from platformio.commands.org import cli as cmd_org +from platformio.commands.team import cli as cmd_team + +pytestmark = pytest.mark.skipif( + not ( + os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") + and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") + ), + reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", +) + + +@pytest.fixture(scope="session") +def credentials(): + return { + "login": os.environ["PLATFORMIO_TEST_ACCOUNT_LOGIN"], + "password": os.environ["PLATFORMIO_TEST_ACCOUNT_PASSWORD"], + } + + +def test_teams(clirunner, credentials, validate_cliresult, isolated_pio_home): + orgname = "" + teamname = "test-" + str(int(time.time() * 1000)) + try: + result = clirunner.invoke( + cmd_account, + ["login", "-u", credentials["login"], "-p", credentials["password"]], + ) + validate_cliresult(result) + assert "Successfully logged in!" in result.output + + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + if len(json_result) < 3: + for i in range(3 - len(json_result)): + result = clirunner.invoke( + cmd_org, + [ + "create", + "%s-%s" % (i, credentials["login"]), + "--email", + "test@test.com", + "--display-name", + "TEST ORG %s" % i, + ], + ) + validate_cliresult(result) + result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) >= 3 + orgname = json_result[0].get("orgname") + + result = clirunner.invoke( + cmd_team, + [ + "create", + "%s:%s" % (orgname, teamname), + "--description", + "team for CI test", + ], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) >= 1 + check = False + for team in json_result: + assert team["id"] + assert team["name"] + if team["name"] == teamname: + check = True + assert "description" in team + assert "members" in team + assert check + + result = clirunner.invoke( + cmd_team, ["add", "%s:%s" % (orgname, teamname), credentials["login"]], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + check = False + for team in json_result: + assert team["id"] + assert team["name"] + assert "description" in team + assert "members" in team + if ( + len(team["members"]) > 0 + and team["members"][0]["username"] == credentials["login"] + ): + check = True + assert check + + result = clirunner.invoke( + cmd_team, ["remove", "%s:%s" % (orgname, teamname), credentials["login"]], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_team, + [ + "update", + "%s:%s" % (orgname, teamname), + "--description", + "Updated Description", + ], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert len(json_result) >= 1 + check = False + for team in json_result: + assert team["id"] + assert team["name"] + assert "description" in team + if team.get("description") == "Updated Description": + check = True + assert "members" in team + assert check + finally: + clirunner.invoke( + cmd_team, ["destroy", "%s:%s" % (orgname, teamname),], + ) + clirunner.invoke(cmd_account, ["logout"]) From 6fa7cb4af530125b34be5d794ee4516db2ed70ed Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jun 2020 22:59:05 +0300 Subject: [PATCH 033/223] Add new dev-platform "ASR Microelectronics ASR605x" --- README.rst | 1 + docs | 2 +- examples | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 44c251e0f1..fe7345d5e2 100644 --- a/README.rst +++ b/README.rst @@ -81,6 +81,7 @@ Development Platforms --------------------- * `Aceinna IMU `_ +* `ASR Microelectronics ASR605x `_ * `Atmel AVR `_ * `Atmel SAM `_ * `Espressif 32 `_ diff --git a/docs b/docs index b993a3e8cc..b6bc6eb15f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b993a3e8ccbb0fe0c5103f667e4b26d7b1c87502 +Subproject commit b6bc6eb15fdf9cf0d055eeae9d114ddd9e9d6e0d diff --git a/examples b/examples index 7793b677f7..c442de34a5 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 7793b677f72ce3c3e9ed92b7915859ca2bfa313f +Subproject commit c442de34a57b54451170dbe39f3411a06a05b3f2 From ced244d30a10885c98fb11804ec365f787238812 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Fri, 5 Jun 2020 11:30:15 +0300 Subject: [PATCH 034/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b6bc6eb15f..1bf316215c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b6bc6eb15fdf9cf0d055eeae9d114ddd9e9d6e0d +Subproject commit 1bf316215c614f6dd4a7502e990137b005043212 From 27fd3b0b14995dc87ab345c954d150d14ec0177c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Jun 2020 14:17:19 +0300 Subject: [PATCH 035/223] Improve detecting if PlatformIO Core is run in container --- platformio/app.py | 11 +++++++---- platformio/maintenance.py | 5 ++--- platformio/proc.py | 28 ++++++++++++++-------------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 6c7c7b1ad0..f4ae15b9d9 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,9 +25,8 @@ import requests -from platformio import __version__, exception, fs, lockfile +from platformio import __version__, exception, fs, lockfile, proc from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data -from platformio.proc import is_ci from platformio.project.helpers import ( get_default_projects_dir, get_project_cache_dir, @@ -383,7 +382,7 @@ def is_disabled_progressbar(): return any( [ get_session_var("force_option"), - is_ci(), + proc.is_ci(), getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", ] ) @@ -420,7 +419,11 @@ def get_cid(): def get_user_agent(): - data = ["PlatformIO/%s" % __version__, "CI/%d" % int(is_ci())] + data = [ + "PlatformIO/%s" % __version__, + "CI/%d" % int(proc.is_ci()), + "Container/%d" % int(proc.is_container()), + ] if get_session_var("caller_id"): data.append("Caller/%s" % get_session_var("caller_id")) if os.getenv("PLATFORMIO_IDE"): diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 16712872bb..0c8ee2dfa0 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -66,10 +66,9 @@ def on_platformio_exception(e): def set_caller(caller=None): + caller = caller or getenv("PLATFORMIO_CALLER") if not caller: - if getenv("PLATFORMIO_CALLER"): - caller = getenv("PLATFORMIO_CALLER") - elif getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"): + if getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"): caller = "vscode" elif is_container(): if getenv("C9_UID"): diff --git a/platformio/proc.py b/platformio/proc.py index 80e50201a0..04f15a579c 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -15,7 +15,6 @@ import os import subprocess import sys -from os.path import isdir, isfile, join, normpath from threading import Thread from platformio import exception @@ -143,18 +142,16 @@ def is_ci(): def is_container(): - if not isfile("/proc/1/cgroup"): + if os.path.exists("/.dockerenv"): + return True + if not os.path.isfile("/proc/1/cgroup"): return False with open("/proc/1/cgroup") as fp: - for line in fp: - line = line.strip() - if ":" in line and not line.endswith(":/"): - return True - return False + return ":/docker/" in fp.read() def get_pythonexe_path(): - return os.environ.get("PYTHONEXEPATH", normpath(sys.executable)) + return os.environ.get("PYTHONEXEPATH", os.path.normpath(sys.executable)) def copy_pythonpath_to_osenv(): @@ -164,7 +161,10 @@ def copy_pythonpath_to_osenv(): for p in os.sys.path: conditions = [p not in _PYTHONPATH] if not WINDOWS: - conditions.append(isdir(join(p, "click")) or isdir(join(p, "platformio"))) + conditions.append( + os.path.isdir(os.path.join(p, "click")) + or os.path.isdir(os.path.join(p, "platformio")) + ) if all(conditions): _PYTHONPATH.append(p) os.environ["PYTHONPATH"] = os.pathsep.join(_PYTHONPATH) @@ -178,16 +178,16 @@ def where_is_program(program, envpath=None): # try OS's built-in commands try: result = exec_command(["where" if WINDOWS else "which", program], env=env) - if result["returncode"] == 0 and isfile(result["out"].strip()): + if result["returncode"] == 0 and os.path.isfile(result["out"].strip()): return result["out"].strip() except OSError: pass # look up in $PATH for bin_dir in env.get("PATH", "").split(os.pathsep): - if isfile(join(bin_dir, program)): - return join(bin_dir, program) - if isfile(join(bin_dir, "%s.exe" % program)): - return join(bin_dir, "%s.exe" % program) + if os.path.isfile(os.path.join(bin_dir, program)): + return os.path.join(bin_dir, program) + if os.path.isfile(os.path.join(bin_dir, "%s.exe" % program)): + return os.path.join(bin_dir, "%s.exe" % program) return program From f5e6820903ba960184fe4ecd9f5223f140cc0fa0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Jun 2020 14:18:24 +0300 Subject: [PATCH 036/223] Bump version to 4.3.5a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 60621751c8..b4db9ecf71 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, "5a1") +VERSION = (4, 3, "5a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From e0e97a36297852128016b2ba8360d77f64b276db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Jun 2020 18:29:11 +0300 Subject: [PATCH 037/223] Cache the latest news in PIO Home for 180 days --- platformio/commands/home/rpc/handlers/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index d1851d1397..a216344e96 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -24,7 +24,7 @@ class MiscRPC(object): def load_latest_tweets(self, data_url): cache_key = app.ContentCache.key_from_args(data_url, "tweets") - cache_valid = "7d" + cache_valid = "180d" with app.ContentCache() as cc: cache_data = cc.get(cache_key) if cache_data: From 7457ef043b91e3eaf3c6001da66b95120c265a98 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 Jun 2020 12:00:19 +0300 Subject: [PATCH 038/223] Docs: Sync ASR Micro dev-platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 1bf316215c..af312fb70f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1bf316215c614f6dd4a7502e990137b005043212 +Subproject commit af312fb70f3bf528d6f59169dc92f088764eb2a4 From 78546e9246d515d32b66dc12957bdc1368332138 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 Jun 2020 19:26:48 +0300 Subject: [PATCH 039/223] Docs: Add "TensorFlow, Meet The ESP32" to articles list --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index af312fb70f..cfba4f4568 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit af312fb70f3bf528d6f59169dc92f088764eb2a4 +Subproject commit cfba4f456843f1069d9ebd083e359186054c8659 From a5547491edccaca360f6f1fc4b0e095383be0726 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Tue, 9 Jun 2020 15:50:37 +0300 Subject: [PATCH 040/223] Add account and org destroy commands. Fix tests (#3552) * Add account and org destroy commands. Fix tests * fix tests * fix * fix texts --- .github/workflows/core.yml | 5 +- platformio/clients/account.py | 10 +- platformio/commands/account.py | 17 + platformio/commands/org.py | 22 +- tests/commands/test_account.py | 584 +++++++-------------------------- tests/commands/test_orgs.py | 218 ++++++------ tests/commands/test_teams.py | 191 ++++++----- tests/conftest.py | 43 +++ 8 files changed, 432 insertions(+), 658 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index c84a97d71d..c2e9547c71 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -28,8 +28,9 @@ jobs: tox -e lint - name: Integration Tests env: - PLATFORMIO_TEST_ACCOUNT_LOGIN: ${{ secrets.PLATFORMIO_TEST_ACCOUNT_LOGIN }} - PLATFORMIO_TEST_ACCOUNT_PASSWORD: ${{ secrets.PLATFORMIO_TEST_ACCOUNT_PASSWORD }} + TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }} + TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }} + TEST_EMAIL_POP3_SERVER: ${{ secrets.TEST_EMAIL_POP3_SERVER }} run: | tox -e testcore diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 9534777d71..31e34f7889 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -189,11 +189,14 @@ def get_account_info(self, offline=False): app.set_state_item("account", account) return result - def create_org(self, orgname, email, display_name): + def destroy_account(self): + return self.send_auth_request("delete", "/v1/account") + + def create_org(self, orgname, email, displayname): return self.send_auth_request( "post", "/v1/orgs", - data={"orgname": orgname, "email": email, "displayname": display_name}, + data={"orgname": orgname, "email": email, "displayname": displayname}, ) def get_org(self, orgname): @@ -207,6 +210,9 @@ def update_org(self, orgname, data): "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v} ) + def destroy_org(self, orgname): + return self.send_auth_request("delete", "/v1/orgs/%s" % orgname,) + def add_org_owner(self, orgname, username): return self.send_auth_request( "post", "/v1/orgs/%s/owners" % orgname, data={"username": username}, diff --git a/platformio/commands/account.py b/platformio/commands/account.py index c254dbee46..3a1492ec23 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -178,6 +178,23 @@ def account_update(current_password, **kwargs): return click.secho("Please re-login.", fg="yellow") +@cli.command("destroy", short_help="Destroy account") +def account_destroy(): + client = AccountClient() + click.confirm( + "Are you sure you want to delete the %s user account?\n" + "Warning! All linked data will be permanently removed and can not be restored." + % client.get_account_info().get("profile").get("username"), + abort=True, + ) + client.destroy_account() + try: + client.logout() + except AccountNotAuthorized: + pass + return click.secho("User account has been destroyed.", fg="green",) + + @cli.command("show", short_help="PIO Account information") @click.option("--offline", is_flag=True) @click.option("--json-output", is_flag=True) diff --git a/platformio/commands/org.py b/platformio/commands/org.py index 7d62120f90..a7e0f1e974 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -39,10 +39,10 @@ def validate_orgname(value): @click.option( "--email", callback=lambda _, __, value: validate_email(value) if value else value ) -@click.option("--display-name",) -def org_create(orgname, email, display_name): +@click.option("--displayname",) +def org_create(orgname, email, displayname): client = AccountClient() - client.create_org(orgname, email, display_name) + client.create_org(orgname, email, displayname) return click.secho( "The organization %s has been successfully created." % orgname, fg="green", ) @@ -82,7 +82,7 @@ def org_list(json_output): "--new-orgname", callback=lambda _, __, value: validate_orgname(value), ) @click.option("--email") -@click.option("--display-name",) +@click.option("--displayname",) def org_update(orgname, **kwargs): client = AccountClient() org = client.get_org(orgname) @@ -107,6 +107,20 @@ def org_update(orgname, **kwargs): ) +@cli.command("destroy", short_help="Destroy organization") +@click.argument("orgname") +def account_destroy(orgname): + client = AccountClient() + click.confirm( + "Are you sure you want to delete the %s organization account?\n" + "Warning! All linked data will be permanently removed and can not be restored." + % orgname, + abort=True, + ) + client.destroy_org(orgname) + return click.secho("Organization %s has been destroyed." % orgname, fg="green",) + + @cli.command("add", short_help="Add a new owner to organization") @click.argument("orgname",) @click.argument("username",) diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py index 1be778eb71..221b724a7e 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account.py @@ -17,34 +17,29 @@ import time import pytest +import requests from platformio.commands.account import cli as cmd_account +from platformio.commands.package import cli as cmd_package +from platformio.downloader import FileDownloader +from platformio.unpacker import FileUnpacker -pytestmark = pytest.mark.skipif( - not ( - os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") - and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") - ), - reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", -) - -@pytest.fixture(scope="session") -def credentials(): - return { - "login": os.environ["PLATFORMIO_TEST_ACCOUNT_LOGIN"], - "password": os.environ["PLATFORMIO_TEST_ACCOUNT_PASSWORD"], - } - - -def test_account_register_with_already_exists_username( - clirunner, credentials, isolated_pio_home +@pytest.mark.skipif( + not os.environ.get("TEST_EMAIL_LOGIN"), + reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", +) # pylint:disable=too-many-arguments +def test_account( + clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory ): - username = credentials["login"] - email = "test@test.com" - if "@" in credentials["login"]: - username = "Testusername" - email = credentials["login"] + username = "test-piocore-%s" % str(int(time.time() * 1000)) + splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") + email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) + firstname = "Test" + lastname = "User" + password = "Qwerty123!" + + # pio account register result = clirunner.invoke( cmd_account, [ @@ -54,345 +49,33 @@ def test_account_register_with_already_exists_username( "-e", email, "-p", - credentials["password"], + password, "--firstname", - "First", + firstname, "--lastname", - "Last", + lastname, ], ) - assert result.exit_code > 0 - assert result.exception - assert "User with same username already exists" in str( - result.exception - ) or "User with same email already exists" in str(result.exception) - - -@pytest.mark.skip_ci -def test_account_login_with_invalid_creds(clirunner, credentials, isolated_pio_home): - result = clirunner.invoke(cmd_account, ["login", "-u", "123", "-p", "123"]) - assert result.exit_code > 0 - assert result.exception - assert "Invalid user credentials" in str(result.exception) - - -def test_account_login(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - assert "Successfully logged in!" in result.output - - with open(str(isolated_pio_home.join("appstate.json"))) as fp: - appstate = json.load(fp) - assert appstate.get("account") - assert appstate.get("account").get("email") - assert appstate.get("account").get("username") - assert appstate.get("account").get("auth") - assert appstate.get("account").get("auth").get("access_token") - assert appstate.get("account").get("auth").get("access_token_expire") - assert appstate.get("account").get("auth").get("refresh_token") - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are already authorized with" in str(result.exception) - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_logout(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_account, ["logout"]) - validate_cliresult(result) - assert "Successfully logged out" in result.output - - result = clirunner.invoke(cmd_account, ["logout"]) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_password_change_with_invalid_old_password( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - ["password", "--old-password", "test", "--new-password", "test"], - ) - assert result.exit_code > 0 - assert result.exception - assert ( - "Invalid request data for new_password -> " - "'Password must contain at least 8 " - "characters including a number and a lowercase letter'" - in str(result.exception) - ) - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_password_change_with_invalid_new_password_format( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - credentials["password"], - "--new-password", - "test", - ], - ) - assert result.exit_code > 0 - assert result.exception - assert ( - "Invalid request data for new_password -> " - "'Password must contain at least 8 characters" - " including a number and a lowercase letter'" in str(result.exception) - ) - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_password_change( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - credentials["password"], - "--new-password", - "Testpassword123", - ], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - credentials["password"], - "--new-password", - "Testpassword123", - ], - ) - validate_cliresult(result) - assert "Password successfully changed!" in result.output - - result = clirunner.invoke(cmd_account, ["logout"]) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, ["login", "-u", credentials["login"], "-p", "Testpassword123"], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - "Testpassword123", - "--new-password", - credentials["password"], - ], - ) - validate_cliresult(result) - assert "Password successfully changed!" in result.output - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_token_with_invalid_password( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, ["token", "--password", credentials["password"],], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_account, ["token", "--password", "test",],) - assert result.exit_code > 0 - assert result.exception - assert "Invalid user password" in str(result.exception) - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_token(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, ["token", "--password", credentials["password"],], - ) - validate_cliresult(result) - assert "Personal Authentication Token:" in result.output - token = result.output.strip().split(": ")[-1] - - result = clirunner.invoke( - cmd_account, - ["token", "--password", credentials["password"], "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") == token - token = json_result.get("result") - - clirunner.invoke(cmd_account, ["logout"]) - - result = clirunner.invoke( - cmd_account, ["token", "--password", credentials["password"],], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - os.environ["PLATFORMIO_AUTH_TOKEN"] = token - - result = clirunner.invoke( - cmd_account, - ["token", "--password", credentials["password"], "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") == token - - os.environ.pop("PLATFORMIO_AUTH_TOKEN") - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_token_with_refreshing( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - ["token", "--password", credentials["password"], "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") - token = json_result.get("result") - - result = clirunner.invoke( - cmd_account, - [ - "token", - "--password", - credentials["password"], - "--json-output", - "--regenerate", - ], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") - assert token != json_result.get("result") - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pio_home): + validate_cliresult(result) + + # email verification + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split(' 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - + # pio account summary result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) validate_cliresult(result) json_result = json.loads(result.output.strip()) @@ -405,9 +88,8 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi result = clirunner.invoke(cmd_account, ["show"]) validate_cliresult(result) - assert credentials["login"] in result.output - assert "Community" in result.output - assert "100 Concurrent Remote Agents" in result.output + assert username in result.output + # assert "100 Concurrent Remote Agents" in result.output result = clirunner.invoke(cmd_account, ["show", "--json-output"]) validate_cliresult(result) @@ -416,9 +98,9 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi assert json_result.get("profile") assert json_result.get("profile").get("username") assert json_result.get("profile").get("email") - assert credentials["login"] == json_result.get("profile").get( + assert username == json_result.get("profile").get( "username" - ) or credentials["login"] == json_result.get("profile").get("email") + ) or username == json_result.get("profile").get("email") assert json_result.get("profile").get("firstname") assert json_result.get("profile").get("lastname") assert json_result.get("packages") @@ -433,147 +115,121 @@ def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pi assert json_result.get("profile") assert json_result.get("profile").get("username") assert json_result.get("profile").get("email") - assert credentials["login"] == json_result.get("profile").get( + assert username == json_result.get("profile").get( "username" - ) or credentials["login"] == json_result.get("profile").get("email") + ) or username == json_result.get("profile").get("email") assert json_result.get("profile").get("firstname") assert json_result.get("profile").get("lastname") assert json_result.get("packages") assert json_result.get("packages")[0].get("name") assert json_result.get("packages")[0].get("path") assert json_result.get("subscriptions") is not None - finally: - clirunner.invoke(cmd_account, ["logout"]) - -@pytest.mark.skip_ci -def test_account_profile_update_with_invalid_password( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, ["update", "--current-password", credentials["password"]], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) + # pio account token + result = clirunner.invoke(cmd_account, ["token", "--password", password,],) + validate_cliresult(result) + assert "Personal Authentication Token:" in result.output + token = result.output.strip().split(": ")[-1] result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], + cmd_account, ["token", "--password", password, "--json-output"], ) validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result + assert json_result.get("status") == "success" + assert json_result.get("result") == token + token = json_result.get("result") - firstname = "First " + str(int(time.time() * 1000)) - - result = clirunner.invoke( - cmd_account, - ["update", "--current-password", "test", "--firstname", firstname], - ) - assert result.exit_code > 0 - assert result.exception - assert "Invalid user password" in str(result.exception) - finally: clirunner.invoke(cmd_account, ["logout"]) - -@pytest.mark.skip_ci -def test_account_profile_update_only_firstname_and_lastname( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: - result = clirunner.invoke( - cmd_account, ["update", "--current-password", credentials["password"]], - ) + result = clirunner.invoke(cmd_account, ["token", "--password", password,],) assert result.exit_code > 0 assert result.exception assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) + os.environ["PLATFORMIO_AUTH_TOKEN"] = token + result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], + cmd_account, ["token", "--password", password, "--json-output"], ) validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result + assert json_result.get("status") == "success" + assert json_result.get("result") == token - firstname = "First " + str(int(time.time() * 1000)) - lastname = "Last" + str(int(time.time() * 1000)) + os.environ.pop("PLATFORMIO_AUTH_TOKEN") result = clirunner.invoke( - cmd_account, - [ - "update", - "--current-password", - credentials["password"], - "--firstname", - firstname, - "--lastname", - lastname, - ], + cmd_account, ["login", "-u", username, "-p", password], ) validate_cliresult(result) - assert "Profile successfully updated!" in result.output - result = clirunner.invoke(cmd_account, ["show", "--json-output"]) + # pio account password + new_password = "Testpassword123" + result = clirunner.invoke( + cmd_account, + ["password", "--old-password", password, "--new-password", new_password,], + ) validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result.get("profile").get("firstname") == firstname - assert json_result.get("profile").get("lastname") == lastname + assert "Password successfully changed!" in result.output - finally: clirunner.invoke(cmd_account, ["logout"]) - -@pytest.mark.skip_ci -def test_account_profile_update( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: result = clirunner.invoke( - cmd_account, ["update", "--current-password", credentials["password"]], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception + cmd_account, ["login", "-u", username, "-p", new_password], ) + validate_cliresult(result) result = clirunner.invoke( cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], + ["password", "--old-password", new_password, "--new-password", password,], ) validate_cliresult(result) - result = clirunner.invoke(cmd_account, ["show", "--json-output"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - + # pio account update firstname = "First " + str(int(time.time() * 1000)) lastname = "Last" + str(int(time.time() * 1000)) - old_username = json_result.get("profile").get("username") new_username = "username" + str(int(time.time() * 1000))[-5:] - + new_email = "%s+new-%s@%s" % (splited_email[0], username, splited_email[1]) result = clirunner.invoke( cmd_account, [ "update", "--current-password", - credentials["password"], + password, "--firstname", firstname, "--lastname", lastname, "--username", new_username, + "--email", + new_email, ], ) validate_cliresult(result) assert "Profile successfully updated!" in result.output - assert "Please re-login." in result.output + assert ( + "Please check your mail to verify your new email address and re-login. " + in result.output + ) + + result = receive_email(new_email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split(' 0 @@ -583,27 +239,39 @@ def test_account_profile_update( ) result = clirunner.invoke( - cmd_account, ["login", "-u", new_username, "-p", credentials["password"]], + cmd_account, ["login", "-u", new_username, "-p", password], ) validate_cliresult(result) - result = clirunner.invoke( - cmd_account, - [ - "update", - "--current-password", - credentials["password"], - "--username", - old_username, - ], - ) + # pio account destroy with linked resource + + package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" + + tmp_dir = tmpdir_factory.mktemp("package") + fd = FileDownloader(package_url, str(tmp_dir)) + pkg_dir = tmp_dir.mkdir("raw_package") + fd.start(with_progress=False, silent=True) + with FileUnpacker(fd.get_filepath()) as unpacker: + unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) + + result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) validate_cliresult(result) - assert "Profile successfully updated!" in result.output - assert "Please re-login." in result.output + try: + result = receive_email(new_email) + assert "Congrats" in result + assert "was published" in result + except: # pylint:disable=bare-except + pass - result = clirunner.invoke( - cmd_account, ["login", "-u", old_username, "-p", credentials["password"]], + result = clirunner.invoke(cmd_account, ["destroy"], "y") + assert result.exit_code != 0 + assert ( + "We can not destroy the %s account due to 1 linked resources from registry" + % username ) + + result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) validate_cliresult(result) finally: - clirunner.invoke(cmd_account, ["logout"]) + result = clirunner.invoke(cmd_account, ["destroy"], "y") + validate_cliresult(result) diff --git a/tests/commands/test_orgs.py b/tests/commands/test_orgs.py index 4650caaf66..3af38e8374 100644 --- a/tests/commands/test_orgs.py +++ b/tests/commands/test_orgs.py @@ -14,142 +14,150 @@ import json import os +import time import pytest +import requests from platformio.commands.account import cli as cmd_account from platformio.commands.org import cli as cmd_org -pytestmark = pytest.mark.skipif( - not ( - os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") - and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") - ), - reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", -) - -@pytest.fixture(scope="session") -def credentials(): - return { - "login": os.environ["PLATFORMIO_TEST_ACCOUNT_LOGIN"], - "password": os.environ["PLATFORMIO_TEST_ACCOUNT_PASSWORD"], - } - - -def test_orgs(clirunner, credentials, validate_cliresult, isolated_pio_home): +@pytest.mark.skipif( + not os.environ.get("TEST_EMAIL_LOGIN"), + reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", +) # pylint:disable=too-many-arguments +def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): + username = "test-piocore-%s" % str(int(time.time() * 1000)) + splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") + email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) + firstname = "Test" + lastname = "User" + password = "Qwerty123!" + + # pio account register + result = clirunner.invoke( + cmd_account, + [ + "register", + "-u", + username, + "-e", + email, + "-p", + password, + "--firstname", + firstname, + "--lastname", + lastname, + ], + ) + validate_cliresult(result) + + # email verification + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split('= 3 - check = False - for org in json_result: - assert "orgname" in org - orgname = org["orgname"] - assert "displayname" in org - assert "email" in org - assert "owners" in org - for owner in org.get("owners"): - assert "username" in owner - check = owner["username"] == credentials["login"] if not check else True - assert "firstname" in owner - assert "lastname" in owner - assert check - - result = clirunner.invoke(cmd_org, ["add", orgname, "ivankravets"],) + assert json_result == [ + { + "orgname": orgname, + "displayname": display_name, + "email": email, + "owners": [ + {"username": username, "firstname": firstname, "lastname": lastname} + ], + } + ] + + # pio org add (owner) + result = clirunner.invoke(cmd_org, ["add", orgname, second_username]) validate_cliresult(result) - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) >= 3 - check = False - for item in json_result: - if item["orgname"] != orgname: - continue - for owner in item.get("owners"): - check = owner["username"] == "ivankravets" if not check else True - assert check - - result = clirunner.invoke(cmd_org, ["remove", orgname, "ivankravets"],) + assert second_username in result.output + + # pio org remove (owner) + result = clirunner.invoke(cmd_org, ["remove", orgname, second_username]) validate_cliresult(result) - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert len(json_result) >= 3 - check = False - for item in json_result: - if item["orgname"] != orgname: - continue - for owner in item.get("owners"): - check = owner["username"] == "ivankravets" if not check else True - assert not check - finally: - clirunner.invoke(cmd_account, ["logout"]) + assert second_username not in result.output + # pio org update + new_orgname = "neworg-piocore-%s" % str(int(time.time() * 1000)) + new_display_name = "Test Org for PIO Core" -@pytest.mark.skip -def test_org_update(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], + cmd_org, + [ + "update", + orgname, + "--new-orgname", + new_orgname, + "--displayname", + new_display_name, + ], ) validate_cliresult(result) - assert "Successfully logged in!" in result.output - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) validate_cliresult(result) json_result = json.loads(result.output.strip()) - assert len(json_result) >= 3 - org = json_result[0] - assert "orgname" in org - assert "displayname" in org - assert "email" in org - assert "owners" in org - - old_orgname = org["orgname"] - if len(old_orgname) > 10: - new_orgname = "neworg" + org["orgname"][6:] + assert json_result == [ + { + "orgname": new_orgname, + "displayname": new_display_name, + "email": email, + "owners": [ + {"username": username, "firstname": firstname, "lastname": lastname} + ], + } + ] result = clirunner.invoke( - cmd_org, ["update", old_orgname, "--new-orgname", new_orgname], + cmd_org, + [ + "update", + new_orgname, + "--new-orgname", + orgname, + "--displayname", + display_name, + ], ) validate_cliresult(result) - - result = clirunner.invoke( - cmd_org, ["update", new_orgname, "--new-orgname", old_orgname], - ) + finally: + result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"],) + result = clirunner.invoke(cmd_account, ["destroy"], "y") validate_cliresult(result) - assert json.loads(result.output.strip()) == json_result - finally: - clirunner.invoke(cmd_account, ["logout"]) diff --git a/tests/commands/test_teams.py b/tests/commands/test_teams.py index 13e30ce016..92d5226da2 100644 --- a/tests/commands/test_teams.py +++ b/tests/commands/test_teams.py @@ -17,123 +17,128 @@ import time import pytest +import requests from platformio.commands.account import cli as cmd_account from platformio.commands.org import cli as cmd_org from platformio.commands.team import cli as cmd_team -pytestmark = pytest.mark.skipif( - not ( - os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") - and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") - ), - reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", -) +@pytest.mark.skipif( + not os.environ.get("TEST_EMAIL_LOGIN"), + reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", +) # pylint:disable=too-many-arguments +def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): + username = "test-piocore-%s" % str(int(time.time() * 1000)) + splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") + email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) + firstname = "Test" + lastname = "User" + password = "Qwerty123!" + + # pio account register + result = clirunner.invoke( + cmd_account, + [ + "register", + "-u", + username, + "-e", + email, + "-p", + password, + "--firstname", + firstname, + "--lastname", + lastname, + ], + ) + validate_cliresult(result) + + # email verification + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split('= 3 - orgname = json_result[0].get("orgname") - + # pio team create result = clirunner.invoke( cmd_team, [ "create", "%s:%s" % (orgname, teamname), "--description", - "team for CI test", + team_description, ], ) validate_cliresult(result) + # pio team list result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) validate_cliresult(result) json_result = json.loads(result.output.strip()) - assert len(json_result) >= 1 - check = False - for team in json_result: - assert team["id"] - assert team["name"] - if team["name"] == teamname: - check = True - assert "description" in team - assert "members" in team - assert check + for item in json_result: + del item["id"] + assert json_result == [ + {"name": teamname, "description": team_description, "members": []} + ] + # pio team add (member) result = clirunner.invoke( - cmd_team, ["add", "%s:%s" % (orgname, teamname), credentials["login"]], + cmd_team, ["add", "%s:%s" % (orgname, teamname), second_username], ) validate_cliresult(result) result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) validate_cliresult(result) - json_result = json.loads(result.output.strip()) - check = False - for team in json_result: - assert team["id"] - assert team["name"] - assert "description" in team - assert "members" in team - if ( - len(team["members"]) > 0 - and team["members"][0]["username"] == credentials["login"] - ): - check = True - assert check + assert second_username in result.output + # pio team remove (member) result = clirunner.invoke( - cmd_team, ["remove", "%s:%s" % (orgname, teamname), credentials["login"]], + cmd_team, ["remove", "%s:%s" % (orgname, teamname), second_username], ) validate_cliresult(result) result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) validate_cliresult(result) + assert second_username not in result.output + # pio team update + new_teamname = "new-" + str(int(time.time() * 1000)) + newteam_description = "Updated Description" result = clirunner.invoke( cmd_team, [ "update", "%s:%s" % (orgname, teamname), + "--name", + new_teamname, "--description", - "Updated Description", + newteam_description, ], ) validate_cliresult(result) @@ -141,18 +146,30 @@ def test_teams(clirunner, credentials, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) validate_cliresult(result) json_result = json.loads(result.output.strip()) - assert len(json_result) >= 1 - check = False - for team in json_result: - assert team["id"] - assert team["name"] - assert "description" in team - if team.get("description") == "Updated Description": - check = True - assert "members" in team - assert check + for item in json_result: + del item["id"] + assert json_result == [ + {"name": new_teamname, "description": newteam_description, "members": []} + ] + + result = clirunner.invoke( + cmd_team, + [ + "update", + "%s:%s" % (orgname, new_teamname), + "--name", + teamname, + "--description", + team_description, + ], + ) + validate_cliresult(result) finally: - clirunner.invoke( - cmd_team, ["destroy", "%s:%s" % (orgname, teamname),], + result = clirunner.invoke( + cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y" ) - clirunner.invoke(cmd_account, ["logout"]) + validate_cliresult(result) + result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") + validate_cliresult(result) + result = clirunner.invoke(cmd_account, ["destroy"], "y") + validate_cliresult(result) diff --git a/tests/conftest.py b/tests/conftest.py index f0529146f2..4b2259ac72 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import email import os +import poplib +import time import pytest from click.testing import CliRunner @@ -53,3 +56,43 @@ def fin(): @pytest.fixture(scope="function") def without_internet(monkeypatch): monkeypatch.setattr(util, "_internet_on", lambda: False) + + +@pytest.fixture +def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals + def _receive_email(from_who): + test_email = os.environ.get("TEST_EMAIL_LOGIN") + test_password = os.environ.get("TEST_EMAIL_PASSWORD") + pop_server = os.environ.get("TEST_EMAIL_POP3_SERVER") or "pop.gmail.com" + if "gmail" in pop_server: + test_email = "recent:" + test_email + + def get_body(msg): + if msg.is_multipart(): + return get_body(msg.get_payload(0)) + return msg.get_payload(None, True) + + result = None + start_time = time.time() + while not result: + time.sleep(5) + server = poplib.POP3_SSL(pop_server) + server.user(test_email) + server.pass_(test_password) + _, mails, _ = server.list() + for index, _ in enumerate(mails): + _, lines, _ = server.retr(index + 1) + msg_content = b"\n".join(lines) + msg = email.message_from_string( + msg_content.decode("ASCII", errors="surrogateescape") + ) + if from_who not in msg.get("To"): + continue + server.dele(index + 1) + result = get_body(msg).decode() + if time.time() - start_time > 60: + break + server.quit() + return result + + return _receive_email From e0023bb9086b6732290b7a28eb3bf56c18f31af8 Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Tue, 9 Jun 2020 17:05:11 +0300 Subject: [PATCH 041/223] increase tests email receiving time --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4b2259ac72..542527b1d3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -90,7 +90,7 @@ def get_body(msg): continue server.dele(index + 1) result = get_body(msg).decode() - if time.time() - start_time > 60: + if time.time() - start_time > 120: break server.quit() return result From 3c8e0b17a7c42691e8fe4e0c418e88e926d8c453 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Jun 2020 18:43:50 +0300 Subject: [PATCH 042/223] Added support for custom targets --- HISTORY.rst | 9 ++- docs | 2 +- platformio/builder/main.py | 3 +- platformio/builder/tools/pioide.py | 4 +- platformio/builder/tools/piomisc.py | 58 ++++---------- platformio/builder/tools/piotarget.py | 109 ++++++++++++++++++++++++++ 6 files changed, 139 insertions(+), 46 deletions(-) create mode 100644 platformio/builder/tools/piotarget.py diff --git a/HISTORY.rst b/HISTORY.rst index 250e680ac8..65392477b6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,9 +6,16 @@ Release Notes PlatformIO Core 4 ----------------- -4.3.5 (2020-??-??) +4.4.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +* New `Account Management System `__ (preview) + + - Manage own organizations + - Manage organization teams + - Manage resource access + +* Added support for `custom targets `__ (user cases: command shortcuts, pre/post processing based on dependencies, custom command launcher with options, etc.) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. 4.3.4 (2020-05-23) diff --git a/docs b/docs index cfba4f4568..439f402c5b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit cfba4f456843f1069d9ebd083e359186054c8659 +Subproject commit 439f402c5b882af01b48068180810a58fc6db5ae diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 7184da7c21..a0a8ab1208 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -55,6 +55,7 @@ "c++", "link", "platformio", + "piotarget", "pioplatform", "pioproject", "piomaxlen", @@ -217,7 +218,7 @@ click.echo( "\n%s\n" % dump_json_to_unicode( - projenv.DumpIDEData() # pylint: disable=undefined-variable + projenv.DumpIDEData(env) # pylint: disable=undefined-variable ) ) env.Exit(0) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 65203ab785..acb36ae4ed 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -143,7 +143,8 @@ def _escape_build_flag(flags): return [flag if " " not in flag else '"%s"' % flag for flag in flags] -def DumpIDEData(env): +def DumpIDEData(env, globalenv): + """ env here is `projenv`""" env["__escape_build_flag"] = _escape_build_flag @@ -169,6 +170,7 @@ def DumpIDEData(env): ], "svd_path": _get_svd_path(env), "compiler_type": env.GetCompilerType(), + "targets": globalenv.DumpTargets(), } env_ = env.Clone() diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 1079f40252..aa5158fee0 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -16,15 +16,12 @@ import atexit import io +import os import re import sys -from os import environ, remove, walk -from os.path import basename, isdir, isfile, join, realpath, relpath, sep from tempfile import mkstemp import click -from SCons.Action import Action # pylint: disable=import-error -from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import fs, util from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape @@ -126,11 +123,11 @@ def _gcc_preprocess(self, contents, out_file): '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( out_file, tmp_path ), - "Converting " + basename(out_file[:-4]), + "Converting " + os.path.basename(out_file[:-4]), ) ) atexit.register(_delete_file, tmp_path) - return isfile(out_file) + return os.path.isfile(out_file) def _join_multiline_strings(self, contents): if "\\\n" not in contents: @@ -233,7 +230,9 @@ def append_prototypes(self, contents): def ConvertInoToCpp(env): src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR")) - ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde")) + ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob( + os.path.join(src_dir, "*.pde") + ) if not ino_nodes: return c = InoToCPPConverter(env) @@ -244,8 +243,8 @@ def ConvertInoToCpp(env): def _delete_file(path): try: - if isfile(path): - remove(path) + if os.path.isfile(path): + os.remove(path) except: # pylint: disable=bare-except pass @@ -255,7 +254,7 @@ def _get_compiler_type(env): if env.subst("$CC").endswith("-gcc"): return "gcc" try: - sysenv = environ.copy() + sysenv = os.environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) result = exec_command([env.subst("$CC"), "-v"], env=sysenv) except OSError: @@ -277,8 +276,8 @@ def GetCompilerType(env): def GetActualLDScript(env): def _lookup_in_ldpath(script): for d in env.get("LIBPATH", []): - path = join(env.subst(d), script) - if isfile(path): + path = os.path.join(env.subst(d), script) + if os.path.isfile(path): return path return None @@ -297,7 +296,7 @@ def _lookup_in_ldpath(script): else: continue script = env.subst(raw_script.replace('"', "").strip()) - if isfile(script): + if os.path.isfile(script): return script path = _lookup_in_ldpath(script) if path: @@ -319,29 +318,6 @@ def _lookup_in_ldpath(script): env.Exit(1) -def VerboseAction(_, act, actstr): - if int(ARGUMENTS.get("PIOVERBOSE", 0)): - return act - return Action(act, actstr) - - -def PioClean(env, clean_dir): - if not isdir(clean_dir): - print("Build environment is clean") - env.Exit(0) - clean_rel_path = relpath(clean_dir) - for root, _, files in walk(clean_dir): - for f in files: - dst = join(root, f) - remove(dst) - print( - "Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst)) - ) - print("Done cleaning") - fs.rmtree(clean_dir) - env.Exit(0) - - def ConfigureDebugFlags(env): def _cleanup_debug_flags(scope): if scope not in env: @@ -370,16 +346,16 @@ def _cleanup_debug_flags(scope): def ConfigureTestTarget(env): env.Append( CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], - CPPPATH=[join("$BUILD_DIR", "UnityTestLib")], + CPPPATH=[os.path.join("$BUILD_DIR", "UnityTestLib")], ) unitylib = env.BuildLibrary( - join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity") + os.path.join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity") ) env.Prepend(LIBS=[unitylib]) src_filter = ["+<*.cpp>", "+<*.c>"] if "PIOTEST_RUNNING_NAME" in env: - src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], sep)) + src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], os.path.sep)) env.Replace(PIOTEST_SRC_FILTER=src_filter) @@ -393,7 +369,7 @@ def GetExtraScripts(env, scope): if not items: return items with fs.cd(env.subst("$PROJECT_DIR")): - return [realpath(item) for item in items] + return [os.path.realpath(item) for item in items] def exists(_): @@ -404,8 +380,6 @@ def generate(env): env.AddMethod(ConvertInoToCpp) env.AddMethod(GetCompilerType) env.AddMethod(GetActualLDScript) - env.AddMethod(VerboseAction) - env.AddMethod(PioClean) env.AddMethod(ConfigureDebugFlags) env.AddMethod(ConfigureTestTarget) env.AddMethod(GetExtraScripts) diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py new file mode 100644 index 0000000000..cbe9045543 --- /dev/null +++ b/platformio/builder/tools/piotarget.py @@ -0,0 +1,109 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +from SCons.Action import Action # pylint: disable=import-error +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import AlwaysBuild # pylint: disable=import-error + +from platformio import fs + + +def VerboseAction(_, act, actstr): + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + return act + return Action(act, actstr) + + +def PioClean(env, clean_dir): + if not os.path.isdir(clean_dir): + print("Build environment is clean") + env.Exit(0) + clean_rel_path = os.path.relpath(clean_dir) + for root, _, files in os.walk(clean_dir): + for f in files: + dst = os.path.join(root, f) + os.remove(dst) + print( + "Removed %s" + % (dst if clean_rel_path.startswith(".") else os.path.relpath(dst)) + ) + print("Done cleaning") + fs.rmtree(clean_dir) + env.Exit(0) + + +def _add_pio_target( # pylint: disable=too-many-arguments + env, + scope, + name, + dependencies, + actions, + title=None, + description=None, + always_build=True, +): + if "__PIO_TARGETS" not in env: + env["__PIO_TARGETS"] = {} + assert name not in env["__PIO_TARGETS"] + env["__PIO_TARGETS"][name] = dict( + name=name, scope=scope, title=title, description=description + ) + target = env.Alias(name, dependencies, actions) + if always_build: + AlwaysBuild(target) + return target + + +def AddSystemTarget(env, *args, **kwargs): + return _add_pio_target(env, "system", *args, **kwargs) + + +def AddCustomTarget(env, *args, **kwargs): + return _add_pio_target(env, "custom", *args, **kwargs) + + +def DumpTargets(env): + print("DumpTargets", id(env)) + targets = env.get("__PIO_TARGETS") or {} + # pre-fill default system targets + if ( + not any(t["scope"] == "system" for t in targets.values()) + and env.PioPlatform().is_embedded() + ): + targets["upload"] = dict(name="upload", scope="system", title="Upload") + targets["compiledb"] = dict( + name="compiledb", + scope="system", + title="Compilation database", + description="Generate compilation database `compile_commands.json`", + ) + targets["clean"] = dict(name="clean", scope="system", title="Clean") + return list(targets.values()) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(VerboseAction) + env.AddMethod(PioClean) + env.AddMethod(AddSystemTarget) + env.AddMethod(AddCustomTarget) + env.AddMethod(DumpTargets) + return env From 89cc6f9bf353bfe417f781de2a96b0dcf51674cf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Jun 2020 18:44:49 +0300 Subject: [PATCH 043/223] Bump version to 4.4.0a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b4db9ecf71..3dab3e3a9b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, "5a2") +VERSION = (4, 4, "0a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 062a82c89e07f04989b5882ef2bf6f5f74784b43 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Jun 2020 20:59:23 +0300 Subject: [PATCH 044/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 439f402c5b..4ceaa801e6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 439f402c5b882af01b48068180810a58fc6db5ae +Subproject commit 4ceaa801e6f1eb25b40edcb709746c85c86706fe From e6fbd6acf1a335b5198a1931f4a6c43a6c19bc70 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 9 Jun 2020 23:26:49 +0300 Subject: [PATCH 045/223] Remove debug code --- platformio/builder/tools/piotarget.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index cbe9045543..c4ddbade13 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -78,7 +78,6 @@ def AddCustomTarget(env, *args, **kwargs): def DumpTargets(env): - print("DumpTargets", id(env)) targets = env.get("__PIO_TARGETS") or {} # pre-fill default system targets if ( From a182cca5e9a339b1a754c7d2f09e516bd9c2be62 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 10 Jun 2020 11:07:19 +0300 Subject: [PATCH 046/223] tests fix (#3555) * replace timestamp with randint in tests * replace pop3 with imap --- .github/workflows/core.yml | 2 +- tests/commands/test_account.py | 10 +++++----- tests/commands/test_orgs.py | 8 ++++---- tests/commands/test_teams.py | 10 +++++----- tests/conftest.py | 29 +++++++++++++++-------------- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index c2e9547c71..8332e9447c 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -30,7 +30,7 @@ jobs: env: TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }} TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }} - TEST_EMAIL_POP3_SERVER: ${{ secrets.TEST_EMAIL_POP3_SERVER }} + TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }} run: | tox -e testcore diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py index 221b724a7e..dff24f52ba 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account.py @@ -14,7 +14,7 @@ import json import os -import time +import random import pytest import requests @@ -32,7 +32,7 @@ def test_account( clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory ): - username = "test-piocore-%s" % str(int(time.time() * 1000)) + username = "test-piocore-%s" % str(random.randint(0, 100000)) splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) firstname = "Test" @@ -191,10 +191,10 @@ def test_account( validate_cliresult(result) # pio account update - firstname = "First " + str(int(time.time() * 1000)) - lastname = "Last" + str(int(time.time() * 1000)) + firstname = "First " + str(random.randint(0, 100000)) + lastname = "Last" + str(random.randint(0, 100000)) - new_username = "username" + str(int(time.time() * 1000))[-5:] + new_username = "username" + str(random.randint(0, 100000)) new_email = "%s+new-%s@%s" % (splited_email[0], username, splited_email[1]) result = clirunner.invoke( cmd_account, diff --git a/tests/commands/test_orgs.py b/tests/commands/test_orgs.py index 3af38e8374..d2001d0c57 100644 --- a/tests/commands/test_orgs.py +++ b/tests/commands/test_orgs.py @@ -14,7 +14,7 @@ import json import os -import time +import random import pytest import requests @@ -28,7 +28,7 @@ reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", ) # pylint:disable=too-many-arguments def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(int(time.time() * 1000)) + username = "test-piocore-%s" % str(random.randint(0, 100000)) splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) firstname = "Test" @@ -71,7 +71,7 @@ def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) validate_cliresult(result) - orgname = "testorg-piocore-%s" % str(int(time.time() * 1000)) + orgname = "testorg-piocore-%s" % str(random.randint(0, 100000)) display_name = "Test Org for PIO Core" second_username = "ivankravets" try: @@ -114,7 +114,7 @@ def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): assert second_username not in result.output # pio org update - new_orgname = "neworg-piocore-%s" % str(int(time.time() * 1000)) + new_orgname = "neworg-piocore-%s" % str(random.randint(0, 100000)) new_display_name = "Test Org for PIO Core" result = clirunner.invoke( diff --git a/tests/commands/test_teams.py b/tests/commands/test_teams.py index 92d5226da2..57085502b1 100644 --- a/tests/commands/test_teams.py +++ b/tests/commands/test_teams.py @@ -14,7 +14,7 @@ import json import os -import time +import random import pytest import requests @@ -29,7 +29,7 @@ reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", ) # pylint:disable=too-many-arguments def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(int(time.time() * 1000)) + username = "test-piocore-%s" % str(random.randint(0, 100000)) splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) firstname = "Test" @@ -72,7 +72,7 @@ def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) validate_cliresult(result) - orgname = "testorg-piocore-%s" % str(int(time.time() * 1000)) + orgname = "testorg-piocore-%s" % str(random.randint(0, 100000)) display_name = "Test Org for PIO Core" # pio org create @@ -81,7 +81,7 @@ def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): ) validate_cliresult(result) - teamname = "test-" + str(int(time.time() * 1000)) + teamname = "test-" + str(random.randint(0, 100000)) team_description = "team for CI test" second_username = "ivankravets" try: @@ -128,7 +128,7 @@ def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): assert second_username not in result.output # pio team update - new_teamname = "new-" + str(int(time.time() * 1000)) + new_teamname = "new-" + str(random.randint(0, 100000)) newteam_description = "Updated Description" result = clirunner.invoke( cmd_team, diff --git a/tests/conftest.py b/tests/conftest.py index 542527b1d3..9fa3578be3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,8 +13,8 @@ # limitations under the License. import email +import imaplib import os -import poplib import time import pytest @@ -63,9 +63,7 @@ def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals def _receive_email(from_who): test_email = os.environ.get("TEST_EMAIL_LOGIN") test_password = os.environ.get("TEST_EMAIL_PASSWORD") - pop_server = os.environ.get("TEST_EMAIL_POP3_SERVER") or "pop.gmail.com" - if "gmail" in pop_server: - test_email = "recent:" + test_email + imap_server = os.environ.get("TEST_EMAIL_IMAP_SERVER") or "imap.gmail.com" def get_body(msg): if msg.is_multipart(): @@ -76,23 +74,26 @@ def get_body(msg): start_time = time.time() while not result: time.sleep(5) - server = poplib.POP3_SSL(pop_server) - server.user(test_email) - server.pass_(test_password) - _, mails, _ = server.list() - for index, _ in enumerate(mails): - _, lines, _ = server.retr(index + 1) - msg_content = b"\n".join(lines) + server = imaplib.IMAP4_SSL(imap_server) + server.login(test_email, test_password) + server.select("INBOX") + _, mails = server.search(None, "ALL") + for index in mails[0].split(): + _, data = server.fetch(index, "(RFC822)") msg = email.message_from_string( - msg_content.decode("ASCII", errors="surrogateescape") + data[0][1].decode("ASCII", errors="surrogateescape") ) if from_who not in msg.get("To"): continue - server.dele(index + 1) + if "gmail" in imap_server: + server.store(index, "+X-GM-LABELS", "\\Trash") + server.store(index, "+FLAGS", "\\Deleted") + server.expunge() result = get_body(msg).decode() if time.time() - start_time > 120: break - server.quit() + server.close() + server.logout() return result return _receive_email From 0d8272890c901f5bb4d56d875ce743a01fc4cf61 Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Wed, 10 Jun 2020 12:02:34 +0300 Subject: [PATCH 047/223] merge account, org and team tests into one file --- ...st_account.py => test_account_org_team.py} | 293 ++++++++++++++++++ tests/commands/test_orgs.py | 163 ---------- tests/commands/test_teams.py | 175 ----------- 3 files changed, 293 insertions(+), 338 deletions(-) rename tests/commands/{test_account.py => test_account_org_team.py} (51%) delete mode 100644 tests/commands/test_orgs.py delete mode 100644 tests/commands/test_teams.py diff --git a/tests/commands/test_account.py b/tests/commands/test_account_org_team.py similarity index 51% rename from tests/commands/test_account.py rename to tests/commands/test_account_org_team.py index dff24f52ba..49530f5d23 100644 --- a/tests/commands/test_account.py +++ b/tests/commands/test_account_org_team.py @@ -20,7 +20,9 @@ import requests from platformio.commands.account import cli as cmd_account +from platformio.commands.org import cli as cmd_org from platformio.commands.package import cli as cmd_package +from platformio.commands.team import cli as cmd_team from platformio.downloader import FileDownloader from platformio.unpacker import FileUnpacker @@ -275,3 +277,294 @@ def test_account( finally: result = clirunner.invoke(cmd_account, ["destroy"], "y") validate_cliresult(result) + + +@pytest.mark.skipif( + not os.environ.get("TEST_EMAIL_LOGIN"), + reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", +) # pylint:disable=too-many-arguments +def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): + username = "test-piocore-%s" % str(random.randint(0, 100000)) + splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") + email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) + firstname = "Test" + lastname = "User" + password = "Qwerty123!" + + # pio account register + result = clirunner.invoke( + cmd_account, + [ + "register", + "-u", + username, + "-e", + email, + "-p", + password, + "--firstname", + firstname, + "--lastname", + lastname, + ], + ) + validate_cliresult(result) + + # email verification + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split(' -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import random - -import pytest -import requests - -from platformio.commands.account import cli as cmd_account -from platformio.commands.org import cli as cmd_org - - -@pytest.mark.skipif( - not os.environ.get("TEST_EMAIL_LOGIN"), - reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", -) # pylint:disable=too-many-arguments -def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(random.randint(0, 100000)) - splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") - email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) - firstname = "Test" - lastname = "User" - password = "Qwerty123!" - - # pio account register - result = clirunner.invoke( - cmd_account, - [ - "register", - "-u", - username, - "-e", - email, - "-p", - password, - "--firstname", - firstname, - "--lastname", - lastname, - ], - ) - validate_cliresult(result) - - # email verification - result = receive_email(email) - link = ( - result.split("Click on the link below to start this process.")[1] - .split("This link will expire within 12 hours.")[0] - .strip() - ) - session = requests.Session() - result = session.get(link).text - link = result.split(' -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import random - -import pytest -import requests - -from platformio.commands.account import cli as cmd_account -from platformio.commands.org import cli as cmd_org -from platformio.commands.team import cli as cmd_team - - -@pytest.mark.skipif( - not os.environ.get("TEST_EMAIL_LOGIN"), - reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", -) # pylint:disable=too-many-arguments -def test_teams(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(random.randint(0, 100000)) - splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") - email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) - firstname = "Test" - lastname = "User" - password = "Qwerty123!" - - # pio account register - result = clirunner.invoke( - cmd_account, - [ - "register", - "-u", - username, - "-e", - email, - "-p", - password, - "--firstname", - firstname, - "--lastname", - lastname, - ], - ) - validate_cliresult(result) - - # email verification - result = receive_email(email) - link = ( - result.split("Click on the link below to start this process.")[1] - .split("This link will expire within 12 hours.")[0] - .strip() - ) - session = requests.Session() - result = session.get(link).text - link = result.split(' Date: Wed, 10 Jun 2020 12:22:28 +0300 Subject: [PATCH 048/223] cleaning --- tests/commands/test_account_org_team.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index 49530f5d23..770a692e8b 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -275,8 +275,7 @@ def test_account( result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) validate_cliresult(result) finally: - result = clirunner.invoke(cmd_account, ["destroy"], "y") - validate_cliresult(result) + clirunner.invoke(cmd_account, ["destroy"], "y") @pytest.mark.skipif( @@ -413,10 +412,8 @@ def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): ) validate_cliresult(result) finally: - result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") - validate_cliresult(result) - result = clirunner.invoke(cmd_account, ["destroy"], "y") - validate_cliresult(result) + clirunner.invoke(cmd_org, ["destroy", orgname], "y") + clirunner.invoke(cmd_account, ["destroy"], "y") @pytest.mark.skipif( @@ -560,11 +557,6 @@ def test_team(clirunner, validate_cliresult, receive_email, isolated_pio_home): ) validate_cliresult(result) finally: - result = clirunner.invoke( - cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y" - ) - validate_cliresult(result) - result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") - validate_cliresult(result) - result = clirunner.invoke(cmd_account, ["destroy"], "y") - validate_cliresult(result) + clirunner.invoke(cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y") + clirunner.invoke(cmd_org, ["destroy", orgname], "y") + clirunner.invoke(cmd_account, ["destroy"], "y") From 9e3ba11e8ad748b6871efd894223c23c3103dbe0 Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Wed, 10 Jun 2020 12:36:07 +0300 Subject: [PATCH 049/223] skip account tests --- tests/commands/test_account_org_team.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index 770a692e8b..a7711ee255 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -26,6 +26,8 @@ from platformio.downloader import FileDownloader from platformio.unpacker import FileUnpacker +pytestmark = pytest.mark.skip() + @pytest.mark.skipif( not os.environ.get("TEST_EMAIL_LOGIN"), From b71b939307c5eb5139fa683ef404453f1635ba90 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jun 2020 14:25:53 +0300 Subject: [PATCH 050/223] Rename "AddSystemTarget" to "AddPlatformTarget" --- platformio/builder/tools/piotarget.py | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index c4ddbade13..1dc5ff3245 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -47,21 +47,21 @@ def PioClean(env, clean_dir): env.Exit(0) -def _add_pio_target( # pylint: disable=too-many-arguments +def AddTarget( # pylint: disable=too-many-arguments env, - scope, name, dependencies, actions, title=None, description=None, + group="Generic", always_build=True, ): if "__PIO_TARGETS" not in env: env["__PIO_TARGETS"] = {} assert name not in env["__PIO_TARGETS"] env["__PIO_TARGETS"][name] = dict( - name=name, scope=scope, title=title, description=description + name=name, title=title, description=description, group=group ) target = env.Alias(name, dependencies, actions) if always_build: @@ -69,29 +69,28 @@ def _add_pio_target( # pylint: disable=too-many-arguments return target -def AddSystemTarget(env, *args, **kwargs): - return _add_pio_target(env, "system", *args, **kwargs) +def AddPlatformTarget(env, *args, **kwargs): + return env.AddTarget(group="Platform", *args, **kwargs) def AddCustomTarget(env, *args, **kwargs): - return _add_pio_target(env, "custom", *args, **kwargs) + return env.AddTarget(group="Custom", *args, **kwargs) def DumpTargets(env): targets = env.get("__PIO_TARGETS") or {} - # pre-fill default system targets - if ( - not any(t["scope"] == "system" for t in targets.values()) - and env.PioPlatform().is_embedded() + # pre-fill default targets if embedded dev-platform + if env.PioPlatform().is_embedded() and not any( + t["group"] == "Platform" for t in targets.values() ): - targets["upload"] = dict(name="upload", scope="system", title="Upload") + targets["upload"] = dict(name="upload", group="Platform", title="Upload") targets["compiledb"] = dict( name="compiledb", - scope="system", title="Compilation database", description="Generate compilation database `compile_commands.json`", + group="Advanced", ) - targets["clean"] = dict(name="clean", scope="system", title="Clean") + targets["clean"] = dict(name="clean", title="Clean", group="Generic") return list(targets.values()) @@ -102,7 +101,8 @@ def exists(_): def generate(env): env.AddMethod(VerboseAction) env.AddMethod(PioClean) - env.AddMethod(AddSystemTarget) + env.AddMethod(AddTarget) + env.AddMethod(AddPlatformTarget) env.AddMethod(AddCustomTarget) env.AddMethod(DumpTargets) return env From ef8a9835b070826b093791af3ef7ca4e1304d68f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jun 2020 14:26:48 +0300 Subject: [PATCH 051/223] Bump version to 4.4.0a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 3dab3e3a9b..d373bbad9f 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a1") +VERSION = (4, 4, "0a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f571ad9d4756424e0d1875964bc8028d0997626e Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 11 Jun 2020 11:03:48 +0300 Subject: [PATCH 052/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 4ceaa801e6..37b313fd15 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4ceaa801e6f1eb25b40edcb709746c85c86706fe +Subproject commit 37b313fd1576995a61d49a0520ea8d063b1a4da6 From 2722e2741525a6db6583c25203e9779a7a6b9e59 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 11 Jun 2020 15:15:46 +0300 Subject: [PATCH 053/223] Sync docs --- docs | 2 +- platformio/builder/tools/piotarget.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 37b313fd15..ec54ae9917 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 37b313fd1576995a61d49a0520ea8d063b1a4da6 +Subproject commit ec54ae991784ce854dfe939bcaf4c2545abcf55e diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 1dc5ff3245..7106a40d36 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -86,7 +86,7 @@ def DumpTargets(env): targets["upload"] = dict(name="upload", group="Platform", title="Upload") targets["compiledb"] = dict( name="compiledb", - title="Compilation database", + title="Compilation Database", description="Generate compilation database `compile_commands.json`", group="Advanced", ) From 266612bbdf753326deda5e1897aed80d23e46127 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 11 Jun 2020 15:27:51 +0300 Subject: [PATCH 054/223] Run CI on pull requests --- .github/workflows/core.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/examples.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 8332e9447c..c6e15bfd9c 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -1,6 +1,6 @@ name: Core -on: [push] +on: [push, pull_request] jobs: build: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bfe2c116ca..39de401e6a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,6 +1,6 @@ name: Docs -on: [push] +on: [push, pull_request] jobs: build: diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index b5452909e2..f1db5c388b 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -1,6 +1,6 @@ name: Examples -on: [push] +on: [push, pull_request] jobs: build: From 405dcda8247cdfbd66c9a93b78faa683693b7aa7 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Thu, 11 Jun 2020 16:02:38 +0300 Subject: [PATCH 055/223] Feature/update account tests (#3556) * update account tests * change second user * refactoring * clean * fix tests email receiving * fix --- tests/commands/test_account_org_team.py | 819 +++++++++++------------- tests/conftest.py | 4 +- 2 files changed, 376 insertions(+), 447 deletions(-) diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index a7711ee255..cc97b33fb2 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -26,16 +26,29 @@ from platformio.downloader import FileDownloader from platformio.unpacker import FileUnpacker -pytestmark = pytest.mark.skip() +pytestmark = pytest.mark.skipif( + not (os.environ.get("TEST_EMAIL_LOGIN") and os.environ.get("TEST_EMAIL_PASSWORD")), + reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", +) +username = None +email = None +firstname = None +lastname = None +password = None + +orgname = None +display_name = None +second_username = None + +teamname = None +team_description = None + + +def test_prepare(): + global username, splited_email, email, firstname, lastname + global password, orgname, display_name, second_username, teamname, team_description -@pytest.mark.skipif( - not os.environ.get("TEST_EMAIL_LOGIN"), - reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", -) # pylint:disable=too-many-arguments -def test_account( - clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory -): username = "test-piocore-%s" % str(random.randint(0, 100000)) splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) @@ -43,7 +56,17 @@ def test_account( lastname = "User" password = "Qwerty123!" - # pio account register + orgname = "testorg-piocore-%s" % str(random.randint(0, 100000)) + display_name = "Test Org for PIO Core" + second_username = "ivankravets" + + teamname = "test-" + str(random.randint(0, 100000)) + team_description = "team for CI test" + + +def test_account_register( + clirunner, validate_cliresult, receive_email, isolated_pio_home +): result = clirunner.invoke( cmd_account, [ @@ -75,243 +98,168 @@ def test_account( link = link.replace("&", "&") session.get(link) - # pio account login + +def test_account_login( + clirunner, validate_cliresult, isolated_pio_home, +): result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) validate_cliresult(result) - try: - # pio account summary - result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert not json_result.get("user_id") - assert json_result.get("profile") - assert json_result.get("profile").get("username") - assert json_result.get("profile").get("email") - assert not json_result.get("packages") - assert not json_result.get("subscriptions") - - result = clirunner.invoke(cmd_account, ["show"]) - validate_cliresult(result) - assert username in result.output - # assert "100 Concurrent Remote Agents" in result.output - - result = clirunner.invoke(cmd_account, ["show", "--json-output"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result.get("user_id") - assert json_result.get("profile") - assert json_result.get("profile").get("username") - assert json_result.get("profile").get("email") - assert username == json_result.get("profile").get( - "username" - ) or username == json_result.get("profile").get("email") - assert json_result.get("profile").get("firstname") - assert json_result.get("profile").get("lastname") - assert json_result.get("packages") - assert json_result.get("packages")[0].get("name") - assert json_result.get("packages")[0].get("path") - assert json_result.get("subscriptions") is not None - - result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result.get("user_id") - assert json_result.get("profile") - assert json_result.get("profile").get("username") - assert json_result.get("profile").get("email") - assert username == json_result.get("profile").get( - "username" - ) or username == json_result.get("profile").get("email") - assert json_result.get("profile").get("firstname") - assert json_result.get("profile").get("lastname") - assert json_result.get("packages") - assert json_result.get("packages")[0].get("name") - assert json_result.get("packages")[0].get("path") - assert json_result.get("subscriptions") is not None - - # pio account token - result = clirunner.invoke(cmd_account, ["token", "--password", password,],) - validate_cliresult(result) - assert "Personal Authentication Token:" in result.output - token = result.output.strip().split(": ")[-1] - - result = clirunner.invoke( - cmd_account, ["token", "--password", password, "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") == token - token = json_result.get("result") - - clirunner.invoke(cmd_account, ["logout"]) - - result = clirunner.invoke(cmd_account, ["token", "--password", password,],) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - os.environ["PLATFORMIO_AUTH_TOKEN"] = token - - result = clirunner.invoke( - cmd_account, ["token", "--password", password, "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") == token - - os.environ.pop("PLATFORMIO_AUTH_TOKEN") - - result = clirunner.invoke( - cmd_account, ["login", "-u", username, "-p", password], - ) - validate_cliresult(result) - - # pio account password - new_password = "Testpassword123" - result = clirunner.invoke( - cmd_account, - ["password", "--old-password", password, "--new-password", new_password,], - ) - validate_cliresult(result) - assert "Password successfully changed!" in result.output - - clirunner.invoke(cmd_account, ["logout"]) - - result = clirunner.invoke( - cmd_account, ["login", "-u", username, "-p", new_password], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - ["password", "--old-password", new_password, "--new-password", password,], - ) - validate_cliresult(result) - - # pio account update - firstname = "First " + str(random.randint(0, 100000)) - lastname = "Last" + str(random.randint(0, 100000)) - - new_username = "username" + str(random.randint(0, 100000)) - new_email = "%s+new-%s@%s" % (splited_email[0], username, splited_email[1]) - result = clirunner.invoke( - cmd_account, - [ - "update", - "--current-password", - password, - "--firstname", - firstname, - "--lastname", - lastname, - "--username", - new_username, - "--email", - new_email, - ], - ) - validate_cliresult(result) - assert "Profile successfully updated!" in result.output - assert ( - "Please check your mail to verify your new email address and re-login. " - in result.output - ) - - result = receive_email(new_email) - link = ( - result.split("Click on the link below to start this process.")[1] - .split("This link will expire within 12 hours.")[0] - .strip() - ) - session = requests.Session() - result = session.get(link).text - link = result.split(' 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, ["login", "-u", new_username, "-p", password], - ) - validate_cliresult(result) - - # pio account destroy with linked resource - - package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" - - tmp_dir = tmpdir_factory.mktemp("package") - fd = FileDownloader(package_url, str(tmp_dir)) - pkg_dir = tmp_dir.mkdir("raw_package") - fd.start(with_progress=False, silent=True) - with FileUnpacker(fd.get_filepath()) as unpacker: - unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) - - result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) - validate_cliresult(result) - try: - result = receive_email(new_email) - assert "Congrats" in result - assert "was published" in result - except: # pylint:disable=bare-except - pass - - result = clirunner.invoke(cmd_account, ["destroy"], "y") - assert result.exit_code != 0 - assert ( - "We can not destroy the %s account due to 1 linked resources from registry" - % username - ) - - result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) - validate_cliresult(result) - finally: - clirunner.invoke(cmd_account, ["destroy"], "y") - - -@pytest.mark.skipif( - not os.environ.get("TEST_EMAIL_LOGIN"), - reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", -) # pylint:disable=too-many-arguments -def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(random.randint(0, 100000)) - splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") - email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) - firstname = "Test" - lastname = "User" - password = "Qwerty123!" - # pio account register + +def test_account_summary( + clirunner, validate_cliresult, isolated_pio_home, +): + result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert not json_result.get("user_id") + assert json_result.get("profile") + assert json_result.get("profile").get("username") + assert json_result.get("profile").get("email") + assert not json_result.get("packages") + assert not json_result.get("subscriptions") + + result = clirunner.invoke(cmd_account, ["show"]) + validate_cliresult(result) + assert username in result.output + # assert "100 Concurrent Remote Agents" in result.output + + result = clirunner.invoke(cmd_account, ["show", "--json-output"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result.get("user_id") + assert json_result.get("profile") + assert json_result.get("profile").get("username") + assert json_result.get("profile").get("email") + assert username == json_result.get("profile").get( + "username" + ) or username == json_result.get("profile").get("email") + assert json_result.get("profile").get("firstname") + assert json_result.get("profile").get("lastname") + assert json_result.get("packages") + assert json_result.get("packages")[0].get("name") + assert json_result.get("packages")[0].get("path") + assert json_result.get("subscriptions") is not None + + result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result.get("user_id") + assert json_result.get("profile") + assert json_result.get("profile").get("username") + assert json_result.get("profile").get("email") + assert username == json_result.get("profile").get( + "username" + ) or username == json_result.get("profile").get("email") + assert json_result.get("profile").get("firstname") + assert json_result.get("profile").get("lastname") + assert json_result.get("packages") + assert json_result.get("packages")[0].get("name") + assert json_result.get("packages")[0].get("path") + assert json_result.get("subscriptions") is not None + + +def test_account_token(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke(cmd_account, ["token", "--password", password,],) + validate_cliresult(result) + assert "Personal Authentication Token:" in result.output + token = result.output.strip().split(": ")[-1] + + result = clirunner.invoke( + cmd_account, ["token", "--password", password, "--json-output"], + ) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result + assert json_result.get("status") == "success" + assert json_result.get("result") == token + token = json_result.get("result") + + clirunner.invoke(cmd_account, ["logout"]) + + result = clirunner.invoke(cmd_account, ["token", "--password", password,],) + assert result.exit_code > 0 + assert result.exception + assert "You are not authorized! Please log in to PIO Account" in str( + result.exception + ) + + os.environ["PLATFORMIO_AUTH_TOKEN"] = token + + result = clirunner.invoke( + cmd_account, ["token", "--password", password, "--json-output"], + ) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result + assert json_result.get("status") == "success" + assert json_result.get("result") == token + + os.environ.pop("PLATFORMIO_AUTH_TOKEN") + + result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) + validate_cliresult(result) + + +def test_account_change_password(clirunner, validate_cliresult, isolated_pio_home): + new_password = "Testpassword123" + result = clirunner.invoke( + cmd_account, + ["password", "--old-password", password, "--new-password", new_password,], + ) + validate_cliresult(result) + assert "Password successfully changed!" in result.output + + clirunner.invoke(cmd_account, ["logout"]) + + result = clirunner.invoke( + cmd_account, ["login", "-u", username, "-p", new_password], + ) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_account, + ["password", "--old-password", new_password, "--new-password", password,], + ) + validate_cliresult(result) + + +def test_account_update( + clirunner, validate_cliresult, receive_email, isolated_pio_home +): + global username + global email + global firstname + global lastname + + firstname = "First " + str(random.randint(0, 100000)) + lastname = "Last" + str(random.randint(0, 100000)) + + username = "username" + str(random.randint(0, 100000)) + email = "%s+new-%s@%s" % (splited_email[0], username, splited_email[1]) result = clirunner.invoke( cmd_account, [ - "register", - "-u", - username, - "-e", - email, - "-p", + "update", + "--current-password", password, "--firstname", firstname, "--lastname", lastname, + "--username", + username, + "--email", + email, ], ) validate_cliresult(result) + assert "Profile successfully updated!" in result.output + assert ( + "Please check your mail to verify your new email address and re-login. " + in result.output + ) - # email verification result = receive_email(email) link = ( result.split("Click on the link below to start this process.")[1] @@ -324,241 +272,220 @@ def test_org(clirunner, validate_cliresult, receive_email, isolated_pio_home): link = link.replace("&", "&") session.get(link) - # pio account login + result = clirunner.invoke(cmd_account, ["show"],) + assert result.exit_code > 0 + assert result.exception + assert "You are not authorized! Please log in to PIO Account" in str( + result.exception + ) + result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) validate_cliresult(result) - orgname = "testorg-piocore-%s" % str(random.randint(0, 100000)) - display_name = "Test Org for PIO Core" - second_username = "ivankravets" + +def test_account_destroy_with_linked_resources( + clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory +): + package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" + + tmp_dir = tmpdir_factory.mktemp("package") + fd = FileDownloader(package_url, str(tmp_dir)) + pkg_dir = tmp_dir.mkdir("raw_package") + fd.start(with_progress=False, silent=True) + with FileUnpacker(fd.get_filepath()) as unpacker: + unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) + + result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) + validate_cliresult(result) try: - # pio org create - result = clirunner.invoke( - cmd_org, - ["create", "--email", email, "--displayname", display_name, orgname], - ) - validate_cliresult(result) - - # pio org list - result = clirunner.invoke(cmd_org, ["list", "--json-output"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result == [ - { - "orgname": orgname, - "displayname": display_name, - "email": email, - "owners": [ - {"username": username, "firstname": firstname, "lastname": lastname} - ], - } - ] - - # pio org add (owner) - result = clirunner.invoke(cmd_org, ["add", orgname, second_username]) - validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"]) - validate_cliresult(result) - assert second_username in result.output - - # pio org remove (owner) - result = clirunner.invoke(cmd_org, ["remove", orgname, second_username]) - validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"]) - validate_cliresult(result) - assert second_username not in result.output - - # pio org update - new_orgname = "neworg-piocore-%s" % str(random.randint(0, 100000)) - new_display_name = "Test Org for PIO Core" - - result = clirunner.invoke( - cmd_org, - [ - "update", - orgname, - "--new-orgname", - new_orgname, - "--displayname", - new_display_name, - ], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_org, ["list", "--json-output"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result == [ - { - "orgname": new_orgname, - "displayname": new_display_name, - "email": email, - "owners": [ - {"username": username, "firstname": firstname, "lastname": lastname} - ], - } - ] - - result = clirunner.invoke( - cmd_org, - [ - "update", - new_orgname, - "--new-orgname", - orgname, - "--displayname", - display_name, + result = receive_email(email) + assert "Congrats" in result + assert "was published" in result + except: # pylint:disable=bare-except + pass + + result = clirunner.invoke(cmd_account, ["destroy"], "y") + assert result.exit_code != 0 + assert ( + "We can not destroy the %s account due to 1 linked resources from registry" + % username + ) + + result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) + validate_cliresult(result) + + +def test_org_create(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke( + cmd_org, ["create", "--email", email, "--displayname", display_name, orgname], + ) + validate_cliresult(result) + + +def test_org_list(clirunner, validate_cliresult, isolated_pio_home): + # pio org list + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result == [ + { + "orgname": orgname, + "displayname": display_name, + "email": email, + "owners": [ + {"username": username, "firstname": firstname, "lastname": lastname} ], - ) - validate_cliresult(result) - finally: - clirunner.invoke(cmd_org, ["destroy", orgname], "y") - clirunner.invoke(cmd_account, ["destroy"], "y") + } + ] -@pytest.mark.skipif( - not os.environ.get("TEST_EMAIL_LOGIN"), - reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", -) # pylint:disable=too-many-arguments -def test_team(clirunner, validate_cliresult, receive_email, isolated_pio_home): - username = "test-piocore-%s" % str(random.randint(0, 100000)) - splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") - email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) - firstname = "Test" - lastname = "User" - password = "Qwerty123!" +def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke(cmd_org, ["add", orgname, second_username]) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + assert second_username in result.output + + +def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_home): + result = clirunner.invoke(cmd_org, ["remove", orgname, second_username]) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + assert second_username not in result.output + + +def test_org_update(clirunner, validate_cliresult, isolated_pio_home): + new_orgname = "neworg-piocore-%s" % str(random.randint(0, 100000)) + new_display_name = "Test Org for PIO Core" - # pio account register result = clirunner.invoke( - cmd_account, + cmd_org, [ - "register", - "-u", - username, - "-e", - email, - "-p", - password, - "--firstname", - firstname, - "--lastname", - lastname, + "update", + orgname, + "--new-orgname", + new_orgname, + "--displayname", + new_display_name, ], ) validate_cliresult(result) - # email verification - result = receive_email(email) - link = ( - result.split("Click on the link below to start this process.")[1] - .split("This link will expire within 12 hours.")[0] - .strip() + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result == [ + { + "orgname": new_orgname, + "displayname": new_display_name, + "email": email, + "owners": [ + {"username": username, "firstname": firstname, "lastname": lastname} + ], + } + ] + + result = clirunner.invoke( + cmd_org, + [ + "update", + new_orgname, + "--new-orgname", + orgname, + "--displayname", + display_name, + ], ) - session = requests.Session() - result = session.get(link).text - link = result.split(' Date: Thu, 11 Jun 2020 21:16:06 +0300 Subject: [PATCH 056/223] Update PIO Home front-end to 3.2.3 --- platformio/managers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 53f435fd91..27bee8c21d 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig CORE_PACKAGES = { - "contrib-piohome": "~3.2.1", + "contrib-piohome": "~3.2.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", From fdb83c24be01d19ed48d42b332b082348a6b2cfd Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 11 Jun 2020 23:53:52 +0300 Subject: [PATCH 057/223] Clean autogenerated files before running tests // Resolve #3523 Fixes possible conflicts between auxiliary test transport files when project contains multiple environments with different platforms --- HISTORY.rst | 1 + platformio/commands/test/processor.py | 31 ++++++++++-------- tests/commands/test_test.py | 45 ++++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 65392477b6..0add5842b8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ PlatformIO Core 4 * Added support for `custom targets `__ (user cases: command shortcuts, pre/post processing based on dependencies, custom command launcher with options, etc.) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. +* Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 9024ed0eec..bb7c4a1237 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -13,7 +13,7 @@ # limitations under the License. import atexit -from os import remove +from os import remove, listdir from os.path import isdir, isfile, join from string import Template @@ -194,24 +194,29 @@ def generate_output_file(self, test_dir): ] ) - def delete_tmptest_file(file_): - try: - remove(file_) - except: # pylint: disable=bare-except - if isfile(file_): - click.secho( - "Warning: Could not remove temporary file '%s'. " - "Please remove it manually." % file_, - fg="yellow", - ) + tmp_file_prefix = "tmp_pio_test_transport" + + def delete_tmptest_files(test_dir): + for item in listdir(test_dir): + if item.startswith(tmp_file_prefix) and isfile(join(test_dir, item)): + try: + remove(join(test_dir, item)) + except: # pylint: disable=bare-except + click.secho( + "Warning: Could not remove temporary file '%s'. " + "Please remove it manually." % join(test_dir, item), + fg="yellow", + ) transport_options = TRANSPORT_OPTIONS[self.get_transport()] tpl = Template(file_tpl).substitute(transport_options) data = Template(tpl).substitute(baudrate=self.get_baudrate()) + + delete_tmptest_files(test_dir) tmp_file = join( - test_dir, "output_export." + transport_options.get("language", "c") + test_dir, "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")) ) with open(tmp_file, "w") as fp: fp.write(data) - atexit.register(delete_tmptest_file, tmp_file) + atexit.register(delete_tmptest_files, test_dir) diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index a201e7232e..38fb7eb2d4 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -17,6 +17,7 @@ import pytest from platformio import util +from platformio.commands.test.command import cli as cmd_test def test_local_env(): @@ -31,7 +32,49 @@ def test_local_env(): ] ) if result["returncode"] != 1: - pytest.fail(result) + pytest.fail(str(result)) assert all([s in result["err"] for s in ("PASSED", "IGNORED", "FAILED")]), result[ "out" ] + + +def test_multiple_env_build(clirunner, validate_cliresult, tmpdir): + + project_dir = tmpdir.mkdir("project") + project_dir.join("platformio.ini").write( + """ +[env:teensy31] +platform = teensy +framework = mbed +board = teensy31 + +[env:native] +platform = native + +[env:espressif32] +platform = espressif32 +framework = arduino +board = esp32dev +""" + ) + + project_dir.mkdir("test").join("test_main.cpp").write( + """ +#ifdef ARDUINO +void setup() {} +void loop() {} +#else +int main() { + UNITY_BEGIN(); + UNITY_END(); +} +#endif +""" + ) + + result = clirunner.invoke( + cmd_test, ["-d", str(project_dir), "--without-testing", "--without-uploading"], + ) + + validate_cliresult(result) + assert "Multiple ways to build" not in result.output From 28d9f25f9abd000a1be45bb61c98c11b27a2a61e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 12 Jun 2020 23:47:12 +0300 Subject: [PATCH 058/223] Added a new "-e, --environment" option to "platformio project init" command --- HISTORY.rst | 1 + docs | 2 +- examples | 2 +- platformio/commands/project.py | 28 ++++++++++++- platformio/ide/projectgenerator.py | 64 +++++++++++------------------- tests/commands/test_init.py | 37 +++++++++++++---- 6 files changed, 83 insertions(+), 51 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0add5842b8..b725d47439 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ PlatformIO Core 4 * Added support for `custom targets `__ (user cases: command shortcuts, pre/post processing based on dependencies, custom command launcher with options, etc.) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. +* Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) 4.3.4 (2020-05-23) diff --git a/docs b/docs index ec54ae9917..2178afaf33 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ec54ae991784ce854dfe939bcaf4c2545abcf55e +Subproject commit 2178afaf333874a1f6d3dff4b45d0613517049b5 diff --git a/examples b/examples index c442de34a5..bcdcf46691 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit c442de34a57b54451170dbe39f3411a06a05b3f2 +Subproject commit bcdcf46691c1fe79e8cb2cc74e1a156f6bac5e90 diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 3d73f4ffcd..c83b44a77c 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -93,6 +93,7 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 ) @click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) @click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) +@click.option("-e", "--environment", help="Update using existing environment") @click.option("-O", "--project-option", multiple=True) @click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) @@ -102,6 +103,7 @@ def project_init( project_dir, board, ide, + environment, project_option, env_prefix, silent, @@ -139,7 +141,11 @@ def project_init( ) if ide: - pg = ProjectGenerator(project_dir, ide, board) + config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) + config.validate() + pg = ProjectGenerator( + config, environment or get_best_envname(config, board), ide + ) pg.generate() if is_new_project: @@ -444,3 +450,23 @@ def _install_dependent_platforms(ctx, platforms): ctx.invoke( cli_platform_install, platforms=list(set(platforms) - set(installed_platforms)) ) + + +def get_best_envname(config, board_ids=None): + envname = None + default_envs = config.default_envs() + if default_envs: + envname = default_envs[0] + if not board_ids: + return envname + + for env in config.envs(): + if not board_ids: + return env + if not envname: + envname = env + items = config.items(env=env, as_dict=True) + if "board" in items and items.get("board") in board_ids: + return env + + return envname diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 8bf735b0a7..30eaa97d62 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -15,47 +15,31 @@ import codecs import os import sys -from os.path import basename, isdir, isfile, join, realpath, relpath import bottle from platformio import fs, util from platformio.proc import where_is_program -from platformio.project.config import ProjectConfig from platformio.project.helpers import load_project_ide_data class ProjectGenerator(object): - def __init__(self, project_dir, ide, boards): - self.config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) - self.config.validate() - self.project_dir = project_dir + def __init__(self, config, env_name, ide): + self.config = config + self.project_dir = os.path.dirname(config.path) + self.env_name = str(env_name) self.ide = str(ide) - self.env_name = str(self.get_best_envname(boards)) @staticmethod def get_supported_ides(): - tpls_dir = join(fs.get_source_dir(), "ide", "tpls") - return sorted([d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) - - def get_best_envname(self, boards=None): - envname = None - default_envs = self.config.default_envs() - if default_envs: - envname = default_envs[0] - if not boards: - return envname - - for env in self.config.envs(): - if not boards: - return env - if not envname: - envname = env - items = self.config.items(env=env, as_dict=True) - if "board" in items and items.get("board") in boards: - return env - - return envname + tpls_dir = os.path.join(fs.get_source_dir(), "ide", "tpls") + return sorted( + [ + d + for d in os.listdir(tpls_dir) + if os.path.isdir(os.path.join(tpls_dir, d)) + ] + ) @staticmethod def filter_includes(includes_map, ignore_scopes=None, to_unix_path=True): @@ -75,12 +59,12 @@ def _load_tplvars(self): tpl_vars = { "config": self.config, "systype": util.get_systype(), - "project_name": basename(self.project_dir), + "project_name": os.path.basename(self.project_dir), "project_dir": self.project_dir, "env_name": self.env_name, - "user_home_dir": realpath(fs.expanduser("~")), + "user_home_dir": os.path.realpath(fs.expanduser("~")), "platformio_path": sys.argv[0] - if isfile(sys.argv[0]) + if os.path.isfile(sys.argv[0]) else where_is_program("platformio"), "env_path": os.getenv("PATH"), "env_pathsep": os.pathsep, @@ -97,7 +81,7 @@ def _load_tplvars(self): "src_files": self.get_src_files(), "project_src_dir": self.config.get_optional_dir("src"), "project_lib_dir": self.config.get_optional_dir("lib"), - "project_libdeps_dir": join( + "project_libdeps_dir": os.path.join( self.config.get_optional_dir("libdeps"), self.env_name ), } @@ -120,12 +104,12 @@ def get_src_files(self): with fs.cd(self.project_dir): for root, _, files in os.walk(self.config.get_optional_dir("src")): for f in files: - result.append(relpath(join(root, f))) + result.append(os.path.relpath(os.path.join(root, f))) return result def get_tpls(self): tpls = [] - tpls_dir = join(fs.get_source_dir(), "ide", "tpls", self.ide) + tpls_dir = os.path.join(fs.get_source_dir(), "ide", "tpls", self.ide) for root, _, files in os.walk(tpls_dir): for f in files: if not f.endswith(".tpl"): @@ -133,7 +117,7 @@ def get_tpls(self): _relpath = root.replace(tpls_dir, "") if _relpath.startswith(os.sep): _relpath = _relpath[1:] - tpls.append((_relpath, join(root, f))) + tpls.append((_relpath, os.path.join(root, f))) return tpls def generate(self): @@ -141,12 +125,12 @@ def generate(self): for tpl_relpath, tpl_path in self.get_tpls(): dst_dir = self.project_dir if tpl_relpath: - dst_dir = join(self.project_dir, tpl_relpath) - if not isdir(dst_dir): + dst_dir = os.path.join(self.project_dir, tpl_relpath) + if not os.path.isdir(dst_dir): os.makedirs(dst_dir) - file_name = basename(tpl_path)[:-4] + file_name = os.path.basename(tpl_path)[:-4] contents = self._render_tpl(tpl_path, tpl_vars) - self._merge_contents(join(dst_dir, file_name), contents) + self._merge_contents(os.path.join(dst_dir, file_name), contents) @staticmethod def _render_tpl(tpl_path, tpl_vars): @@ -155,7 +139,7 @@ def _render_tpl(tpl_path, tpl_vars): @staticmethod def _merge_contents(dst_path, contents): - if basename(dst_path) == ".gitignore" and isfile(dst_path): + if os.path.basename(dst_path) == ".gitignore" and os.path.isfile(dst_path): return with codecs.open(dst_path, "w", encoding="utf8") as fp: fp.write(contents) diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index b874ead763..09bd8cf9b5 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -62,29 +62,50 @@ def test_init_ide_without_board(clirunner, tmpdir): assert isinstance(result.exception, ProjectEnvsNotAvailableError) -def test_init_ide_atom(clirunner, validate_cliresult, tmpdir): +def test_init_ide_vscode(clirunner, validate_cliresult, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke( - cmd_init, ["--ide", "atom", "-b", "uno", "-b", "teensy31"] + cmd_init, ["--ide", "vscode", "-b", "uno", "-b", "teensy31"] ) validate_cliresult(result) validate_pioproject(str(tmpdir)) assert all( - [tmpdir.join(f).check() for f in (".clang_complete", ".gcc-flags.json")] + [ + tmpdir.join(".vscode").join(f).check() + for f in ("c_cpp_properties.json", "launch.json") + ] + ) + assert ( + "framework-arduino-avr" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() ) - assert "framework-arduino" in tmpdir.join(".clang_complete").read() # switch to NodeMCU - result = clirunner.invoke(cmd_init, ["--ide", "atom", "-b", "nodemcuv2"]) + result = clirunner.invoke(cmd_init, ["--ide", "vscode", "-b", "nodemcuv2"]) + validate_cliresult(result) + validate_pioproject(str(tmpdir)) + assert ( + "framework-arduinoespressif8266" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) + + # switch to teensy31 via env name + result = clirunner.invoke(cmd_init, ["--ide", "vscode", "-e", "teensy31"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) - assert "arduinoespressif" in tmpdir.join(".clang_complete").read() + assert ( + "framework-arduinoteensy" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) # switch to the first board - result = clirunner.invoke(cmd_init, ["--ide", "atom"]) + result = clirunner.invoke(cmd_init, ["--ide", "vscode"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) - assert "framework-arduino" in tmpdir.join(".clang_complete").read() + assert ( + "framework-arduino-avr" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) def test_init_ide_eclipse(clirunner, validate_cliresult): From cf2fa37e56cac5838fd8084a3b7c99a4e68818fa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 13 Jun 2020 13:18:54 +0300 Subject: [PATCH 059/223] Bump version to 4.4.0a3 --- docs | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 2178afaf33..3bdbfb58e2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2178afaf333874a1f6d3dff4b45d0613517049b5 +Subproject commit 3bdbfb58e293ad0e2149fc1e83ad4abba8b7aad9 diff --git a/platformio/__init__.py b/platformio/__init__.py index d373bbad9f..a88d3fbc8d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a2") +VERSION = (4, 4, "0a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From cb70e510166968d40d371923329a10e24687a440 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 13 Jun 2020 16:21:15 +0300 Subject: [PATCH 060/223] Update changelog for Custom Targets --- HISTORY.rst | 11 +++++++++-- docs | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b725d47439..1d4040867c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,13 +9,20 @@ PlatformIO Core 4 4.4.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ -* New `Account Management System `__ (preview) +* New `Account Management System `__ - Manage own organizations - Manage organization teams - Manage resource access -* Added support for `custom targets `__ (user cases: command shortcuts, pre/post processing based on dependencies, custom command launcher with options, etc.) +* New `Custom Targets `__ + + - Pre/Post processing based on a dependent sources (other target, source file, etc.) + - Command launcher with own arguments + - Launch command with custom options declared in `"platformio.ini" `__ + - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) + + * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) diff --git a/docs b/docs index 3bdbfb58e2..8e1821918f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3bdbfb58e293ad0e2149fc1e83ad4abba8b7aad9 +Subproject commit 8e1821918ff774f97f97e65c006a7bb669d48c90 From df0e6016bb255e6a3c2becacce2347308edbdd63 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Mon, 15 Jun 2020 21:25:24 +0300 Subject: [PATCH 061/223] Handle possible NodeList in source files when processing Middlewares // Resolve #3531 env.Object() returns a list of objects that breaks the processing of subsequent middlewares since we only expected File nodes. --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 1d4040867c..8b98626ad2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,6 +26,7 @@ PlatformIO Core 4 * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) +* Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index c0cc11de71..bb86dfc05b 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -20,6 +20,7 @@ from SCons import Builder, Util # pylint: disable=import-error from SCons.Node import FS # pylint: disable=import-error +from SCons.Node import NodeList # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error @@ -285,6 +286,8 @@ def CollectBuildFiles( for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []): tmp = [] for node in sources: + if isinstance(node, NodeList): + node = node[0] if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern): tmp.append(node) continue From d3fd1157437e5ad4d983a49378140220585c622b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 15 Jun 2020 22:05:28 +0300 Subject: [PATCH 062/223] Black format --- platformio/commands/test/processor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index bb7c4a1237..cfc0f3ca61 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -13,7 +13,7 @@ # limitations under the License. import atexit -from os import remove, listdir +from os import listdir, remove from os.path import isdir, isfile, join from string import Template @@ -214,7 +214,8 @@ def delete_tmptest_files(test_dir): delete_tmptest_files(test_dir) tmp_file = join( - test_dir, "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")) + test_dir, + "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")), ) with open(tmp_file, "w") as fp: fp.write(data) From a9c13aa20e9b24e4bd8ec39c48f5958aa6a55794 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 15 Jun 2020 22:05:59 +0300 Subject: [PATCH 063/223] Implement "ManifestParserFactory.new_from_archive" API --- platformio/package/manifest/parser.py | 12 ++++++++++++ platformio/package/spec.py | 2 +- tests/package/test_manifest.py | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index bf01772159..837fabd6aa 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -17,6 +17,7 @@ import json import os import re +import tarfile import requests @@ -109,6 +110,17 @@ def new_from_url(remote_url): remote_url, ) + @staticmethod + def new_from_archive(path): + assert path.endswith("tar.gz") + with tarfile.open(path, mode="r:gz") as tf: + for t in sorted(ManifestFileType.items().values()): + try: + return ManifestParserFactory.new(tf.extractfile(t).read(), t) + except KeyError: + pass + raise UnknownManifestError("Unknown manifest file type in %s archive" % path) + @staticmethod def new( # pylint: disable=redefined-builtin contents, type, remote_url=None, package_dir=None diff --git a/platformio/package/spec.py b/platformio/package/spec.py index 0535d4ba4b..f031c71c3c 100644 --- a/platformio/package/spec.py +++ b/platformio/package/spec.py @@ -43,7 +43,7 @@ def get_manifest_map(cls): def from_archive(cls, path): assert path.endswith("tar.gz") manifest_map = cls.get_manifest_map() - with tarfile.open(path, mode="r|gz") as tf: + with tarfile.open(path, mode="r:gz") as tf: for t in sorted(cls.items().values()): for manifest in manifest_map[t]: try: diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 1ad66a7585..e497c0b450 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -14,6 +14,7 @@ import os import re +import tarfile import jsondiff import pytest @@ -790,6 +791,21 @@ def _sort_examples(items): ) +def test_parser_from_archive(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join("package.json").write('{"name": "package.json"}') + pkg_dir.join("library.json").write('{"name": "library.json"}') + pkg_dir.join("library.properties").write("name=library.properties") + + archive_path = os.path.join(str(pkg_dir), "package.tar.gz") + with tarfile.open(archive_path, mode="w|gz") as tf: + for item in os.listdir(str(pkg_dir)): + tf.add(os.path.join(str(pkg_dir), item), item) + + data = parser.ManifestParserFactory.new_from_archive(archive_path).as_dict() + assert data["name"] == "library.json" + + def test_broken_schemas(): # missing required field with pytest.raises( From 21f3dd11f48e3b4593a676e371429f0c415fff33 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 16 Jun 2020 12:27:49 +0300 Subject: [PATCH 064/223] Fix printing relative paths on Windows // Resolve #3542 Fixes "ValueError" when running "clean" target if "build_dir" points to a folder on a different logical drive --- HISTORY.rst | 1 + platformio/builder/tools/piotarget.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8b98626ad2..709cc9b7fe 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ PlatformIO Core 4 * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) * Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) +* Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index 7106a40d36..a6dcaf3b91 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -20,7 +20,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error -from platformio import fs +from platformio import compat, fs def VerboseAction(_, act, actstr): @@ -30,17 +30,24 @@ def VerboseAction(_, act, actstr): def PioClean(env, clean_dir): + def _relpath(path): + if compat.WINDOWS: + prefix = os.getcwd()[:2].lower() + if ":" not in prefix or not path.lower().startswith(prefix): + return path + return os.path.relpath(path) + if not os.path.isdir(clean_dir): print("Build environment is clean") env.Exit(0) - clean_rel_path = os.path.relpath(clean_dir) + clean_rel_path = _relpath(clean_dir) for root, _, files in os.walk(clean_dir): for f in files: dst = os.path.join(root, f) os.remove(dst) print( "Removed %s" - % (dst if clean_rel_path.startswith(".") else os.path.relpath(dst)) + % (dst if not clean_rel_path.startswith(".") else _relpath(dst)) ) print("Done cleaning") fs.rmtree(clean_dir) From cad0ae01130a45d14b3ca556f015c8dd9e6dd070 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 16 Jun 2020 15:06:04 +0300 Subject: [PATCH 065/223] Update slogan to "No more vendor lock-in!" --- README.rst | 2 ++ docs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index fe7345d5e2..17d15bf67b 100644 --- a/README.rst +++ b/README.rst @@ -39,6 +39,8 @@ PlatformIO `PlatformIO `_ a new generation ecosystem for embedded development +**A place where Developers and Teams have true Freedom! No more vendor lock-in!** + * Open source, maximum permissive Apache 2.0 license * Cross-platform IDE and Unified Debugger * Static Code Analyzer and Remote Unit Testing diff --git a/docs b/docs index 8e1821918f..c07127ebc7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8e1821918ff774f97f97e65c006a7bb669d48c90 +Subproject commit c07127ebc702a6558890cc43691a0daf03591ad5 From 1e90c821dcfe5f4ac2048bc43b0207b148fd5b5f Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 17 Jun 2020 00:24:55 +0300 Subject: [PATCH 066/223] Disable package upload test (#3562) --- tests/commands/test_account_org_team.py | 60 ++++++++++++------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index cc97b33fb2..b4e576f0d3 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -283,36 +283,36 @@ def test_account_update( validate_cliresult(result) -def test_account_destroy_with_linked_resources( - clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory -): - package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" - - tmp_dir = tmpdir_factory.mktemp("package") - fd = FileDownloader(package_url, str(tmp_dir)) - pkg_dir = tmp_dir.mkdir("raw_package") - fd.start(with_progress=False, silent=True) - with FileUnpacker(fd.get_filepath()) as unpacker: - unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) - - result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) - validate_cliresult(result) - try: - result = receive_email(email) - assert "Congrats" in result - assert "was published" in result - except: # pylint:disable=bare-except - pass - - result = clirunner.invoke(cmd_account, ["destroy"], "y") - assert result.exit_code != 0 - assert ( - "We can not destroy the %s account due to 1 linked resources from registry" - % username - ) - - result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) - validate_cliresult(result) +# def test_account_destroy_with_linked_resources( +# clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory +# ): +# package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" +# +# tmp_dir = tmpdir_factory.mktemp("package") +# fd = FileDownloader(package_url, str(tmp_dir)) +# pkg_dir = tmp_dir.mkdir("raw_package") +# fd.start(with_progress=False, silent=True) +# with FileUnpacker(fd.get_filepath()) as unpacker: +# unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) +# +# result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) +# validate_cliresult(result) +# try: +# result = receive_email(email) +# assert "Congrats" in result +# assert "was published" in result +# except: # pylint:disable=bare-except +# pass +# +# result = clirunner.invoke(cmd_account, ["destroy"], "y") +# assert result.exit_code != 0 +# assert ( +# "We can not destroy the %s account due to 1 linked resources from registry" +# % username +# ) +# +# result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) +# validate_cliresult(result) def test_org_create(clirunner, validate_cliresult, isolated_pio_home): From 42e8ea29ff677ff72e7eabefdffd88693c6d1ce9 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 17 Jun 2020 13:53:53 +0300 Subject: [PATCH 067/223] CLI to manage access level on PlatformIO resources. Resolve #3534 (#3563) --- platformio/clients/registry.py | 20 ++++ platformio/commands/access.py | 137 ++++++++++++++++++++++++ tests/commands/test_account_org_team.py | 3 - 3 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 platformio/commands/access.py diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 5936d2e994..1a3626de27 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -74,3 +74,23 @@ def unpublish_package( # pylint: disable=redefined-builtin "delete", path, params={"undo": 1 if undo else 0}, ) return response + + def update_resource(self, urn, private): + return self.send_auth_request( + "put", "/v3/resources/%s" % urn, data={"private": int(private)}, + ) + + def grant_access_for_resource(self, urn, client, level): + return self.send_auth_request( + "put", + "/v3/resources/%s/access" % urn, + data={"client": client, "level": level}, + ) + + def revoke_access_from_resource(self, urn, client): + return self.send_auth_request( + "delete", "/v3/resources/%s/access" % urn, data={"client": client}, + ) + + def list_own_resources(self): + return self.send_auth_request("get", "/v3/resources",) diff --git a/platformio/commands/access.py b/platformio/commands/access.py new file mode 100644 index 0000000000..92efce28c3 --- /dev/null +++ b/platformio/commands/access.py @@ -0,0 +1,137 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json +import re + +import click +from tabulate import tabulate + +from platformio.clients.registry import RegistryClient +from platformio.commands.account import validate_username +from platformio.commands.team import validate_orgname_teamname + + +def validate_client(value): + if ":" in value: + validate_orgname_teamname(value) + else: + validate_username(value) + return value + + +@click.group("access", short_help="Manage Resource Access") +def cli(): + pass + + +def validate_urn(value): + value = str(value).strip() + if not re.match(r"^reg:pkg:(\d+)$", value, flags=re.I): + raise click.BadParameter("Invalid URN format.") + return value + + +@cli.command("public", short_help="Make resource public") +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +def access_public(urn, urn_type): + client = RegistryClient() + client.update_resource(urn=urn, private=0) + return click.secho( + "The resource %s has been successfully updated." % urn, fg="green", + ) + + +@cli.command("private", short_help="Make resource private") +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +def access_private(urn, urn_type): + client = RegistryClient() + client.update_resource(urn=urn, private=1) + return click.secho( + "The resource %s has been successfully updated." % urn, fg="green", + ) + + +@cli.command("grant", short_help="Grant access") +@click.argument("level", type=click.Choice(["admin", "maintainer", "guest"])) +@click.argument( + "client", + metavar="[ORGNAME:TEAMNAME|USERNAME]", + callback=lambda _, __, value: validate_client(value), +) +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +def access_grant(level, client, urn, urn_type): + reg_client = RegistryClient() + reg_client.grant_access_for_resource(urn=urn, client=client, level=level) + return click.secho( + "Access for resource %s has been granted for %s" % (urn, client), fg="green", + ) + + +@cli.command("revoke", short_help="Revoke access") +@click.argument( + "client", + metavar="[ORGNAME:TEAMNAME|USERNAME]", + callback=lambda _, __, value: validate_client(value), +) +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +def access_revoke(client, urn, urn_type): + reg_client = RegistryClient() + reg_client.revoke_access_from_resource(urn=urn, client=client) + return click.secho( + "Access for resource %s has been revoked for %s" % (urn, client), fg="green", + ) + + +@cli.command("list", short_help="List resources") +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--json-output", is_flag=True) +def access_list(urn_type, json_output): + reg_client = RegistryClient() + resources = reg_client.list_own_resources() + if json_output: + return click.echo(json.dumps(resources)) + if not resources: + return click.secho("You do not have any resources.", fg="yellow") + for resource in resources: + click.echo() + click.secho(resource.get("name"), fg="cyan") + click.echo("-" * len(resource.get("name"))) + table_data = [] + table_data.append(("URN:", resource.get("urn"))) + table_data.append(("Owner:", resource.get("owner"))) + table_data.append( + ( + "Access level(s):", + ", ".join( + (level.capitalize() for level in resource.get("access_levels")) + ), + ) + ) + click.echo(tabulate(table_data, tablefmt="plain")) + return click.echo() diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index b4e576f0d3..297e4c12a8 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -21,10 +21,7 @@ from platformio.commands.account import cli as cmd_account from platformio.commands.org import cli as cmd_org -from platformio.commands.package import cli as cmd_package from platformio.commands.team import cli as cmd_team -from platformio.downloader import FileDownloader -from platformio.unpacker import FileUnpacker pytestmark = pytest.mark.skipif( not (os.environ.get("TEST_EMAIL_LOGIN") and os.environ.get("TEST_EMAIL_PASSWORD")), From e853d61e162017603346aada3fc068b0cda9d484 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 17 Jun 2020 18:55:40 +0300 Subject: [PATCH 068/223] Add orgname filter for access list (#3564) * add orgname filter for access list * fix * fix namings --- platformio/clients/registry.py | 6 ++++-- platformio/commands/access.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 1a3626de27..c48094ee5a 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -92,5 +92,7 @@ def revoke_access_from_resource(self, urn, client): "delete", "/v3/resources/%s/access" % urn, data={"client": client}, ) - def list_own_resources(self): - return self.send_auth_request("get", "/v3/resources",) + def list_resources(self, owner): + return self.send_auth_request( + "get", "/v3/resources", params={"owner": owner} if owner else None + ) diff --git a/platformio/commands/access.py b/platformio/commands/access.py index 92efce28c3..2ca72fd6c7 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -109,11 +109,12 @@ def access_revoke(client, urn, urn_type): @cli.command("list", short_help="List resources") +@click.argument("owner", required=False) @click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") @click.option("--json-output", is_flag=True) -def access_list(urn_type, json_output): +def access_list(owner, urn_type, json_output): reg_client = RegistryClient() - resources = reg_client.list_own_resources() + resources = reg_client.list_resources(owner=owner) if json_output: return click.echo(json.dumps(resources)) if not resources: From 03d99657585df7860a4bdf4e455f7aca4c66c8f3 Mon Sep 17 00:00:00 2001 From: ShahRustam Date: Wed, 17 Jun 2020 23:46:50 +0300 Subject: [PATCH 069/223] Replace urn with prn (#3565) * Replace urn with prn * fix * fix text --- platformio/commands/access.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/platformio/commands/access.py b/platformio/commands/access.py index 2ca72fd6c7..4d8017552b 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -40,7 +40,7 @@ def cli(): def validate_urn(value): value = str(value).strip() - if not re.match(r"^reg:pkg:(\d+)$", value, flags=re.I): + if not re.match(r"^reg:pkg:(\d+):(\w+)$", value, flags=re.I): raise click.BadParameter("Invalid URN format.") return value @@ -49,7 +49,7 @@ def validate_urn(value): @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_public(urn, urn_type): client = RegistryClient() client.update_resource(urn=urn, private=0) @@ -62,7 +62,7 @@ def access_public(urn, urn_type): @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_private(urn, urn_type): client = RegistryClient() client.update_resource(urn=urn, private=1) @@ -81,7 +81,7 @@ def access_private(urn, urn_type): @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_grant(level, client, urn, urn_type): reg_client = RegistryClient() reg_client.grant_access_for_resource(urn=urn, client=client, level=level) @@ -99,7 +99,7 @@ def access_grant(level, client, urn, urn_type): @click.argument( "urn", callback=lambda _, __, value: validate_urn(value), ) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_revoke(client, urn, urn_type): reg_client = RegistryClient() reg_client.revoke_access_from_resource(urn=urn, client=client) @@ -110,7 +110,7 @@ def access_revoke(client, urn, urn_type): @cli.command("list", short_help="List resources") @click.argument("owner", required=False) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="urn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="prn:reg:pkg") @click.option("--json-output", is_flag=True) def access_list(owner, urn_type, json_output): reg_client = RegistryClient() From 260c36727cc5c24c68d975150646cc8d469144ee Mon Sep 17 00:00:00 2001 From: Shahrustam Date: Wed, 17 Jun 2020 23:56:22 +0300 Subject: [PATCH 070/223] fix pio access urn format --- platformio/commands/access.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/access.py b/platformio/commands/access.py index 4d8017552b..6a59be7aae 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -40,7 +40,7 @@ def cli(): def validate_urn(value): value = str(value).strip() - if not re.match(r"^reg:pkg:(\d+):(\w+)$", value, flags=re.I): + if not re.match(r"^prn:reg:pkg:(\d+):(\w+)$", value, flags=re.I): raise click.BadParameter("Invalid URN format.") return value @@ -110,7 +110,7 @@ def access_revoke(client, urn, urn_type): @cli.command("list", short_help="List resources") @click.argument("owner", required=False) -@click.option("--urn-type", type=click.Choice(["urn:reg:pkg"]), default="prn:reg:pkg") +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") @click.option("--json-output", is_flag=True) def access_list(owner, urn_type, json_output): reg_client = RegistryClient() From c20a1f24cd54a6f2d436c973d5b2b805f6fb9e7b Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 18 Jun 2020 20:36:59 +0300 Subject: [PATCH 071/223] Don't print relative paths with double-dot --- platformio/builder/tools/piotarget.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index a6dcaf3b91..948776fc0b 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -33,7 +33,11 @@ def PioClean(env, clean_dir): def _relpath(path): if compat.WINDOWS: prefix = os.getcwd()[:2].lower() - if ":" not in prefix or not path.lower().startswith(prefix): + if ( + ":" not in prefix + or not path.lower().startswith(prefix) + or os.path.relpath(path).startswith("..") + ): return path return os.path.relpath(path) From 87d5997b46766230f655638fcb2b8cec699b16d9 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Mon, 22 Jun 2020 14:42:45 +0300 Subject: [PATCH 072/223] Add a test that ensures setUp and tearDown functions can be compiled --- tests/commands/test_test.py | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 38fb7eb2d4..4110188ff5 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -78,3 +78,87 @@ def test_multiple_env_build(clirunner, validate_cliresult, tmpdir): validate_cliresult(result) assert "Multiple ways to build" not in result.output + + +def test_setup_teardown_are_compilable(clirunner, validate_cliresult, tmpdir): + + project_dir = tmpdir.mkdir("project") + project_dir.join("platformio.ini").write( + """ +[env:embedded] +platform = ststm32 +framework = stm32cube +board = nucleo_f401re +test_transport = custom + +[env:native] +platform = native + +""" + ) + + test_dir = project_dir.mkdir("test") + test_dir.join("test_main.c").write( + """ +#include +#include + +void setUp(){ + printf("setUp called"); +} +void tearDown(){ + printf("tearDown called"); +} + +void dummy_test(void) { + TEST_ASSERT_EQUAL(1, 1); +} + +int main() { + UNITY_BEGIN(); + RUN_TEST(dummy_test); + UNITY_END(); +} +""" + ) + + native_result = clirunner.invoke( + cmd_test, ["-d", str(project_dir), "-e", "native"], + ) + + test_dir.join("unittest_transport.h").write( + """ +#ifdef __cplusplus +extern "C" { +#endif + +void unittest_uart_begin(){} +void unittest_uart_putchar(char c){} +void unittest_uart_flush(){} +void unittest_uart_end(){} + +#ifdef __cplusplus +} +#endif +""" + ) + + embedded_result = clirunner.invoke( + cmd_test, + [ + "-d", + str(project_dir), + "--without-testing", + "--without-uploading", + "-e", + "embedded", + ], + ) + + validate_cliresult(native_result) + validate_cliresult(embedded_result) + + assert all(f in native_result.output for f in ("setUp called", "tearDown called")) + assert all( + "[FAILED]" not in out for out in (native_result.output, embedded_result.output) + ) From 967a856061d1f6f89625eca890e386c5ea6ee11d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 15:25:02 +0300 Subject: [PATCH 073/223] Do not allow ":" and "/" chars in a package name --- platformio/package/manifest/schema.py | 8 +++++++- tests/package/test_manifest.py | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 11d3f902f4..3502550a5c 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -149,7 +149,13 @@ class ExampleSchema(StrictSchema): class ManifestSchema(BaseSchema): # Required fields - name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + name = fields.Str( + required=True, + validate=[ + validate.Length(min=1, max=100), + validate.Regexp(r"^[^:/]+$", error="The next chars [:/] are not allowed"), + ], + ) version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) # Optional fields diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index e497c0b450..73acfdafe7 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -841,3 +841,7 @@ def test_broken_schemas(): version="1.2.3", ) ) + + # invalid package name + with pytest.raises(ManifestValidationError, match=("are not allowed")): + ManifestSchema().load_manifest(dict(name="C/C++ :library", version="1.2.3")) From f19491f90970186422d660e4a6b0871bd6996bd4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 17:55:02 +0300 Subject: [PATCH 074/223] Docs: Sync articles --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index c07127ebc7..53c8b74e70 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c07127ebc702a6558890cc43691a0daf03591ad5 +Subproject commit 53c8b74e709cae5e4678d41e491af118edd426ea From 9f05519ccd98e621e10e22593f7a84278bbec073 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 19:53:31 +0300 Subject: [PATCH 075/223] =?UTF-8?q?List=20available=20project=20targets=20?= =?UTF-8?q?with=20a=20new=20"platformio=20run=20=E2=80=93list-targets"=20c?= =?UTF-8?q?ommand=20//=20Resolve=20#3544?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/run/command.py | 45 ++++++++++++++++++++++++++---- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 709cc9b7fe..2e9af05c1b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 4 - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) +* List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) diff --git a/docs b/docs index 53c8b74e70..e2ed400698 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 53c8b74e709cae5e4678d41e491af118edd426ea +Subproject commit e2ed4006983b5400dee022def8774b13e15466ee diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 378eaf0d9a..c21427236d 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import operator +import os from multiprocessing import cpu_count -from os import getcwd -from os.path import isfile from time import time import click @@ -26,7 +26,7 @@ from platformio.commands.run.processor import EnvironmentProcessor from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING from platformio.project.config import ProjectConfig -from platformio.project.helpers import find_project_dir_above +from platformio.project.helpers import find_project_dir_above, load_project_ide_data # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @@ -43,7 +43,7 @@ @click.option( "-d", "--project-dir", - default=getcwd, + default=os.getcwd, type=click.Path( exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True ), @@ -68,6 +68,7 @@ @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--disable-auto-clean", is_flag=True) +@click.option("--list-targets", is_flag=True) @click.pass_context def cli( ctx, @@ -80,11 +81,12 @@ def cli( silent, verbose, disable_auto_clean, + list_targets, ): app.set_session_var("custom_project_conf", project_conf) # find project directory on upper level - if isfile(project_dir): + if os.path.isfile(project_dir): project_dir = find_project_dir_above(project_dir) is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta @@ -93,6 +95,9 @@ def cli( config = ProjectConfig.get_instance(project_conf) config.validate(environment) + if list_targets: + return print_target_list(list(environment) or config.envs()) + # clean obsolete build dir if not disable_auto_clean: build_dir = config.get_optional_dir("build") @@ -261,3 +266,33 @@ def print_processing_summary(results): is_error=failed_nums, fg="red" if failed_nums else "green", ) + + +def print_target_list(envs): + tabular_data = [] + for env, data in load_project_ide_data(os.getcwd(), envs).items(): + tabular_data.extend( + sorted( + [ + ( + click.style(env, fg="cyan"), + t["group"], + click.style(t.get("name"), fg="yellow"), + t["title"], + t.get("description"), + ) + for t in data.get("targets", []) + ], + key=operator.itemgetter(1, 2), + ) + ) + tabular_data.append((None, None, None, None, None)) + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Group", "Name", "Title", "Description") + ], + ), + ) From 3aae791bee05aa82a1569a0a3b945e5db3248c25 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 20:02:43 +0300 Subject: [PATCH 076/223] Change slogan to "collaborative platform" --- README.rst | 2 +- docs | 2 +- platformio/__init__.py | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 17d15bf67b..104a7debc1 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ PlatformIO .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png :target: https://platformio.org?utm_source=github&utm_medium=core -`PlatformIO `_ a new generation ecosystem for embedded development +`PlatformIO `_ a new generation collaborative platform for embedded development **A place where Developers and Teams have true Freedom! No more vendor lock-in!** diff --git a/docs b/docs index e2ed400698..478d089d27 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e2ed4006983b5400dee022def8774b13e15466ee +Subproject commit 478d089d27feb4588d0aeb55a4edd33317fe9af4 diff --git a/platformio/__init__.py b/platformio/__init__.py index a88d3fbc8d..95560a1708 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -17,13 +17,15 @@ __title__ = "platformio" __description__ = ( - "A new generation ecosystem for embedded development. " + "A new generation collaborative platform for embedded development. " "Cross-platform IDE and Unified Debugger. " "Static Code Analyzer and Remote Unit Testing. " "Multi-platform and Multi-architecture Build System. " "Firmware File Explorer and Memory Inspection. " - "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, " - "RISC-V, FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3" + "Professional development environment for Embedded, IoT, Arduino, CMSIS, ESP-IDF, " + "FreeRTOS, libOpenCM3, mbedOS, Pulp OS, SPL, STM32Cube, Zephyr RTOS, ARM, AVR, " + "Espressif (ESP8266/ESP32), FPGA, MCS-51 (8051), MSP430, Nordic (nRF51/nRF52), " + "NXP i.MX RT, PIC32, RISC-V, STMicroelectronics (STM8/STM32), Teensy" ) __url__ = "https://platformio.org" From 5ee90f4e618aa32c5c5c39ef76ffdfac0465ec66 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 23:04:36 +0300 Subject: [PATCH 077/223] Display system-wide information using `platformio system info` command // Resolve #3521 --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/system/command.py | 49 +++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2e9af05c1b..1e07cb2865 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 4 - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) +* Display system-wide information using `platformio system info `__ command (`issue #3521 `_) * List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment diff --git a/docs b/docs index 478d089d27..f33f42cc9c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 478d089d27feb4588d0aeb55a4edd33317fe9af4 +Subproject commit f33f42cc9c5ec30267dc8c0c845aeda63adda598 diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index 48336bfd27..fed66c408b 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -12,17 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. - +import json +import platform import subprocess +import sys import click +from tabulate import tabulate -from platformio import proc +from platformio import __version__, proc, util from platformio.commands.system.completion import ( get_completion_install_path, install_completion_code, uninstall_completion_code, ) +from platformio.managers.lib import LibraryManager +from platformio.managers.package import PackageManager +from platformio.managers.platform import PlatformManager +from platformio.project.config import ProjectConfig @click.group("system", short_help="Miscellaneous system commands") @@ -30,6 +37,44 @@ def cli(): pass +@cli.command("info", short_help="Display system-wide information") +@click.option("--json-output", is_flag=True) +def system_info(json_output): + project_config = ProjectConfig() + data = {} + data["core_version"] = {"title": "PlatformIO Core", "value": __version__} + data["python_version"] = { + "title": "Python", + "value": "{0}.{1}.{2}-{3}.{4}".format(*list(sys.version_info)), + } + data["system"] = {"title": "System Type", "value": util.get_systype()} + data["platform"] = {"title": "Platform", "value": platform.platform(terse=True)} + data["core_dir"] = { + "title": "PlatformIO Core Directory", + "value": project_config.get_optional_dir("core"), + } + data["global_lib_nums"] = { + "title": "Global Libraries", + "value": len(LibraryManager().get_installed()), + } + data["dev_platform_nums"] = { + "title": "Development Platforms", + "value": len(PlatformManager().get_installed()), + } + data["package_tool_nums"] = { + "title": "Package Tools", + "value": len( + PackageManager(project_config.get_optional_dir("packages")).get_installed() + ), + } + + click.echo( + json.dumps(data) + if json_output + else tabulate([(item["title"], item["value"]) for item in data.values()]) + ) + + @cli.group("completion", short_help="Shell completion support") def completion(): # pylint: disable=import-error,import-outside-toplevel From a172a17c815e8fcbe0f8473c6bac1ea1d9714817 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Jun 2020 23:09:28 +0300 Subject: [PATCH 078/223] Bump version to 4.4.0a4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 95560a1708..86f894d342 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a3") +VERSION = (4, 4, "0a4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 164ae2bcbc1dfde2b7e2fd745553bb4a86ff431f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Jun 2020 11:20:29 +0300 Subject: [PATCH 079/223] Extend system info with Python and PIO Core executables // Issue #3521 --- platformio/commands/system/command.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index fed66c408b..e5d6672105 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -53,6 +53,16 @@ def system_info(json_output): "title": "PlatformIO Core Directory", "value": project_config.get_optional_dir("core"), } + data["platformio_exe"] = { + "title": "PlatformIO Core Executable", + "value": proc.where_is_program( + "platformio.exe" if proc.WINDOWS else "platformio" + ), + } + data["python_exe"] = { + "title": "Python Executable", + "value": proc.get_pythonexe_path(), + } data["global_lib_nums"] = { "title": "Global Libraries", "value": len(LibraryManager().get_installed()), From 9fb4cde2a5dac33e5897738b0e7828feefe0921f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Jun 2020 11:26:22 +0300 Subject: [PATCH 080/223] Do not generate ".travis.yml" for a new project, let the user have a choice --- HISTORY.rst | 1 + platformio/commands/project.py | 78 ---------------------------------- 2 files changed, 1 insertion(+), 78 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1e07cb2865..2d9bb80085 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ PlatformIO Core 4 * List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment +* Do not generate ".travis.yml" for a new project, let the user have a choice * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) * Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) * Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) diff --git a/platformio/commands/project.py b/platformio/commands/project.py index c83b44a77c..5bd5efcb81 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -149,7 +149,6 @@ def project_init( pg.generate() if is_new_project: - init_ci_conf(project_dir) init_cvs_ignore(project_dir) if silent: @@ -310,83 +309,6 @@ def init_test_readme(test_dir): ) -def init_ci_conf(project_dir): - conf_path = os.path.join(project_dir, ".travis.yml") - if os.path.isfile(conf_path): - return - with open(conf_path, "w") as fp: - fp.write( - """# Continuous Integration (CI) is the practice, in software -# engineering, of merging all developer working copies with a shared mainline -# several times a day < https://docs.platformio.org/page/ci/index.html > -# -# Documentation: -# -# * Travis CI Embedded Builds with PlatformIO -# < https://docs.travis-ci.com/user/integration/platformio/ > -# -# * PlatformIO integration with Travis CI -# < https://docs.platformio.org/page/ci/travis.html > -# -# * User Guide for `platformio ci` command -# < https://docs.platformio.org/page/userguide/cmd_ci.html > -# -# -# Please choose one of the following templates (proposed below) and uncomment -# it (remove "# " before each line) or use own configuration according to the -# Travis CI documentation (see above). -# - - -# -# Template #1: General project. Test it using existing `platformio.ini`. -# - -# language: python -# python: -# - "2.7" -# -# sudo: false -# cache: -# directories: -# - "~/.platformio" -# -# install: -# - pip install -U platformio -# - platformio update -# -# script: -# - platformio run - - -# -# Template #2: The project is intended to be used as a library with examples. -# - -# language: python -# python: -# - "2.7" -# -# sudo: false -# cache: -# directories: -# - "~/.platformio" -# -# env: -# - PLATFORMIO_CI_SRC=path/to/test/file.c -# - PLATFORMIO_CI_SRC=examples/file.ino -# - PLATFORMIO_CI_SRC=path/to/test/directory -# -# install: -# - pip install -U platformio -# - platformio update -# -# script: -# - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N -""", - ) - - def init_cvs_ignore(project_dir): conf_path = os.path.join(project_dir, ".gitignore") if os.path.isfile(conf_path): From 82735dd5718947322cc4a099b7891839b2c17297 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Jun 2020 11:46:00 +0300 Subject: [PATCH 081/223] Fixed an issue with improper processing of source files added via multiple Build Middlewares // Resolve #3531 --- platformio/builder/tools/platformio.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index bb86dfc05b..560cbe3754 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -20,7 +20,6 @@ from SCons import Builder, Util # pylint: disable=import-error from SCons.Node import FS # pylint: disable=import-error -from SCons.Node import NodeList # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error @@ -283,20 +282,21 @@ def CollectBuildFiles( if fs.path_endswith_ext(item, SRC_BUILD_EXT): sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) - for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []): - tmp = [] - for node in sources: - if isinstance(node, NodeList): - node = node[0] + middlewares = env.get("__PIO_BUILD_MIDDLEWARES") + if not middlewares: + return sources + + new_sources = [] + for node in sources: + new_node = node + for callback, pattern in middlewares: if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern): - tmp.append(node) continue - n = callback(node) - if n: - tmp.append(n) - sources = tmp + new_node = callback(new_node) + if new_node: + new_sources.append(new_node) - return sources + return new_sources def AddBuildMiddleware(env, callback, pattern=None): From 5dadb8749e86e97d429927e0c82add4455acea4a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Jun 2020 12:33:00 +0300 Subject: [PATCH 082/223] Change slogan to "PlatformIO is a professional collaborative platform for embedded development" --- HISTORY.rst | 3 ++- README.rst | 2 +- docs | 2 +- platformio/__init__.py | 10 +++++----- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2d9bb80085..cea8fb1b07 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ PlatformIO Core 4 4.4.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ +**A professional collaborative platform for embedded development** + * New `Account Management System `__ - Manage own organizations @@ -22,7 +24,6 @@ PlatformIO Core 4 - Launch command with custom options declared in `"platformio.ini" `__ - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) - * Display system-wide information using `platformio system info `__ command (`issue #3521 `_) * List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. diff --git a/README.rst b/README.rst index 104a7debc1..fcff06c8d4 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ PlatformIO .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png :target: https://platformio.org?utm_source=github&utm_medium=core -`PlatformIO `_ a new generation collaborative platform for embedded development +`PlatformIO `_ is a professional collaborative platform for embedded development **A place where Developers and Teams have true Freedom! No more vendor lock-in!** diff --git a/docs b/docs index f33f42cc9c..2c2dce47ab 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f33f42cc9c5ec30267dc8c0c845aeda63adda598 +Subproject commit 2c2dce47ab2ee3eb20893d6fb8e01a34fef5c5ee diff --git a/platformio/__init__.py b/platformio/__init__.py index 86f894d342..7ad284a11d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -17,15 +17,15 @@ __title__ = "platformio" __description__ = ( - "A new generation collaborative platform for embedded development. " + "A professional collaborative platform for embedded development. " "Cross-platform IDE and Unified Debugger. " "Static Code Analyzer and Remote Unit Testing. " "Multi-platform and Multi-architecture Build System. " "Firmware File Explorer and Memory Inspection. " - "Professional development environment for Embedded, IoT, Arduino, CMSIS, ESP-IDF, " - "FreeRTOS, libOpenCM3, mbedOS, Pulp OS, SPL, STM32Cube, Zephyr RTOS, ARM, AVR, " - "Espressif (ESP8266/ESP32), FPGA, MCS-51 (8051), MSP430, Nordic (nRF51/nRF52), " - "NXP i.MX RT, PIC32, RISC-V, STMicroelectronics (STM8/STM32), Teensy" + "IoT, Arduino, CMSIS, ESP-IDF, FreeRTOS, libOpenCM3, mbedOS, Pulp OS, SPL, " + "STM32Cube, Zephyr RTOS, ARM, AVR, Espressif (ESP8266/ESP32), FPGA, " + "MCS-51 (8051), MSP430, Nordic (nRF51/nRF52), NXP i.MX RT, PIC32, RISC-V, " + "STMicroelectronics (STM8/STM32), Teensy" ) __url__ = "https://platformio.org" From efc2242046b1bbe4d3644ac687234f9b67fa19d3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 25 Jun 2020 14:51:53 +0300 Subject: [PATCH 083/223] Remove empty data from board information --- platformio/managers/platform.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index c667a956cb..5fa38a4368 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -865,7 +865,7 @@ def manifest(self): return self._manifest def get_brief_data(self): - return { + result = { "id": self.id, "name": self._manifest["name"], "platform": self._manifest.get("platform"), @@ -881,12 +881,16 @@ def get_brief_data(self): ), "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), "rom": self._manifest.get("upload", {}).get("maximum_size", 0), - "connectivity": self._manifest.get("connectivity"), "frameworks": self._manifest.get("frameworks"), - "debug": self.get_debug_data(), "vendor": self._manifest["vendor"], "url": self._manifest["url"], } + if self._manifest.get("connectivity"): + result["connectivity"] = self._manifest.get("connectivity") + debug = self.get_debug_data() + if debug: + result["debug"] = debug + return result def get_debug_data(self): if not self._manifest.get("debug", {}).get("tools"): @@ -895,7 +899,7 @@ def get_debug_data(self): for name, options in self._manifest["debug"]["tools"].items(): tools[name] = {} for key, value in options.items(): - if key in ("default", "onboard"): + if key in ("default", "onboard") and value: tools[name][key] = value return {"tools": tools} From 7bc22353cc8b64ac70380b68afb45b8151c7ffd5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 25 Jun 2020 18:04:04 +0300 Subject: [PATCH 084/223] Docs: Sync dev-platforms --- docs | 2 +- examples | 2 +- platformio/managers/platform.py | 2 +- scripts/docspregen.py | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs b/docs index 2c2dce47ab..438d910cc4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2c2dce47ab2ee3eb20893d6fb8e01a34fef5c5ee +Subproject commit 438d910cc4d35337df90a6b23955f89a0675b52c diff --git a/examples b/examples index bcdcf46691..2fdead5df5 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit bcdcf46691c1fe79e8cb2cc74e1a156f6bac5e90 +Subproject commit 2fdead5df5f146c6f7c70eeef63669bebab7a225 diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 5fa38a4368..52842ca0df 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -609,7 +609,7 @@ def homepage(self): @property def vendor_url(self): - return self._manifest.get("url") + return self._manifest.get("homepage") @property def docs_url(self): diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 3628f6092d..3c498309fc 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -88,12 +88,12 @@ def generate_boards_table(boards, skip_columns=None): lines.append(prefix + name) for data in sorted(boards, key=lambda item: item['name']): - has_onboard_debug = (data['debug'] and any( + has_onboard_debug = (data.get('debug') and any( t.get("onboard") for (_, t) in data['debug']['tools'].items())) debug = "No" if has_onboard_debug: debug = "On-board" - elif data['debug']: + elif data.get('debug'): debug = "External" variables = dict(id=data['id'], @@ -170,11 +170,11 @@ def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): skip_board_columns.append("Debug") lines = [] onboard_debug = [ - b for b in boards if b['debug'] and any( + b for b in boards if b.get('debug') and any( t.get("onboard") for (_, t) in b['debug']['tools'].items()) ] external_debug = [ - b for b in boards if b['debug'] and b not in onboard_debug + b for b in boards if b.get('debug') and b not in onboard_debug ] if not onboard_debug and not external_debug: return lines @@ -723,7 +723,7 @@ def update_embedded_board(rst_path, board): # lines.append("Debugging") lines.append("---------") - if not board['debug']: + if not board.get('debug'): lines.append( ":ref:`piodebug` currently does not support {name} board.".format( **variables)) @@ -781,7 +781,7 @@ def update_debugging(): platforms = [] frameworks = [] for data in BOARDS: - if not data['debug']: + if not data.get('debug'): continue for tool in data['debug']['tools']: From a1ec3e0a22179c9b743ae76cb04454794fb29191 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 25 Jun 2020 23:23:55 +0300 Subject: [PATCH 085/223] Remove "vendor_url" and "docs_url" from Platform API --- platformio/commands/platform.py | 3 +-- platformio/managers/platform.py | 8 -------- scripts/docspregen.py | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index c4a9ca5d1a..deaeb4317d 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -76,9 +76,8 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru description=p.description, version=p.version, homepage=p.homepage, + url=p.homepage, repository=p.repository_url, - url=p.vendor_url, - docs=p.docs_url, license=p.license, forDesktop=not p.is_embedded(), frameworks=sorted(list(p.frameworks) if p.frameworks else []), diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 52842ca0df..ada4f4accd 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -607,14 +607,6 @@ def src_url(self): def homepage(self): return self._manifest.get("homepage") - @property - def vendor_url(self): - return self._manifest.get("homepage") - - @property - def docs_url(self): - return self._manifest.get("docs") - @property def repository_url(self): return self._manifest.get("repository", {}).get("url") diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 3c498309fc..f2dc275761 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -316,7 +316,7 @@ def generate_platform(name, rst_dir): lines.append(p.description) lines.append(""" For more detailed information please visit `vendor site <%s>`_.""" % - campaign_url(p.vendor_url)) + campaign_url(p.homepage)) lines.append(""" .. contents:: Contents :local: From 0bec1f1585b89508727f8151f070885257eef1f3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 26 Jun 2020 18:38:17 +0300 Subject: [PATCH 086/223] Extend system info with "file system" and "locale" encodings --- platformio/commands/system/command.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index e5d6672105..a684e14a93 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -20,7 +20,7 @@ import click from tabulate import tabulate -from platformio import __version__, proc, util +from platformio import __version__, compat, proc, util from platformio.commands.system.completion import ( get_completion_install_path, install_completion_code, @@ -49,6 +49,14 @@ def system_info(json_output): } data["system"] = {"title": "System Type", "value": util.get_systype()} data["platform"] = {"title": "Platform", "value": platform.platform(terse=True)} + data["filesystem_encoding"] = { + "title": "File System Encoding", + "value": compat.get_filesystem_encoding(), + } + data["locale_encoding"] = { + "title": "Locale Encoding", + "value": compat.get_locale_encoding(), + } data["core_dir"] = { "title": "PlatformIO Core Directory", "value": project_config.get_optional_dir("core"), From bc2eb0d79f45f2ede4ed38e1a5e35ab83d993024 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 26 Jun 2020 19:49:25 +0300 Subject: [PATCH 087/223] Parse dev-platform keywords --- platformio/package/manifest/parser.py | 34 ++++++++++++++------------- tests/package/test_manifest.py | 7 +++--- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 837fabd6aa..720d4ea39e 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -159,6 +159,21 @@ def parse(self, contents): def as_dict(self): return self._data + @staticmethod + def str_to_list(value, sep=",", lowercase=True): + if isinstance(value, string_types): + value = value.split(sep) + assert isinstance(value, list) + result = [] + for item in value: + item = item.strip() + if not item: + continue + if lowercase: + item = item.lower() + result.append(item) + return result + @staticmethod def normalize_author(author): assert isinstance(author, dict) @@ -296,7 +311,7 @@ def parse(self, contents): # normalize Union[str, list] fields for k in ("keywords", "platforms", "frameworks"): if k in data: - data[k] = self._str_to_list(data[k], sep=",") + data[k] = self.str_to_list(data[k], sep=",") if "authors" in data: data["authors"] = self._parse_authors(data["authors"]) @@ -309,21 +324,6 @@ def parse(self, contents): return data - @staticmethod - def _str_to_list(value, sep=",", lowercase=True): - if isinstance(value, string_types): - value = value.split(sep) - assert isinstance(value, list) - result = [] - for item in value: - item = item.strip() - if not item: - continue - if lowercase: - item = item.lower() - result.append(item) - return result - @staticmethod def _process_renamed_fields(data): if "url" in data: @@ -617,6 +617,8 @@ class PlatformJsonManifestParser(BaseManifestParser): def parse(self, contents): data = json.loads(contents) + if "keywords" in data: + data["keywords"] = self.str_to_list(data["keywords"], sep=",") if "frameworks" in data: data["frameworks"] = self._parse_frameworks(data["frameworks"]) if "packages" in data: diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 73acfdafe7..13dff94ee7 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -544,8 +544,8 @@ def test_platform_json_schema(): "name": "atmelavr", "title": "Atmel AVR", "description": "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of performance, power efficiency and design flexibility. Optimized to speed time to market-and easily adapt to new ones-they are based on the industrys most code-efficient architecture for C and assembly programming.", - "url": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", - "homepage": "http://platformio.org/platforms/atmelavr", + "keywords": "arduino, atmel, avr", + "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", "license": "Apache-2.0", "engines": { "platformio": "<5" @@ -603,7 +603,8 @@ def test_platform_json_schema(): "on the industrys most code-efficient architecture for C and " "assembly programming." ), - "homepage": "http://platformio.org/platforms/atmelavr", + "keywords": ["arduino", "atmel", "avr"], + "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", "license": "Apache-2.0", "repository": { "url": "https://github.com/platformio/platform-atmelavr.git", From 29fb803be1559a7dc85a5cb847375a7db0d09ea7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 27 Jun 2020 12:36:57 +0300 Subject: [PATCH 088/223] Enable PIO Core tests on Python 3.8 --- .github/workflows/core.yml | 2 +- platformio/commands/project.py | 3 +-- tests/commands/test_account_org_team.py | 36 ++++++++++++------------- tests/commands/test_check.py | 2 +- tests/commands/test_ci.py | 14 ++++++---- tests/commands/test_lib.py | 16 +++++------ tests/commands/test_platform.py | 22 +++++++-------- tests/commands/test_test.py | 8 +++--- tests/conftest.py | 23 ++++++++++++---- tests/package/test_manifest.py | 3 ++- tests/test_builder.py | 2 +- tests/test_ino2cpp.py | 4 +-- tests/test_maintenance.py | 12 ++++----- tests/test_managers.py | 4 +-- tests/test_misc.py | 4 +-- 15 files changed, 87 insertions(+), 68 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index c6e15bfd9c..e0aed6a973 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [2.7, 3.7] + python-version: [2.7, 3.7, 3.8] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 5bd5efcb81..27e33455ba 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-arguments,too-many-locals, too-many-branches +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,line-too-long import os @@ -238,7 +238,6 @@ def init_include_readme(include_dir): def init_lib_readme(lib_dir): - # pylint: disable=line-too-long with open(os.path.join(lib_dir, "README"), "w") as fp: fp.write( """ diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index 297e4c12a8..b6d5b6d4f1 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -62,7 +62,7 @@ def test_prepare(): def test_account_register( - clirunner, validate_cliresult, receive_email, isolated_pio_home + clirunner, validate_cliresult, receive_email, isolated_pio_core ): result = clirunner.invoke( cmd_account, @@ -97,14 +97,14 @@ def test_account_register( def test_account_login( - clirunner, validate_cliresult, isolated_pio_home, + clirunner, validate_cliresult, isolated_pio_core, ): result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) validate_cliresult(result) def test_account_summary( - clirunner, validate_cliresult, isolated_pio_home, + clirunner, validate_cliresult, isolated_pio_core, ): result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) validate_cliresult(result) @@ -156,7 +156,7 @@ def test_account_summary( assert json_result.get("subscriptions") is not None -def test_account_token(clirunner, validate_cliresult, isolated_pio_home): +def test_account_token(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_account, ["token", "--password", password,],) validate_cliresult(result) assert "Personal Authentication Token:" in result.output @@ -198,7 +198,7 @@ def test_account_token(clirunner, validate_cliresult, isolated_pio_home): validate_cliresult(result) -def test_account_change_password(clirunner, validate_cliresult, isolated_pio_home): +def test_account_change_password(clirunner, validate_cliresult, isolated_pio_core): new_password = "Testpassword123" result = clirunner.invoke( cmd_account, @@ -222,7 +222,7 @@ def test_account_change_password(clirunner, validate_cliresult, isolated_pio_hom def test_account_update( - clirunner, validate_cliresult, receive_email, isolated_pio_home + clirunner, validate_cliresult, receive_email, isolated_pio_core ): global username global email @@ -281,7 +281,7 @@ def test_account_update( # def test_account_destroy_with_linked_resources( -# clirunner, validate_cliresult, receive_email, isolated_pio_home, tmpdir_factory +# clirunner, validate_cliresult, receive_email, isolated_pio_core, tmpdir_factory # ): # package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" # @@ -312,14 +312,14 @@ def test_account_update( # validate_cliresult(result) -def test_org_create(clirunner, validate_cliresult, isolated_pio_home): +def test_org_create(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_org, ["create", "--email", email, "--displayname", display_name, orgname], ) validate_cliresult(result) -def test_org_list(clirunner, validate_cliresult, isolated_pio_home): +def test_org_list(clirunner, validate_cliresult, isolated_pio_core): # pio org list result = clirunner.invoke(cmd_org, ["list", "--json-output"]) validate_cliresult(result) @@ -336,7 +336,7 @@ def test_org_list(clirunner, validate_cliresult, isolated_pio_home): ] -def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_home): +def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_org, ["add", orgname, second_username]) validate_cliresult(result) @@ -345,7 +345,7 @@ def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_home): assert second_username in result.output -def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_home): +def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_org, ["remove", orgname, second_username]) validate_cliresult(result) @@ -354,7 +354,7 @@ def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_home): assert second_username not in result.output -def test_org_update(clirunner, validate_cliresult, isolated_pio_home): +def test_org_update(clirunner, validate_cliresult, isolated_pio_core): new_orgname = "neworg-piocore-%s" % str(random.randint(0, 100000)) new_display_name = "Test Org for PIO Core" @@ -399,7 +399,7 @@ def test_org_update(clirunner, validate_cliresult, isolated_pio_home): validate_cliresult(result) -def test_team_create(clirunner, validate_cliresult, isolated_pio_home): +def test_team_create(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, ["create", "%s:%s" % (orgname, teamname), "--description", team_description,], @@ -407,7 +407,7 @@ def test_team_create(clirunner, validate_cliresult, isolated_pio_home): validate_cliresult(result) -def test_team_list(clirunner, validate_cliresult, isolated_pio_home): +def test_team_list(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) validate_cliresult(result) json_result = json.loads(result.output.strip()) @@ -418,7 +418,7 @@ def test_team_list(clirunner, validate_cliresult, isolated_pio_home): ] -def test_team_add_member(clirunner, validate_cliresult, isolated_pio_home): +def test_team_add_member(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, ["add", "%s:%s" % (orgname, teamname), second_username], ) @@ -429,7 +429,7 @@ def test_team_add_member(clirunner, validate_cliresult, isolated_pio_home): assert second_username in result.output -def test_team_remove(clirunner, validate_cliresult, isolated_pio_home): +def test_team_remove(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, ["remove", "%s:%s" % (orgname, teamname), second_username], ) @@ -440,7 +440,7 @@ def test_team_remove(clirunner, validate_cliresult, isolated_pio_home): assert second_username not in result.output -def test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_home): +def test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_core): new_teamname = "new-" + str(random.randint(0, 100000)) newteam_description = "Updated Description" result = clirunner.invoke( @@ -479,7 +479,7 @@ def test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_ validate_cliresult(result) -def test_cleanup(clirunner, validate_cliresult, receive_email, isolated_pio_home): +def test_cleanup(clirunner, validate_cliresult, receive_email, isolated_pio_core): result = clirunner.invoke(cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y") validate_cliresult(result) result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index c5125b1e95..94064bda18 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -357,7 +357,7 @@ def test_check_fails_on_defects_only_on_specified_level(clirunner, tmpdir): assert low_result.exit_code != 0 -def test_check_pvs_studio_free_license(clirunner, tmpdir): +def test_check_pvs_studio_free_license(clirunner, isolated_pio_core, tmpdir): config = """ [env:test] platform = teensy diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 0f7aceeda2..8a59741352 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -24,7 +24,7 @@ def test_ci_empty(clirunner): assert "Invalid value: Missing argument 'src'" in result.output -def test_ci_boards(clirunner, validate_cliresult): +def test_ci_boards(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_ci, [ @@ -38,7 +38,7 @@ def test_ci_boards(clirunner, validate_cliresult): validate_cliresult(result) -def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): +def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) result = clirunner.invoke( cmd_ci, @@ -54,7 +54,9 @@ def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): assert not isfile(join(build_dir, "platformio.ini")) -def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): +def test_ci_keep_build_dir( + clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core +): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) result = clirunner.invoke( cmd_ci, @@ -88,7 +90,7 @@ def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): assert "board: metro" in result.output -def test_ci_project_conf(clirunner, validate_cliresult): +def test_ci_project_conf(clirunner, validate_cliresult, isolated_pio_core): project_dir = join("examples", "wiring-blink") result = clirunner.invoke( cmd_ci, @@ -102,7 +104,9 @@ def test_ci_project_conf(clirunner, validate_cliresult): assert "uno" in result.output -def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): +def test_ci_lib_and_board( + clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core +): storage_dir = str(tmpdir_factory.mktemp("lib")) result = clirunner.invoke( cmd_lib, ["--storage-dir", storage_dir, "install", "1@2.3.2"] diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 752c2c30bc..3341ef71bc 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -34,7 +34,7 @@ def test_search(clirunner, validate_cliresult): assert int(match.group(1)) > 1 -def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_home): +def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_lib, [ @@ -54,7 +54,7 @@ def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_hom assert result.exit_code != 0 assert isinstance(result.exception, exception.LibNotFound) - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "ArduinoJson_ID64", "ArduinoJson_ID64@5.10.1", @@ -68,7 +68,7 @@ def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_hom assert set(items1) == set(items2) -def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_home): +def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_lib, [ @@ -93,12 +93,12 @@ def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_home ) assert result.exit_code != 0 - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire_ID1", "ESP32WebServer"] assert set(items1) >= set(items2) -def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_home): +def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_lib, [ @@ -114,7 +114,7 @@ def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_h ], ) validate_cliresult(result) - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d", @@ -260,7 +260,7 @@ def test_global_lib_update(clirunner, validate_cliresult): assert isinstance(result.exception, exception.UnknownPackage) -def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): +def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): # uninstall using package directory result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) validate_cliresult(result) @@ -284,7 +284,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): ) validate_cliresult(result) - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "rs485-nodeproto", "platformio-libmirror", diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index b2db5d83f9..377c1e28df 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -18,7 +18,7 @@ from platformio.commands import platform as cli_platform -def test_search_json_output(clirunner, validate_cliresult, isolated_pio_home): +def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_search, ["arduino", "--json-output"] ) @@ -48,7 +48,7 @@ def test_install_unknown_from_registry(clirunner): assert isinstance(result.exception, exception.UnknownPackage) -def test_install_known_version(clirunner, validate_cliresult, isolated_pio_home): +def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package", "--with-package", "tool-avrdude"], @@ -56,10 +56,10 @@ def test_install_known_version(clirunner, validate_cliresult, isolated_pio_home) validate_cliresult(result) assert "atmelavr @ 1.2.0" in result.output assert "Installing tool-avrdude @" in result.output - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 -def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home): +def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, [ @@ -69,7 +69,7 @@ def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home): ) validate_cliresult(result) assert "espressif8266" in result.output - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 def test_list_json_output(clirunner, validate_cliresult): @@ -88,7 +88,7 @@ def test_list_raw_output(clirunner, validate_cliresult): assert all([s in result.output for s in ("atmelavr", "espressif8266")]) -def test_update_check(clirunner, validate_cliresult, isolated_pio_home): +def test_update_check(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_update, ["--only-check", "--json-output"] ) @@ -96,20 +96,20 @@ def test_update_check(clirunner, validate_cliresult, isolated_pio_home): output = json.loads(result.output) assert len(output) == 1 assert output[0]["name"] == "atmelavr" - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 -def test_update_raw(clirunner, validate_cliresult, isolated_pio_home): +def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cli_platform.platform_update) validate_cliresult(result) assert "Uninstalling atmelavr @ 1.2.0:" in result.output assert "PlatformManager: Installing atmelavr @" in result.output - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 -def test_uninstall(clirunner, validate_cliresult, isolated_pio_home): +def test_uninstall(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_uninstall, ["atmelavr", "espressif8266"] ) validate_cliresult(result) - assert not isolated_pio_home.join("platforms").listdir() + assert not isolated_pio_core.join("platforms").listdir() diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 4110188ff5..608b03107e 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -20,7 +20,7 @@ from platformio.commands.test.command import cli as cmd_test -def test_local_env(): +def test_local_env(isolated_pio_core): result = util.exec_command( [ "platformio", @@ -38,7 +38,7 @@ def test_local_env(): ] -def test_multiple_env_build(clirunner, validate_cliresult, tmpdir): +def test_multiple_env_build(clirunner, validate_cliresult, isolated_pio_core, tmpdir): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write( @@ -80,7 +80,9 @@ def test_multiple_env_build(clirunner, validate_cliresult, tmpdir): assert "Multiple ways to build" not in result.output -def test_setup_teardown_are_compilable(clirunner, validate_cliresult, tmpdir): +def test_setup_teardown_are_compilable( + clirunner, validate_cliresult, isolated_pio_core, tmpdir +): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write( diff --git a/tests/conftest.py b/tests/conftest.py index eda521848f..0978b2efab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,15 +42,28 @@ def clirunner(): @pytest.fixture(scope="module") -def isolated_pio_home(request, tmpdir_factory): - home_dir = tmpdir_factory.mktemp(".platformio") - os.environ["PLATFORMIO_CORE_DIR"] = str(home_dir) +def isolated_pio_core(request, tmpdir_factory): + core_dir = tmpdir_factory.mktemp(".platformio") + backup_env_vars = { + "PLATFORMIO_CORE_DIR": {"new": str(core_dir)}, + "PLATFORMIO_WORKSPACE_DIR": {"new": None}, + } + for key, item in backup_env_vars.items(): + backup_env_vars[key]["old"] = os.environ.get(key) + if item["new"] is not None: + os.environ[key] = item["new"] + elif key in os.environ: + del os.environ[key] def fin(): - del os.environ["PLATFORMIO_CORE_DIR"] + for key, item in backup_env_vars.items(): + if item["old"] is None: + del os.environ[key] + else: + os.environ[key] = item["old"] request.addfinalizer(fin) - return home_dir + return core_dir @pytest.fixture(scope="function") diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 13dff94ee7..cef5900d12 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -83,7 +83,7 @@ def test_library_json_parser(): }, "dependencies": [ {"name": "deps1", "version": "1.0.0"}, - {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino, espidf"}, + {"name": "@owner/deps2", "version": "1.0.0", "platforms": "atmelavr, espressif32", "frameworks": "arduino, espidf"}, {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} ] } @@ -101,6 +101,7 @@ def test_library_json_parser(): { "name": "@owner/deps2", "version": "1.0.0", + "platforms": ["atmelavr", "espressif32"], "frameworks": ["arduino", "espidf"], }, {"name": "deps1", "version": "1.0.0"}, diff --git a/tests/test_builder.py b/tests/test_builder.py index f220e50c3f..dd0fe1f256 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -15,7 +15,7 @@ from platformio.commands.run.command import cli as cmd_run -def test_build_flags(clirunner, validate_cliresult, tmpdir): +def test_build_flags(clirunner, isolated_pio_core, validate_cliresult, tmpdir): build_flags = [ ("-D TEST_INT=13", "-DTEST_INT=13"), ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), diff --git a/tests/test_ino2cpp.py b/tests/test_ino2cpp.py index d1434df731..af91da8dd2 100644 --- a/tests/test_ino2cpp.py +++ b/tests/test_ino2cpp.py @@ -31,12 +31,12 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("piotest_dir", test_dirs) -def test_example(clirunner, validate_cliresult, piotest_dir): +def test_example(clirunner, isolated_pio_core, validate_cliresult, piotest_dir): result = clirunner.invoke(cmd_ci, [piotest_dir, "-b", "uno"]) validate_cliresult(result) -def test_warning_line(clirunner, validate_cliresult): +def test_warning_line(clirunner, isolated_pio_core, validate_cliresult): result = clirunner.invoke(cmd_ci, [join(INOTEST_DIR, "basic"), "-b", "uno"]) validate_cliresult(result) assert 'basic.ino:16:14: warning: #warning "Line number is 16"' in result.output diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index c004f28f87..34d4ce6865 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -22,7 +22,7 @@ from platformio.managers.platform import PlatformManager -def test_check_pio_upgrade(clirunner, isolated_pio_home, validate_cliresult): +def test_check_pio_upgrade(clirunner, isolated_pio_core, validate_cliresult): def _patch_pio_version(version): maintenance.__version__ = version cmd_upgrade.VERSION = version.split(".", 3) @@ -51,7 +51,7 @@ def _patch_pio_version(version): _patch_pio_version(origin_version) -def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): +def test_check_lib_updates(clirunner, isolated_pio_core, validate_cliresult): # install obsolete library result = clirunner.invoke(cli_pio, ["lib", "-g", "install", "ArduinoJson@<6.13"]) validate_cliresult(result) @@ -66,7 +66,7 @@ def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): assert "There are the new updates for libraries (ArduinoJson)" in result.output -def test_check_and_update_libraries(clirunner, isolated_pio_home, validate_cliresult): +def test_check_and_update_libraries(clirunner, isolated_pio_core, validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_libraries", "Yes"] @@ -96,11 +96,11 @@ def test_check_and_update_libraries(clirunner, isolated_pio_home, validate_clire assert prev_data[0]["version"] != json.loads(result.output)[0]["version"] -def test_check_platform_updates(clirunner, isolated_pio_home, validate_cliresult): +def test_check_platform_updates(clirunner, isolated_pio_core, validate_cliresult): # install obsolete platform result = clirunner.invoke(cli_pio, ["platform", "install", "native"]) validate_cliresult(result) - manifest_path = isolated_pio_home.join("platforms", "native", "platform.json") + manifest_path = isolated_pio_core.join("platforms", "native", "platform.json") manifest = json.loads(manifest_path.read()) manifest["version"] = "0.0.0" manifest_path.write(json.dumps(manifest)) @@ -117,7 +117,7 @@ def test_check_platform_updates(clirunner, isolated_pio_home, validate_cliresult assert "There are the new updates for platforms (native)" in result.output -def test_check_and_update_platforms(clirunner, isolated_pio_home, validate_cliresult): +def test_check_and_update_platforms(clirunner, isolated_pio_core, validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_platforms", "Yes"] diff --git a/tests/test_managers.py b/tests/test_managers.py index f4ab2ed852..308523cd1e 100644 --- a/tests/test_managers.py +++ b/tests/test_managers.py @@ -165,7 +165,7 @@ def test_pkg_input_parser(): assert PackageManager.parse_pkg_uri(params) == result -def test_install_packages(isolated_pio_home, tmpdir): +def test_install_packages(isolated_pio_core, tmpdir): packages = [ dict(id=1, name="name_1", version="shasum"), dict(id=1, name="name_1", version="2.0.0"), @@ -198,7 +198,7 @@ def test_install_packages(isolated_pio_home, tmpdir): "name_2@src-f863b537ab00f4c7b5011fc44b120e1f", ] assert set( - [p.basename for p in isolated_pio_home.join("packages").listdir()] + [p.basename for p in isolated_pio_core.join("packages").listdir()] ) == set(pkg_dirnames) diff --git a/tests/test_misc.py b/tests/test_misc.py index aee9f113e3..d01cc46d41 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -29,12 +29,12 @@ def test_ping_internet_ips(): requests.get("http://%s" % host, allow_redirects=False, timeout=2) -def test_api_internet_offline(without_internet, isolated_pio_home): +def test_api_internet_offline(without_internet, isolated_pio_core): with pytest.raises(exception.InternetIsOffline): util.get_api_result("/stats") -def test_api_cache(monkeypatch, isolated_pio_home): +def test_api_cache(monkeypatch, isolated_pio_core): api_kwargs = {"url": "/stats", "cache_valid": "10s"} result = util.get_api_result(**api_kwargs) assert result and "boards" in result From b046f21e0de3ddd9d933e143606a1cbda188a57d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 27 Jun 2020 12:46:04 +0300 Subject: [PATCH 089/223] Fix "RuntimeError: dictionary keys changed during iteration" when parsing "library.json" dependencies --- platformio/package/manifest/parser.py | 2 -- tests/package/test_manifest.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 720d4ea39e..193fbde1a1 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -384,8 +384,6 @@ def _parse_dependencies(raw): for k, v in dependency.items(): if k not in ("platforms", "frameworks", "authors"): continue - if "*" in v: - del raw[i][k] raw[i][k] = util.items_to_list(v) else: raw[i] = {"name": dependency} diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index cef5900d12..332cb7b5b7 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -83,7 +83,7 @@ def test_library_json_parser(): }, "dependencies": [ {"name": "deps1", "version": "1.0.0"}, - {"name": "@owner/deps2", "version": "1.0.0", "platforms": "atmelavr, espressif32", "frameworks": "arduino, espidf"}, + {"name": "@owner/deps2", "version": "1.0.0", "platforms": "*", "frameworks": "arduino, espidf"}, {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} ] } @@ -101,7 +101,7 @@ def test_library_json_parser(): { "name": "@owner/deps2", "version": "1.0.0", - "platforms": ["atmelavr", "espressif32"], + "platforms": ["*"], "frameworks": ["arduino", "espidf"], }, {"name": "deps1", "version": "1.0.0"}, From dd18abcac36721b0ba4d6c031af81720064aaf84 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 27 Jun 2020 12:59:12 +0300 Subject: [PATCH 090/223] Fix tests --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0978b2efab..438ced132d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,10 +57,10 @@ def isolated_pio_core(request, tmpdir_factory): def fin(): for key, item in backup_env_vars.items(): - if item["old"] is None: - del os.environ[key] - else: + if item["old"] is not None: os.environ[key] = item["old"] + elif key in os.environ: + del os.environ[key] request.addfinalizer(fin) return core_dir From e9a15b4e9bdd03934ebd3a376f4ec0749c1be3c4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 27 Jun 2020 21:42:13 +0300 Subject: [PATCH 091/223] Parse package.json manifest keywords --- platformio/package/manifest/parser.py | 2 ++ tests/package/test_manifest.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 193fbde1a1..86c98587c2 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -641,6 +641,8 @@ class PackageJsonManifestParser(BaseManifestParser): def parse(self, contents): data = json.loads(contents) + if "keywords" in data: + data["keywords"] = self.str_to_list(data["keywords"], sep=",") data = self._parse_system(data) data = self._parse_homepage(data) return data diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 332cb7b5b7..0c89e01293 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -627,6 +627,7 @@ def test_package_json_schema(): { "name": "tool-scons", "description": "SCons software construction tool", + "keywords": "SCons, build", "url": "http://www.scons.org", "version": "3.30101.0" } @@ -642,6 +643,7 @@ def test_package_json_schema(): { "name": "tool-scons", "description": "SCons software construction tool", + "keywords": ["scons", "build"], "homepage": "http://www.scons.org", "version": "3.30101.0", }, From 2b8aebbdf987339352518e38de09825292e7450d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 29 Jun 2020 15:06:21 +0300 Subject: [PATCH 092/223] Extend test for parsing package manifest when "system" is used as a list --- tests/package/test_manifest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 0c89e01293..884d6af097 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -628,7 +628,8 @@ def test_package_json_schema(): "name": "tool-scons", "description": "SCons software construction tool", "keywords": "SCons, build", - "url": "http://www.scons.org", + "homepage": "http://www.scons.org", + "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l"], "version": "3.30101.0" } """ @@ -645,6 +646,7 @@ def test_package_json_schema(): "description": "SCons software construction tool", "keywords": ["scons", "build"], "homepage": "http://www.scons.org", + "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l"], "version": "3.30101.0", }, ) From 4cbad399f787211e2d13ba9e83757cd969a596a6 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Mon, 29 Jun 2020 19:22:22 +0300 Subject: [PATCH 093/223] Remove mbed framework from several tests --- tests/commands/test_check.py | 2 +- tests/commands/test_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 94064bda18..116e4a5288 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -412,7 +412,7 @@ def test_check_embedded_platform_all_tools(clirunner, tmpdir): """ ) - frameworks = ["arduino", "mbed", "stm32cube"] + frameworks = ["arduino", "stm32cube"] if sys.version_info[0] == 3: # Zephyr only supports Python 3 frameworks.append("zephyr") diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 608b03107e..04a531ed7d 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -45,7 +45,7 @@ def test_multiple_env_build(clirunner, validate_cliresult, isolated_pio_core, tm """ [env:teensy31] platform = teensy -framework = mbed +framework = arduino board = teensy31 [env:native] From 1ac6c5033402ea6756d9be03c1004ec7224eb186 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Mon, 29 Jun 2020 20:52:15 +0300 Subject: [PATCH 094/223] Update multi-environment test for PIO test command --- tests/commands/test_test.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 04a531ed7d..2b6dc348b4 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import join +import os +import sys import pytest from platformio import util from platformio.commands.test.command import cli as cmd_test +from platformio.compat import WINDOWS def test_local_env(isolated_pio_core): @@ -26,7 +28,7 @@ def test_local_env(isolated_pio_core): "platformio", "test", "-d", - join("examples", "unit-testing", "calculator"), + os.path.join("examples", "unit-testing", "calculator"), "-e", "native", ] @@ -51,24 +53,27 @@ def test_multiple_env_build(clirunner, validate_cliresult, isolated_pio_core, tm [env:native] platform = native -[env:espressif32] -platform = espressif32 +[env:espressif8266] +platform = espressif8266 framework = arduino -board = esp32dev +board = nodemcuv2 """ ) project_dir.mkdir("test").join("test_main.cpp").write( """ +#include #ifdef ARDUINO -void setup() {} -void loop() {} +void setup() #else -int main() { +int main() +#endif +{ UNITY_BEGIN(); UNITY_END(); + } -#endif +void loop() {} """ ) From 5cdca9d49045b2c8b8513994571dc8e5ef583d99 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 29 Jun 2020 21:14:34 +0300 Subject: [PATCH 095/223] Optimize tests --- tests/commands/test_ci.py | 14 +++++--------- tests/commands/test_test.py | 10 +++------- tests/conftest.py | 24 +++++++++++++++--------- tests/test_builder.py | 2 +- tests/test_ino2cpp.py | 4 ++-- 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 8a59741352..0f7aceeda2 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -24,7 +24,7 @@ def test_ci_empty(clirunner): assert "Invalid value: Missing argument 'src'" in result.output -def test_ci_boards(clirunner, validate_cliresult, isolated_pio_core): +def test_ci_boards(clirunner, validate_cliresult): result = clirunner.invoke( cmd_ci, [ @@ -38,7 +38,7 @@ def test_ci_boards(clirunner, validate_cliresult, isolated_pio_core): validate_cliresult(result) -def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core): +def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) result = clirunner.invoke( cmd_ci, @@ -54,9 +54,7 @@ def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult, isolated_pi assert not isfile(join(build_dir, "platformio.ini")) -def test_ci_keep_build_dir( - clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core -): +def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) result = clirunner.invoke( cmd_ci, @@ -90,7 +88,7 @@ def test_ci_keep_build_dir( assert "board: metro" in result.output -def test_ci_project_conf(clirunner, validate_cliresult, isolated_pio_core): +def test_ci_project_conf(clirunner, validate_cliresult): project_dir = join("examples", "wiring-blink") result = clirunner.invoke( cmd_ci, @@ -104,9 +102,7 @@ def test_ci_project_conf(clirunner, validate_cliresult, isolated_pio_core): assert "uno" in result.output -def test_ci_lib_and_board( - clirunner, tmpdir_factory, validate_cliresult, isolated_pio_core -): +def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): storage_dir = str(tmpdir_factory.mktemp("lib")) result = clirunner.invoke( cmd_lib, ["--storage-dir", storage_dir, "install", "1@2.3.2"] diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 2b6dc348b4..16e0556cf0 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -13,16 +13,14 @@ # limitations under the License. import os -import sys import pytest from platformio import util from platformio.commands.test.command import cli as cmd_test -from platformio.compat import WINDOWS -def test_local_env(isolated_pio_core): +def test_local_env(): result = util.exec_command( [ "platformio", @@ -40,7 +38,7 @@ def test_local_env(isolated_pio_core): ] -def test_multiple_env_build(clirunner, validate_cliresult, isolated_pio_core, tmpdir): +def test_multiple_env_build(clirunner, validate_cliresult, tmpdir): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write( @@ -85,9 +83,7 @@ def test_multiple_env_build(clirunner, validate_cliresult, isolated_pio_core, tm assert "Multiple ways to build" not in result.output -def test_setup_teardown_are_compilable( - clirunner, validate_cliresult, isolated_pio_core, tmpdir -): +def test_setup_teardown_are_compilable(clirunner, validate_cliresult, tmpdir): project_dir = tmpdir.mkdir("project") project_dir.join("platformio.ini").write( diff --git a/tests/conftest.py b/tests/conftest.py index 438ced132d..56a59cbd9b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,16 +36,9 @@ def decorator(result): return decorator -@pytest.fixture(scope="module") -def clirunner(): - return CliRunner() - - -@pytest.fixture(scope="module") -def isolated_pio_core(request, tmpdir_factory): - core_dir = tmpdir_factory.mktemp(".platformio") +@pytest.fixture(scope="session") +def clirunner(request): backup_env_vars = { - "PLATFORMIO_CORE_DIR": {"new": str(core_dir)}, "PLATFORMIO_WORKSPACE_DIR": {"new": None}, } for key, item in backup_env_vars.items(): @@ -62,6 +55,19 @@ def fin(): elif key in os.environ: del os.environ[key] + request.addfinalizer(fin) + + return CliRunner() + + +@pytest.fixture(scope="module") +def isolated_pio_core(request, tmpdir_factory): + core_dir = tmpdir_factory.mktemp(".platformio") + os.environ["PLATFORMIO_CORE_DIR"] = str(core_dir) + + def fin(): + del os.environ["PLATFORMIO_CORE_DIR"] + request.addfinalizer(fin) return core_dir diff --git a/tests/test_builder.py b/tests/test_builder.py index dd0fe1f256..f220e50c3f 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -15,7 +15,7 @@ from platformio.commands.run.command import cli as cmd_run -def test_build_flags(clirunner, isolated_pio_core, validate_cliresult, tmpdir): +def test_build_flags(clirunner, validate_cliresult, tmpdir): build_flags = [ ("-D TEST_INT=13", "-DTEST_INT=13"), ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), diff --git a/tests/test_ino2cpp.py b/tests/test_ino2cpp.py index af91da8dd2..d1434df731 100644 --- a/tests/test_ino2cpp.py +++ b/tests/test_ino2cpp.py @@ -31,12 +31,12 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("piotest_dir", test_dirs) -def test_example(clirunner, isolated_pio_core, validate_cliresult, piotest_dir): +def test_example(clirunner, validate_cliresult, piotest_dir): result = clirunner.invoke(cmd_ci, [piotest_dir, "-b", "uno"]) validate_cliresult(result) -def test_warning_line(clirunner, isolated_pio_core, validate_cliresult): +def test_warning_line(clirunner, validate_cliresult): result = clirunner.invoke(cmd_ci, [join(INOTEST_DIR, "basic"), "-b", "uno"]) validate_cliresult(result) assert 'basic.ino:16:14: warning: #warning "Line number is 16"' in result.output From 2c24e9eff6b0797ff44673136801dbfa5f5156aa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 30 Jun 2020 14:28:37 +0300 Subject: [PATCH 096/223] Fall back to latin-1 encoding when failed with UTF-8 while parsing manifest --- platformio/package/manifest/parser.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 86c98587c2..0b82647934 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -61,8 +61,14 @@ def from_dir(cls, path): class ManifestParserFactory(object): @staticmethod def read_manifest_contents(path): - with io.open(path, encoding="utf-8") as fp: - return fp.read() + last_err = None + for encoding in ("utf-8", "latin-1"): + try: + with io.open(path, encoding=encoding) as fp: + return fp.read() + except UnicodeDecodeError as e: + last_err = e + raise last_err @classmethod def new_from_file(cls, path, remote_url=False): From 7f48c8c14ec3cc7cf36071f2e23ca0a6011c5d73 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 30 Jun 2020 15:06:40 +0300 Subject: [PATCH 097/223] Fix PyLint for PY 2.7 --- platformio/package/manifest/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 0b82647934..cb0051e022 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -68,7 +68,7 @@ def read_manifest_contents(path): return fp.read() except UnicodeDecodeError as e: last_err = e - raise last_err + raise last_err # pylint: disable=raising-bad-type @classmethod def new_from_file(cls, path, remote_url=False): From 899a6734ee58581a3b2b38583b3d74a30afb5afa Mon Sep 17 00:00:00 2001 From: Rosen Stoyanov Date: Tue, 30 Jun 2020 21:48:44 +0300 Subject: [PATCH 098/223] Add .ccls to .gitignore (vim and emacs) (#3576) * Add .ccls to .gitignore (vim) * Add .ccls to .gitignore (emacs) --- platformio/ide/tpls/emacs/.gitignore.tpl | 1 + platformio/ide/tpls/vim/.gitignore.tpl | 1 + 2 files changed, 2 insertions(+) diff --git a/platformio/ide/tpls/emacs/.gitignore.tpl b/platformio/ide/tpls/emacs/.gitignore.tpl index b8e379fa64..6f8bafd31b 100644 --- a/platformio/ide/tpls/emacs/.gitignore.tpl +++ b/platformio/ide/tpls/emacs/.gitignore.tpl @@ -1,2 +1,3 @@ .pio .clang_complete +.ccls diff --git a/platformio/ide/tpls/vim/.gitignore.tpl b/platformio/ide/tpls/vim/.gitignore.tpl index bbdd36c798..1159b2d8b4 100644 --- a/platformio/ide/tpls/vim/.gitignore.tpl +++ b/platformio/ide/tpls/vim/.gitignore.tpl @@ -1,3 +1,4 @@ .pio .clang_complete .gcc-flags.json +.ccls From b3dabb221d8aaf4d73858d146dde057589995b17 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Jul 2020 16:07:36 +0300 Subject: [PATCH 099/223] Allow "+" in a package name --- platformio/package/pack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 1e18c55ae6..0e3921fa44 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -55,7 +55,7 @@ def pack(self, dst=None): manifest = self.load_manifest(src) filename = re.sub( - r"[^\da-zA-Z\-\._]+", + r"[^\da-zA-Z\-\._\+]+", "", "{name}{system}-{version}.tar.gz".format( name=manifest["name"], From 08a87f3a21db3168ac17933d67da51547fcc0eed Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Jul 2020 19:14:58 +0300 Subject: [PATCH 100/223] Do not allow [;.<>] chars for a package name --- platformio/package/manifest/schema.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 3502550a5c..e19e6f259a 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -153,7 +153,9 @@ class ManifestSchema(BaseSchema): required=True, validate=[ validate.Length(min=1, max=100), - validate.Regexp(r"^[^:/]+$", error="The next chars [:/] are not allowed"), + validate.Regexp( + r"^[^:;/,@\<\>]+$", error="The next chars [:;/,@<>] are not allowed" + ), ], ) version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) From ef53bcf601c02c5091bedcbe86cacd24113f6cc9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Jul 2020 14:17:00 +0300 Subject: [PATCH 101/223] Ignore empty fields in library.properties manifest --- platformio/package/manifest/parser.py | 2 ++ tests/package/test_manifest.py | 1 + 2 files changed, 3 insertions(+) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index cb0051e022..d49d7b9ce7 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -482,6 +482,8 @@ def _parse_properties(contents): if line.startswith("#"): continue key, value = line.split("=", 1) + if not value.strip(): + continue data[key.strip()] = value.strip() return data diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 884d6af097..53a31e9bad 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -202,6 +202,7 @@ def test_library_properties_parser(): sentence=This is Arduino library customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third +ignore_empty_field= """ raw_data = parser.LibraryPropertiesManifestParser(contents).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) From a79e933c377a459d8a5642c8d01fa914dc9e0d63 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Jul 2020 14:22:35 +0300 Subject: [PATCH 102/223] Ignore author's broken email in a package manifest --- platformio/package/manifest/parser.py | 2 ++ tests/package/test_manifest.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index d49d7b9ce7..66dee43c6c 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -185,6 +185,8 @@ def normalize_author(author): assert isinstance(author, dict) if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) + if "@" not in author["email"]: + author["email"] = None for key in list(author.keys()): if author[key] is None: del author[key] diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 53a31e9bad..fe5054d0a3 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -279,7 +279,7 @@ def test_library_properties_parser(): # Author + Maintainer data = parser.LibraryPropertiesManifestParser( """ -author=Rocket Scream Electronics +author=Rocket Scream Electronics maintainer=Rocket Scream Electronics """ ).as_dict() From f97632202b70bfcc230218aee54f8d9045ce5240 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Jul 2020 15:57:10 +0300 Subject: [PATCH 103/223] Fix issue with KeyError --- platformio/commands/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index d08529fdce..f635bff18e 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -445,7 +445,7 @@ def lib_show(library, json_output): for author in lib.get("authors", []): _data = [] for key in ("name", "email", "url", "maintainer"): - if not author[key]: + if not author.get(key): continue if key == "email": _data.append("<%s>" % author[key]) From 0f8042eeb4a03762211ddfe1a1a1853ab45bf891 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Jul 2020 15:57:49 +0300 Subject: [PATCH 104/223] Implement PackagePacker.get_archive_name API --- platformio/package/pack.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 0e3921fa44..4cbaa6c07e 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -40,6 +40,16 @@ def __init__(self, package, manifest_uri=None): self.package = package self.manifest_uri = manifest_uri + @staticmethod + def get_archive_name(name, version, system=None): + return re.sub( + r"[^\da-zA-Z\-\._\+]+", + "", + "{name}{system}-{version}.tar.gz".format( + name=name, system=("-" + system) if system else "", version=version, + ), + ) + def pack(self, dst=None): tmp_dir = tempfile.mkdtemp() try: @@ -54,14 +64,10 @@ def pack(self, dst=None): src = self.find_source_root(src) manifest = self.load_manifest(src) - filename = re.sub( - r"[^\da-zA-Z\-\._\+]+", - "", - "{name}{system}-{version}.tar.gz".format( - name=manifest["name"], - system="-" + manifest["system"][0] if "system" in manifest else "", - version=manifest["version"], - ), + filename = self.get_archive_name( + manifest["name"], + manifest["version"], + manifest["system"][0] if "system" in manifest else None, ) if not dst: From 8b24b0f657e7482d4b5f91ea4584beef16029329 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Jul 2020 23:37:28 +0300 Subject: [PATCH 105/223] Sync docs & examples --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 438d910cc4..b3e5044e92 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 438d910cc4d35337df90a6b23955f89a0675b52c +Subproject commit b3e5044e92390eb954811e628cddffd6dc17818b diff --git a/examples b/examples index 2fdead5df5..0507e27c58 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 2fdead5df5f146c6f7c70eeef63669bebab7a225 +Subproject commit 0507e27c58dbc184420658762b1037348f5f0e87 From 3c986ed681cb50625a832424bffa2c711559053e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 Jul 2020 16:28:51 +0300 Subject: [PATCH 106/223] Remove recursively .pio folders when packing a package --- platformio/package/pack.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 4cbaa6c07e..0145a61e22 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -28,11 +28,13 @@ class PackagePacker(object): EXCLUDE_DEFAULT = [ "._*", + "__*", ".DS_Store", - ".git", - ".hg", - ".svn", - ".pio", + ".git/", + ".hg/", + ".svn/", + ".pio/", + "**/.pio/", ] INCLUDE_DEFAULT = ManifestFileType.items().values() From abd3f8b3b5d6d6324b0f1d728d30f1c13c9324ef Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 Jul 2020 22:53:01 +0300 Subject: [PATCH 107/223] Docs: Remove legacy library dependency syntax for github --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b3e5044e92..7d24530a56 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b3e5044e92390eb954811e628cddffd6dc17818b +Subproject commit 7d24530a566df810d49c3a04398c94a31b7be14a From 40d6847c96211e7316edd457bd3fef08fddc5b67 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 13:46:36 +0300 Subject: [PATCH 108/223] Add option to pass a custom path where to save package archive --- platformio/commands/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index e5eb7db66b..5a3092e2c7 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -39,9 +39,12 @@ def cli(): @cli.command("pack", short_help="Create a tarball from a package") @click.argument("package", required=True, metavar="") -def package_pack(package): +@click.option( + "-o", "--output", help="A destination path (folder or a full path to file)" +) +def package_pack(package, output): p = PackagePacker(package) - archive_path = p.pack() + archive_path = p.pack(output) click.secho('Wrote a tarball to "%s"' % archive_path, fg="green") From 42fd28456028094919d1db5c9df12932a355f3c8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 20:21:10 +0300 Subject: [PATCH 109/223] Improve parsing "author" field of library.properties manfiest --- platformio/package/manifest/parser.py | 11 +++++++---- tests/package/test_manifest.py | 7 +++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 66dee43c6c..28743e5436 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -198,10 +198,13 @@ def parse_author_name_and_email(raw): return (None, None) name = raw email = None - for ldel, rdel in [("<", ">"), ("(", ")")]: - if ldel in raw and rdel in raw: - name = raw[: raw.index(ldel)] - email = raw[raw.index(ldel) + 1 : raw.index(rdel)] + ldel = "<" + rdel = ">" + if ldel in raw and rdel in raw: + name = raw[: raw.index(ldel)] + email = raw[raw.index(ldel) + 1 : raw.index(rdel)] + if "(" in name: + name = name.split("(")[0] return (name.strip(), email.strip() if email else None) @staticmethod diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index fe5054d0a3..c243632837 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -198,7 +198,7 @@ def test_library_properties_parser(): contents = """ name=TestPackage version=1.2.3 -author=SomeAuthor +author=SomeAuthor , Another Author (nickname) sentence=This is Arduino library customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third @@ -218,7 +218,10 @@ def test_library_properties_parser(): "export": { "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, - "authors": [{"email": "info@author.com", "name": "SomeAuthor"}], + "authors": [ + {"email": "info@author.com", "name": "SomeAuthor"}, + {"name": "Another Author"}, + ], "keywords": ["uncategorized"], "customField": "Custom Value", "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third", From 84132d9459a8249b1e969122a570ae2a3cec9a89 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 21:52:34 +0300 Subject: [PATCH 110/223] Fix tests --- tests/commands/test_check.py | 2 +- tests/commands/test_ci.py | 4 ++-- tests/commands/test_lib.py | 35 +++++++++++++++-------------------- tests/commands/test_update.py | 4 ++-- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 116e4a5288..998d44cf3a 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -357,7 +357,7 @@ def test_check_fails_on_defects_only_on_specified_level(clirunner, tmpdir): assert low_result.exit_code != 0 -def test_check_pvs_studio_free_license(clirunner, isolated_pio_core, tmpdir): +def test_check_pvs_studio_free_license(clirunner, tmpdir): config = """ [env:test] platform = teensy diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 0f7aceeda2..f3308a6a46 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -114,13 +114,13 @@ def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): [ join( storage_dir, - "OneWire_ID1", + "OneWire", "examples", "DS2408_Switch", "DS2408_Switch.pde", ), "-l", - join(storage_dir, "OneWire_ID1"), + join(storage_dir, "OneWire"), "-b", "uno", ], diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 3341ef71bc..f51b9dc250 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -56,14 +56,14 @@ def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_cor items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ - "ArduinoJson_ID64", - "ArduinoJson_ID64@5.10.1", - "NeoPixelBus_ID547", - "AsyncMqttClient_ID346", - "ESPAsyncTCP_ID305", - "AsyncTCP_ID1826", - "Adafruit PN532_ID29", - "Adafruit BusIO_ID6214", + "ArduinoJson", + "ArduinoJson@5.10.1", + "NeoPixelBus", + "AsyncMqttClient", + "ESPAsyncTCP", + "AsyncTCP", + "Adafruit PN532", + "Adafruit BusIO", ] assert set(items1) == set(items2) @@ -94,7 +94,7 @@ def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core assert result.exit_code != 0 items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] - items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire_ID1", "ESP32WebServer"] + items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire", "ESP32WebServer"] assert set(items1) >= set(items2) @@ -135,11 +135,6 @@ def test_install_duplicates(clirunner, validate_cliresult, without_internet): validate_cliresult(result) assert "is already installed" in result.output - # by ID - result = clirunner.invoke(cmd_lib, ["-g", "install", "29"]) - validate_cliresult(result) - assert "is already installed" in result.output - # archive result = clirunner.invoke( cmd_lib, @@ -276,7 +271,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): [ "-g", "uninstall", - "1", + "OneWire", "https://github.com/bblanchon/ArduinoJson.git", "ArduinoJson@!=5.6.7", "Adafruit PN532", @@ -290,15 +285,15 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): "platformio-libmirror", "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "ESPAsyncTCP_ID305", + "ESPAsyncTCP", "ESP32WebServer", - "NeoPixelBus_ID547", + "NeoPixelBus", "PJON", - "AsyncMqttClient_ID346", - "ArduinoJson_ID64", + "AsyncMqttClient", + "ArduinoJson", "SomeLib_ID54", "PJON@src-79de467ebe19de18287becff0a1fb42d", - "AsyncTCP_ID1826", + "AsyncTCP", ] assert set(items1) == set(items2) diff --git a/tests/commands/test_update.py b/tests/commands/test_update.py index 9325e5017a..1817be339f 100644 --- a/tests/commands/test_update.py +++ b/tests/commands/test_update.py @@ -15,8 +15,8 @@ from platformio.commands.update import cli as cmd_update -def test_update(clirunner, validate_cliresult): - matches = ("Platform Manager", "Up-to-date", "Library Manager") +def test_update(clirunner, validate_cliresult, isolated_pio_core): + matches = ("Platform Manager", "Library Manager") result = clirunner.invoke(cmd_update, ["--only-check"]) validate_cliresult(result) assert all([m in result.output for m in matches]) From a00722bef4599fa3f67f81368fe60e2651e4ff19 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 21:53:28 +0300 Subject: [PATCH 111/223] Ignore maintainer's broken email in library.properties manifest --- platformio/package/manifest/parser.py | 16 +++++++++------- tests/package/test_manifest.py | 7 ++++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 28743e5436..8ba94344b4 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -181,7 +181,7 @@ def str_to_list(value, sep=",", lowercase=True): return result @staticmethod - def normalize_author(author): + def cleanup_author(author): assert isinstance(author, dict) if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) @@ -357,7 +357,7 @@ def _parse_authors(self, raw): # normalize Union[dict, list] fields if not isinstance(raw, list): raw = [raw] - return [self.normalize_author(author) for author in raw] + return [self.cleanup_author(author) for author in raw] @staticmethod def _parse_platforms(raw): @@ -430,7 +430,7 @@ def _parse_authors(self, raw): name, email = self.parse_author_name_and_email(author) if not name: continue - result.append(self.normalize_author(dict(name=name, email=email))) + result.append(self.cleanup_author(dict(name=name, email=email))) return result @staticmethod @@ -471,7 +471,9 @@ def parse(self, contents): ) if "author" in data: data["authors"] = self._parse_authors(data) - del data["author"] + for key in ("author", "maintainer"): + if key in data: + del data[key] if "depends" in data: data["dependencies"] = self._parse_dependencies(data["depends"]) return data @@ -544,7 +546,7 @@ def _parse_authors(self, properties): name, email = self.parse_author_name_and_email(author) if not name: continue - authors.append(self.normalize_author(dict(name=name, email=email))) + authors.append(self.cleanup_author(dict(name=name, email=email))) for author in properties.get("maintainer", "").split(","): name, email = self.parse_author_name_and_email(author) if not name: @@ -555,11 +557,11 @@ def _parse_authors(self, properties): continue found = True item["maintainer"] = True - if not item.get("email") and email: + if not item.get("email") and email and "@" in email: item["email"] = email if not found: authors.append( - self.normalize_author(dict(name=name, email=email, maintainer=True)) + self.cleanup_author(dict(name=name, email=email, maintainer=True)) ) return authors diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index c243632837..188c5d8e7f 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -198,7 +198,8 @@ def test_library_properties_parser(): contents = """ name=TestPackage version=1.2.3 -author=SomeAuthor , Another Author (nickname) +author=SomeAuthor , Maintainer Author (nickname) +maintainer=Maintainer Author (nickname) sentence=This is Arduino library customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third @@ -219,8 +220,8 @@ def test_library_properties_parser(): "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, "authors": [ - {"email": "info@author.com", "name": "SomeAuthor"}, - {"name": "Another Author"}, + {"name": "SomeAuthor", "email": "info@author.com"}, + {"name": "Maintainer Author", "maintainer": True}, ], "keywords": ["uncategorized"], "customField": "Custom Value", From 940682255de05a421ec7ffce385023360f70a14a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 22:16:52 +0300 Subject: [PATCH 112/223] Lock Python's isort package to isort<5 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3db3a8ef64..fbe285e24e 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ passenv = * usedevelop = True deps = py36,py37,py38: black - isort + isort<5 pylint pytest pytest-xdist From f27c71a0d4ea3e07474cf0c81f9f2aa132a871e2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 22:56:14 +0300 Subject: [PATCH 113/223] Increase author name length to 100 chars for manifest --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index e19e6f259a..080a7654a7 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -84,7 +84,7 @@ def _deserialize( # pylint: disable=arguments-differ class AuthorSchema(StrictSchema): - name = fields.Str(required=True, validate=validate.Length(min=1, max=50)) + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) email = fields.Email(validate=validate.Length(min=1, max=50)) maintainer = fields.Bool(default=False) url = fields.Url(validate=validate.Length(min=1, max=255)) From f85cf61d68014b1bcfe543cc2b22babdadeb17fd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Jul 2020 23:23:14 +0300 Subject: [PATCH 114/223] Revert back max length of author name to 50 chars --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 080a7654a7..e19e6f259a 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -84,7 +84,7 @@ def _deserialize( # pylint: disable=arguments-differ class AuthorSchema(StrictSchema): - name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + name = fields.Str(required=True, validate=validate.Length(min=1, max=50)) email = fields.Email(validate=validate.Length(min=1, max=50)) maintainer = fields.Bool(default=False) url = fields.Url(validate=validate.Length(min=1, max=255)) From e570aadd72cc7f17877c01e1b532ae941566da76 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 Jul 2020 17:17:34 +0300 Subject: [PATCH 115/223] Docs: Sync --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7d24530a56..8ea9872406 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7d24530a566df810d49c3a04398c94a31b7be14a +Subproject commit 8ea98724069b8783575836e6ac2ee1d8a8414791 From a688edbdf1a2f9b50719a45d282af0d3ee70fdfd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 Jul 2020 21:53:46 +0300 Subject: [PATCH 116/223] Fix an issue with manifest parser when "new_from_archive" API is used --- platformio/package/manifest/parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 8ba94344b4..6507849d42 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -122,7 +122,9 @@ def new_from_archive(path): with tarfile.open(path, mode="r:gz") as tf: for t in sorted(ManifestFileType.items().values()): try: - return ManifestParserFactory.new(tf.extractfile(t).read(), t) + return ManifestParserFactory.new( + tf.extractfile(t).read().decode(), t + ) except KeyError: pass raise UnknownManifestError("Unknown manifest file type in %s archive" % path) From 368c66727bf01b3a20fd30f9304346adeb115e34 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 12 Jul 2020 22:39:32 +0300 Subject: [PATCH 117/223] Fix issue with package packing when re-map is used and manifest is missed in "include" (copy it now) --- platformio/package/pack.py | 16 +++++++++------- tests/package/test_pack.py | 5 ++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 0145a61e22..5823a4f496 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import re import shutil @@ -77,12 +78,7 @@ def pack(self, dst=None): elif os.path.isdir(dst): dst = os.path.join(dst, filename) - return self._create_tarball( - src, - dst, - include=manifest.get("export", {}).get("include"), - exclude=manifest.get("export", {}).get("exclude"), - ) + return self._create_tarball(src, dst, manifest) finally: shutil.rmtree(tmp_dir) @@ -114,7 +110,9 @@ def find_source_root(self, src): return src - def _create_tarball(self, src, dst, include=None, exclude=None): + def _create_tarball(self, src, dst, manifest): + include = manifest.get("export", {}).get("include") + exclude = manifest.get("export", {}).get("exclude") # remap root if ( include @@ -122,6 +120,10 @@ def _create_tarball(self, src, dst, include=None, exclude=None): and os.path.isdir(os.path.join(src, include[0])) ): src = os.path.join(src, include[0]) + with open(os.path.join(src, "library.json"), "w") as fp: + manifest_updated = manifest.copy() + del manifest_updated["export"]["include"] + json.dump(manifest_updated, fp, indent=2, ensure_ascii=False) include = None src_filters = self.compute_src_filters(include, exclude) diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py index 95b435700f..a964f5cdc0 100644 --- a/tests/package/test_pack.py +++ b/tests/package/test_pack.py @@ -61,7 +61,10 @@ def test_filters(tmpdir_factory): ) p = PackagePacker(str(pkg_dir)) with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: - assert set(tar.getnames()) == set(["util/helpers.cpp", "main.cpp"]) + assert set(tar.getnames()) == set( + ["util/helpers.cpp", "main.cpp", "library.json"] + ) + os.unlink(str(src_dir.join("library.json"))) # test include "src" and "include" pkg_dir.join("library.json").write( From cca3099d13447af6149b537a6136e3a85eec1c91 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 14 Jul 2020 18:55:29 +0300 Subject: [PATCH 118/223] Ensure that module.json keywords are lowercased --- platformio/package/manifest/parser.py | 2 ++ tests/package/test_manifest.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 6507849d42..b4a93d982e 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -422,6 +422,8 @@ def parse(self, contents): del data["licenses"] if "dependencies" in data: data["dependencies"] = self._parse_dependencies(data["dependencies"]) + if "keywords" in data: + data["keywords"] = self.str_to_list(data["keywords"], sep=",") return data def _parse_authors(self, raw): diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 188c5d8e7f..899f1cdf83 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -172,7 +172,7 @@ def test_module_json_parser(): "name": "YottaLibrary", "description": "This is Yotta library", "homepage": "https://yottabuild.org", - "keywords": ["mbed", "Yotta"], + "keywords": ["mbed", "yotta"], "license": "Apache-2.0", "platforms": ["*"], "frameworks": ["mbed"], From 1368fa4c3b980656752e6273aeab239ed6af2e32 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 14 Jul 2020 21:07:09 +0300 Subject: [PATCH 119/223] Implement new fields (id, ownername, url, requirements) for PackageSpec API --- platformio/commands/package.py | 6 +- platformio/package/spec.py | 134 +++++++++++++++++++++++++++------ tests/package/test_spec.py | 112 ++++++++++++++++++++++++--- 3 files changed, 215 insertions(+), 37 deletions(-) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 5a3092e2c7..b5e4ad069f 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -78,7 +78,7 @@ def package_publish(package, owner, released_at, private, notify): @cli.command("unpublish", short_help="Remove a pushed package from the registry") @click.argument( - "package", required=True, metavar="[<@organization>/][@]" + "package", required=True, metavar="[/][@]" ) @click.option( "--type", @@ -96,8 +96,8 @@ def package_unpublish(package, type, undo): # pylint: disable=redefined-builtin response = RegistryClient().unpublish_package( type=type, name=spec.name, - owner=spec.organization, - version=spec.version, + owner=spec.ownername, + version=spec.requirements, undo=undo, ) click.secho(response.get("message"), fg="green") diff --git a/platformio/package/spec.py b/platformio/package/spec.py index f031c71c3c..f240c84311 100644 --- a/platformio/package/spec.py +++ b/platformio/package/spec.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import tarfile -from platformio.compat import get_object_members +from platformio.compat import get_object_members, string_types from platformio.package.manifest.parser import ManifestFileType @@ -55,29 +56,114 @@ def from_archive(cls, path): class PackageSpec(object): - def __init__(self, raw=None, organization=None, name=None, version=None): - if raw is not None: - organization, name, version = self.parse(raw) - - self.organization = organization + def __init__( # pylint: disable=redefined-builtin,too-many-arguments + self, raw=None, ownername=None, id=None, name=None, requirements=None, url=None + ): + self.ownername = ownername + self.id = id self.name = name - self.version = version + self.requirements = requirements + self.url = url - @staticmethod - def parse(raw): - organization = None - name = None - version = None + self._parse(raw) + + def __repr__(self): + return ( + "PackageSpec ".format( + ownername=self.ownername, + id=self.id, + name=self.name, + requirements=self.requirements, + url=self.url, + ) + ) + + def __eq__(self, other): + return all( + [ + self.ownername == other.ownername, + self.id == other.id, + self.name == other.name, + self.requirements == other.requirements, + self.url == other.url, + ] + ) + + def _parse(self, raw): + if raw is None: + return + if not isinstance(raw, string_types): + raw = str(raw) raw = raw.strip() - if raw.startswith("@") and "/" in raw: - tokens = raw[1:].split("/", 1) - organization = tokens[0].strip() - raw = tokens[1] - if "@" in raw: - name, version = raw.split("@", 1) - name = name.strip() - version = version.strip() - else: - name = raw.strip() - - return organization, name, version + + parsers = ( + self._parse_requirements, + self._parse_fixed_name, + self._parse_id, + self._parse_ownername, + self._parse_url, + ) + for parser in parsers: + if raw is None: + break + raw = parser(raw) + + # if name is not fixed, parse it from URL + if not self.name and self.url: + self.name = self._parse_name_from_url(self.url) + elif raw: + # the leftover is a package name + self.name = raw + + def _parse_requirements(self, raw): + if "@" not in raw: + return raw + tokens = raw.rsplit("@", 1) + if any(s in tokens[1] for s in (":", "/")): + return raw + self.requirements = tokens[1].strip() + return tokens[0].strip() + + def _parse_fixed_name(self, raw): + if "=" not in raw or raw.startswith("id="): + return raw + tokens = raw.split("=", 1) + if "/" in tokens[0]: + return raw + self.name = tokens[0].strip() + return tokens[1].strip() + + def _parse_id(self, raw): + if raw.isdigit(): + self.id = int(raw) + return None + if raw.startswith("id="): + return self._parse_id(raw[3:]) + return raw + + def _parse_ownername(self, raw): + if raw.count("/") != 1 or "@" in raw: + return raw + tokens = raw.split("/", 1) + self.ownername = tokens[0].strip() + self.name = tokens[1].strip() + return None + + def _parse_url(self, raw): + if not any(s in raw for s in ("@", ":", "/")): + return raw + self.url = raw.strip() + return None + + @staticmethod + def _parse_name_from_url(url): + if url.endswith("/"): + url = url[:-1] + for c in ("#", "?"): + if c in url: + url = url[: url.index(c)] + name = os.path.basename(url) + if "." in name: + return name.split(".", 1)[0].strip() + return name diff --git a/tests/package/test_spec.py b/tests/package/test_spec.py index 1886a8367b..dce89d7f92 100644 --- a/tests/package/test_spec.py +++ b/tests/package/test_spec.py @@ -15,13 +15,105 @@ from platformio.package.spec import PackageSpec -def test_parser(): - inputs = [ - ("foo", (None, "foo", None)), - ("@org/foo", ("org", "foo", None)), - ("@org/foo @ 1.2.3", ("org", "foo", "1.2.3")), - ("bar @ 1.2.3", (None, "bar", "1.2.3")), - ("cat@^1.2", (None, "cat", "^1.2")), - ] - for raw, result in inputs: - assert PackageSpec.parse(raw) == result +def test_ownername(): + assert PackageSpec("alice/foo library") == PackageSpec( + ownername="alice", name="foo library" + ) + assert PackageSpec(" bob / bar ") == PackageSpec(ownername="bob", name="bar") + + +def test_id(): + assert PackageSpec(13) == PackageSpec(id=13) + assert PackageSpec("20") == PackageSpec(id=20) + assert PackageSpec("id=199") == PackageSpec(id=199) + + +def test_name(): + assert PackageSpec("foo") == PackageSpec(name="foo") + assert PackageSpec(" bar-24 ") == PackageSpec(name="bar-24") + + +def test_requirements(): + assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") + assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") + assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") + assert PackageSpec("id=20 @ !=1.2.3,<2.0") == PackageSpec( + id=20, requirements="!=1.2.3,<2.0" + ) + + +def test_local_urls(): + assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( + url="file:///tmp/foo.tar.gz", name="foo" + ) + assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec( + url="file:///tmp/bar.zip", name="customName" + ) + assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( + url="file:///tmp/some-lib/", name="some-lib" + ) + assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec( + url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1" + ) + + +def test_external_urls(): + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/develop.zip", + name="develop", + ) + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" + " @ !=2" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/" + "develop.zip?param=value", + name="develop", + requirements="!=2", + ) + assert PackageSpec( + "platformio-core=" + "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/develop.tar.gz", + name="platformio-core", + requirements="4.4.0", + ) + + +def test_vcs_urls(): + assert PackageSpec( + "https://github.com/platformio/platformio-core.git" + ) == PackageSpec( + name="platformio-core", url="https://github.com/platformio/platformio-core.git", + ) + assert PackageSpec( + "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) == PackageSpec( + name="wolfSSL", url="https://os.mbed.com/users/wolfSSL/code/wolfSSL/", + ) + assert PackageSpec( + "git+https://github.com/platformio/platformio-core.git#master" + ) == PackageSpec( + name="platformio-core", + url="git+https://github.com/platformio/platformio-core.git#master", + ) + assert PackageSpec( + "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0" + ) == PackageSpec( + name="core", + url="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", + requirements="4.4.0", + ) + assert PackageSpec("git@github.com:platformio/platformio-core.git") == PackageSpec( + name="platformio-core", url="git@github.com:platformio/platformio-core.git", + ) + assert PackageSpec( + "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5" + ) == PackageSpec( + name="pkg", + url="git+git@github.com:platformio/platformio-core.git", + requirements="^1.2.3,!=5", + ) From a6f143d1ca06f27d6286648b41fb61abeab2789d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jul 2020 14:20:29 +0300 Subject: [PATCH 120/223] Dump data intended for IDE extensions/plugins using a new `platformio project idedata` command --- HISTORY.rst | 3 ++- docs | 2 +- platformio/commands/project.py | 48 ++++++++++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index cea8fb1b07..8885bb03a6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,10 +24,11 @@ PlatformIO Core 4 - Launch command with custom options declared in `"platformio.ini" `__ - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) -* Display system-wide information using `platformio system info `__ command (`issue #3521 `_) +* Display system-wide information using a new `platformio system info `__ command (`issue #3521 `_) * List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment +* Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command * Do not generate ".travis.yml" for a new project, let the user have a choice * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) * Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) diff --git a/docs b/docs index 8ea9872406..0da1c2810b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8ea98724069b8783575836e6ac2ee1d8a8414791 +Subproject commit 0da1c2810b62f69179cf797f0af6762d0eb9e8f9 diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 27e33455ba..c261a9d981 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -14,6 +14,7 @@ # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,line-too-long +import json import os import click @@ -25,7 +26,7 @@ from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError -from platformio.project.helpers import is_platformio_project +from platformio.project.helpers import is_platformio_project, load_project_ide_data @click.group(short_help="Project Manager") @@ -38,9 +39,7 @@ def cli(): "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), ) @click.option("--json-output", is_flag=True) def project_config(project_dir, json_output): @@ -54,7 +53,6 @@ def project_config(project_dir, json_output): "Computed project configuration for %s" % click.style(project_dir, fg="cyan") ) for section, options in config.as_tuple(): - click.echo() click.secho(section, fg="cyan") click.echo("-" * len(section)) click.echo( @@ -66,6 +64,46 @@ def project_config(project_dir, json_output): tablefmt="plain", ) ) + click.echo() + return None + + +@cli.command("idedata", short_help="Dump data intended for IDE extensions/plugins") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), +) +@click.option("-e", "--environment", multiple=True) +@click.option("--json-output", is_flag=True) +def project_idedata(project_dir, environment, json_output): + if not is_platformio_project(project_dir): + raise NotPlatformIOProjectError(project_dir) + with fs.cd(project_dir): + config = ProjectConfig.get_instance() + config.validate(environment) + environment = list(environment or config.envs()) + + if json_output: + return click.echo(json.dumps(load_project_ide_data(project_dir, environment))) + + for envname in environment: + click.echo("Environment: " + click.style(envname, fg="cyan", bold=True)) + click.echo("=" * (13 + len(envname))) + click.echo( + tabulate( + [ + (click.style(name, bold=True), "=", json.dumps(value, indent=2)) + for name, value in load_project_ide_data( + project_dir, envname + ).items() + ], + tablefmt="plain", + ) + ) + click.echo() + return None From ca3305863738f43d7ca93eb06a41707d627462ce Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jul 2020 23:16:46 +0300 Subject: [PATCH 121/223] New commands for the registry package management (pack, publish, unpublish) --- HISTORY.rst | 5 +++++ docs | 2 +- platformio/commands/lib.py | 19 +++---------------- platformio/commands/package.py | 14 ++++++++++++-- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8885bb03a6..43898869a4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,11 @@ PlatformIO Core 4 - Manage organization teams - Manage resource access +* Registry Package Management + + - Publish a personal or organization package using `platformio package publish `__ command + - Remove a pushed package from the registry using `platformio package unpublish `__ command + * New `Custom Targets `__ - Pre/Post processing based on a dependent sources (other target, source file, etc.) diff --git a/docs b/docs index 0da1c2810b..0ed8c2da4d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0da1c2810b62f69179cf797f0af6762d0eb9e8f9 +Subproject commit 0ed8c2da4db1952f027fb980045d37dc194614c4 diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index f635bff18e..a15937321c 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -25,8 +25,6 @@ from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib -from platformio.package.manifest.parser import ManifestParserFactory -from platformio.package.manifest.schema import ManifestSchema from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.exception import InvalidProjectConfError @@ -495,24 +493,13 @@ def lib_show(library, json_output): return True -@cli.command("register", short_help="Register a new library") +@cli.command("register", short_help="Deprecated") @click.argument("config_url") def lib_register(config_url): - if not config_url.startswith("http://") and not config_url.startswith("https://"): - raise exception.InvalidLibConfURL(config_url) - - # Validate manifest - ManifestSchema().load_manifest( - ManifestParserFactory.new_from_url(config_url).as_dict() + raise exception.UserSideException( + "This command is deprecated. Please use `pio package publish` command." ) - result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) - if "message" in result and result["message"]: - click.secho( - result["message"], - fg="green" if "successed" in result and result["successed"] else "red", - ) - @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index b5e4ad069f..73eb551ad5 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -38,7 +38,12 @@ def cli(): @cli.command("pack", short_help="Create a tarball from a package") -@click.argument("package", required=True, metavar="") +@click.argument( + "package", + required=True, + default=os.getcwd, + metavar="", +) @click.option( "-o", "--output", help="A destination path (folder or a full path to file)" ) @@ -49,7 +54,12 @@ def package_pack(package, output): @cli.command("publish", short_help="Publish a package to the registry") -@click.argument("package", required=True, metavar="") +@click.argument( + "package", + required=True, + default=os.getcwd, + metavar="", +) @click.option( "--owner", help="PIO Account username (can be organization username). " From a2efd7f7c523e92963bbec6154c6c58ec2f1bc68 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Jul 2020 23:18:07 +0300 Subject: [PATCH 122/223] Bump version to 4.4.0a5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 7ad284a11d..9fc3a77337 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a4") +VERSION = (4, 4, "0a5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 1ed462a29a8a303d78f423d8c161a1d5894bee6d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 Jul 2020 01:00:38 +0300 Subject: [PATCH 123/223] PyLint fix --- platformio/commands/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index a15937321c..5bd38aee9d 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -495,7 +495,7 @@ def lib_show(library, json_output): @cli.command("register", short_help="Deprecated") @click.argument("config_url") -def lib_register(config_url): +def lib_register(config_url): # pylint: disable=unused-argument raise exception.UserSideException( "This command is deprecated. Please use `pio package publish` command." ) From ea30d94324504cb0e8a3a521fddc14624cf90a3a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 21 Jul 2020 12:41:38 +0300 Subject: [PATCH 124/223] =?UTF-8?q?Automatically=20enable=20LDF=20dependen?= =?UTF-8?q?cy=20chain+=20mode=20(evaluates=20C/C++=20Preprocessor=20condit?= =?UTF-8?q?ional=20syntax)=20for=20Arduino=20library=20when=20=E2=80=9Clib?= =?UTF-8?q?rary.properties=E2=80=9D=20has=20=E2=80=9Cdepends=E2=80=9D=20fi?= =?UTF-8?q?eld=20//=20Resolve=20#3607?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/piolib.py | 20 ++++++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 43898869a4..8a2d423f02 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -35,6 +35,7 @@ PlatformIO Core 4 * Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment * Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command * Do not generate ".travis.yml" for a new project, let the user have a choice +* Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) * Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) * Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) diff --git a/docs b/docs index 0ed8c2da4d..922f89a0d4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0ed8c2da4db1952f027fb980045d37dc194614c4 +Subproject commit 922f89a0d4c8e18f5f67d745826953ce2193d6b8 diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 3aa6b36d5c..6f8323827c 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -507,6 +507,26 @@ def src_filter(self): src_filter.append("+" % (sep, ext)) return src_filter + @property + def dependencies(self): + # do not include automatically all libraries for build + # chain+ will decide later + return None + + @property + def lib_ldf_mode(self): + if not self._manifest.get("dependencies"): + return LibBuilderBase.lib_ldf_mode.fget(self) + missing = object() + global_value = self.env.GetProjectConfig().getraw( + "env:" + self.env["PIOENV"], "lib_ldf_mode", missing + ) + if global_value != missing: + return LibBuilderBase.lib_ldf_mode.fget(self) + # automatically enable C++ Preprocessing in runtime + # (Arduino IDE has this behavior) + return "chain+" + def is_frameworks_compatible(self, frameworks): return util.items_in_list(frameworks, ["arduino", "energia"]) From 22f1b94062191a7cfde125667071aafa174f5b5f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 21 Jul 2020 12:42:26 +0300 Subject: [PATCH 125/223] Bump version to 4.4.0a6 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 9fc3a77337..0dd66b1b61 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a5") +VERSION = (4, 4, "0a6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 881c5ea308ad1eafa674d664dea5b217b7264971 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jul 2020 17:37:23 +0300 Subject: [PATCH 126/223] Remove unused code --- platformio/commands/home/helpers.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index 018d5da645..aff9228126 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -14,9 +14,6 @@ # pylint: disable=keyword-arg-before-vararg,arguments-differ,signature-differs -import os -import socket - import requests from twisted.internet import defer # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error @@ -52,18 +49,3 @@ def get_core_fullpath(): return where_is_program( "platformio" + (".exe" if "windows" in util.get_systype() else "") ) - - -@util.memoized(expire="10s") -def is_twitter_blocked(): - ip = "104.244.42.1" - timeout = 2 - try: - if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): - requests.get("http://%s" % ip, allow_redirects=False, timeout=timeout) - else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) - return False - except: # pylint: disable=bare-except - pass - return True From 83110975fa967b90149b539ce7b41ae17727db74 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Thu, 23 Jul 2020 17:42:46 +0300 Subject: [PATCH 127/223] Docs: Sync --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 922f89a0d4..e1c264053f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 922f89a0d4c8e18f5f67d745826953ce2193d6b8 +Subproject commit e1c264053f460fcd6a1a4b7201114a737dd3ab37 From 73740aea89c537decde97a8ce89c60d7179c242b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jul 2020 17:56:41 +0300 Subject: [PATCH 128/223] Sync docs and examples --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 922f89a0d4..da10bb5e00 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 922f89a0d4c8e18f5f67d745826953ce2193d6b8 +Subproject commit da10bb5e00f8e711de2c9f843423db765cf8fce7 diff --git a/examples b/examples index 0507e27c58..c3f6d1f17e 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 0507e27c58dbc184420658762b1037348f5f0e87 +Subproject commit c3f6d1f17e6a901dc82c3bec462fcccda0cecbc9 From c193a4ceb7c795acef8534c36091db0bf24ed1f1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 Jul 2020 19:07:29 +0300 Subject: [PATCH 129/223] Handle proxy environment variables in lower case // Resolve #3606 --- platformio/util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index 6a664c4990..c222310959 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -377,10 +377,12 @@ def _internet_on(): socket.setdefaulttimeout(timeout) for host in PING_REMOTE_HOSTS: try: - if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): + for var in ("HTTP_PROXY", "HTTPS_PROXY"): + if not os.getenv(var, var.lower()): + continue requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) - else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80)) + return True + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80)) return True except: # pylint: disable=bare-except pass From 6ace5668b8e3fe19f7636ef31de508b816920931 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jul 2020 20:57:18 +0300 Subject: [PATCH 130/223] Update the registry publish endpoints --- platformio/clients/registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index c48094ee5a..a199390083 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -43,7 +43,7 @@ def publish_package( with open(archive_path, "rb") as fp: response = self.send_auth_request( "post", - "/v3/package/%s/%s" % (owner, PackageType.from_archive(archive_path)), + "/v3/packages/%s/%s" % (owner, PackageType.from_archive(archive_path)), params={ "private": 1 if private else 0, "notify": 1 if notify else 0, @@ -67,7 +67,7 @@ def unpublish_package( # pylint: disable=redefined-builtin owner = ( account.get_account_info(offline=True).get("profile").get("username") ) - path = "/v3/package/%s/%s/%s" % (owner, type, name) + path = "/v3/packages/%s/%s/%s" % (owner, type, name) if version: path = path + "/version/" + version response = self.send_auth_request( From 85f5a6a84a6680733ab2baeb9eeb8f08be110f49 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 Jul 2020 21:00:58 +0300 Subject: [PATCH 131/223] Bump version to 4.4.0a7 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0dd66b1b61..8da7e6e0c2 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a6") +VERSION = (4, 4, "0a7") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 39cb23813f713fd151a7018e81069e0d7f959a8d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jul 2020 11:51:47 +0300 Subject: [PATCH 132/223] Allow ignoring "platforms" and "frameworks" fields in "library.json" and treat a library as compatible with all --- docs | 2 +- platformio/builder/tools/piolib.py | 15 +++------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/docs b/docs index da10bb5e00..13df46f9cf 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit da10bb5e00f8e711de2c9f843423db765cf8fce7 +Subproject commit 13df46f9cf4e3f822f6f642c9c9b3085a0a93193 diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 6f8323827c..fbd8949cc0 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -531,10 +531,7 @@ def is_frameworks_compatible(self, frameworks): return util.items_in_list(frameworks, ["arduino", "energia"]) def is_platforms_compatible(self, platforms): - items = self._manifest.get("platforms", []) - if not items: - return LibBuilderBase.is_platforms_compatible(self, platforms) - return util.items_in_list(platforms, items) + return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) class MbedLibBuilder(LibBuilderBase): @@ -768,16 +765,10 @@ def lib_compat_mode(self): ) def is_platforms_compatible(self, platforms): - items = self._manifest.get("platforms") - if not items: - return LibBuilderBase.is_platforms_compatible(self, platforms) - return util.items_in_list(platforms, items) + return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) def is_frameworks_compatible(self, frameworks): - items = self._manifest.get("frameworks") - if not items: - return LibBuilderBase.is_frameworks_compatible(self, frameworks) - return util.items_in_list(frameworks, items) + return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"]) def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) From def149a29e6ea00e6898a9d68f70b358003fb2d9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 Jul 2020 17:13:05 +0300 Subject: [PATCH 133/223] Use updated registry API --- platformio/clients/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index a199390083..3d45584b06 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -69,7 +69,7 @@ def unpublish_package( # pylint: disable=redefined-builtin ) path = "/v3/packages/%s/%s/%s" % (owner, type, name) if version: - path = path + "/version/" + version + path += "/" + version response = self.send_auth_request( "delete", path, params={"undo": 1 if undo else 0}, ) From adc2d5fe7c29ac45db3421d81d108da0b9b1cd9f Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 28 Jul 2020 15:10:52 +0300 Subject: [PATCH 134/223] Update VSCode template Starting with cpptools v0.29 escaped paths in compilerArgs field don't work on Windows. --- .../tpls/vscode/.vscode/c_cpp_properties.json.tpl | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index 930854d34e..e6dda8957d 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -10,10 +10,6 @@ % return to_unix_path(text).replace('"', '\\"') % end % -% def _escape_required(flag): -% return " " in flag and systype == "windows" -% end -% % def split_args(args_string): % return click.parser.split_arg_string(to_unix_path(args_string)) % end @@ -53,10 +49,7 @@ % def _find_forced_includes(flags, inc_paths): % result = [] % include_args = ("-include", "-imacros") -% for f in flags: -% if not f.startswith(include_args): -% continue -% end +% for f in filter_args(flags, include_args): % for arg in include_args: % inc = "" % if f.startswith(arg) and f.split(arg)[1].strip(): @@ -66,6 +59,7 @@ % end % if inc: % result.append(_find_abs_path(inc, inc_paths)) +% break % end % end % end @@ -134,8 +128,7 @@ "compilerPath": "{{ cc_path }}", "compilerArgs": [ % for flag in [ -% '"%s"' % _escape(f) if _escape_required(f) else f -% for f in filter_args(cc_m_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) +% f for f in filter_args(cc_m_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) % ]: "{{ flag }}", % end From 933a09f981225f7a8e13d664fe0cf4e3098d7adb Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 28 Jul 2020 15:22:36 +0300 Subject: [PATCH 135/223] Update unit testing support for mbed framework - Take into account Mbed OS6 API changes - RawSerial is used with Mbed OS 5 since Serial doesn't support putc with baremetal profile --- examples | 2 +- platformio/commands/test/processor.py | 40 +++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/examples b/examples index c3f6d1f17e..f0f4e0971b 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit c3f6d1f17e6a901dc82c3bec462fcccda0cecbc9 +Subproject commit f0f4e0971b0f9f7b4d77a47cb436f910bf3b4add diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index cfc0f3ca61..9f1b12c887 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -25,33 +25,33 @@ "arduino": { "include": "#include ", "object": "", - "putchar": "Serial.write(c)", - "flush": "Serial.flush()", - "begin": "Serial.begin($baudrate)", - "end": "Serial.end()", + "putchar": "Serial.write(c);", + "flush": "Serial.flush();", + "begin": "Serial.begin($baudrate);", + "end": "Serial.end();", "language": "cpp", }, "mbed": { "include": "#include ", - "object": "Serial pc(USBTX, USBRX);", - "putchar": "pc.putc(c)", + "object": "#if MBED_MAJOR_VERSION == 6\nUnbufferedSerial pc(USBTX, USBRX);\n#else\nRawSerial pc(USBTX, USBRX);\n#endif", + "putchar": "#if MBED_MAJOR_VERSION == 6\npc.write(&c, 1);\n#else\npc.putc(c);\n#endif", "flush": "", - "begin": "pc.baud($baudrate)", + "begin": "pc.baud($baudrate);", "end": "", "language": "cpp", }, "espidf": { "include": "#include ", "object": "", - "putchar": "putchar(c)", - "flush": "fflush(stdout)", + "putchar": "putchar(c);", + "flush": "fflush(stdout);", "begin": "", "end": "", }, "zephyr": { "include": "#include ", "object": "", - "putchar": 'printk("%c", c)', + "putchar": 'printk("%c", c);', "flush": "", "begin": "", "end": "", @@ -59,18 +59,18 @@ "native": { "include": "#include ", "object": "", - "putchar": "putchar(c)", - "flush": "fflush(stdout)", + "putchar": "putchar(c);", + "flush": "fflush(stdout);", "begin": "", "end": "", }, "custom": { "include": '#include "unittest_transport.h"', "object": "", - "putchar": "unittest_uart_putchar(c)", - "flush": "unittest_uart_flush()", - "begin": "unittest_uart_begin()", - "end": "unittest_uart_end()", + "putchar": "unittest_uart_putchar(c);", + "flush": "unittest_uart_flush();", + "begin": "unittest_uart_begin();", + "end": "unittest_uart_end();", "language": "cpp", }, } @@ -174,22 +174,22 @@ def generate_output_file(self, test_dir): "void output_start(unsigned int baudrate)", "#endif", "{", - " $begin;", + " $begin", "}", "", "void output_char(int c)", "{", - " $putchar;", + " $putchar", "}", "", "void output_flush(void)", "{", - " $flush;", + " $flush", "}", "", "void output_complete(void)", "{", - " $end;", + " $end", "}", ] ) From 2bc47f4e97b12b07be2c3a2bad5df9b7b1ef70df Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 28 Jul 2020 15:55:25 +0300 Subject: [PATCH 136/223] PyLint fix --- platformio/commands/test/processor.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 9f1b12c887..334db858df 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -33,8 +33,14 @@ }, "mbed": { "include": "#include ", - "object": "#if MBED_MAJOR_VERSION == 6\nUnbufferedSerial pc(USBTX, USBRX);\n#else\nRawSerial pc(USBTX, USBRX);\n#endif", - "putchar": "#if MBED_MAJOR_VERSION == 6\npc.write(&c, 1);\n#else\npc.putc(c);\n#endif", + "object": ( + "#if MBED_MAJOR_VERSION == 6\nUnbufferedSerial pc(USBTX, USBRX);\n" + "#else\nRawSerial pc(USBTX, USBRX);\n#endif" + ), + "putchar": ( + "#if MBED_MAJOR_VERSION == 6\npc.write(&c, 1);\n" + "#else\npc.putc(c);\n#endif" + ), "flush": "", "begin": "pc.baud($baudrate);", "end": "", From abc0489ac6cf213ee1235017ecf336344825c825 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 28 Jul 2020 15:59:02 +0300 Subject: [PATCH 137/223] Update changelog --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 8a2d423f02..b068fd59b5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -36,6 +36,8 @@ PlatformIO Core 4 * Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command * Do not generate ".travis.yml" for a new project, let the user have a choice * Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) +* Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 +* Do not escape compiler arguments in VSCode template on Windows * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) * Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) * Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) From d329aef87627d206573bacf86df22daf80544f48 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 31 Jul 2020 15:42:26 +0300 Subject: [PATCH 138/223] Initial version of a new package manager --- platformio/app.py | 10 +- platformio/clients/account.py | 28 +- platformio/clients/{rest.py => http.py} | 26 +- platformio/clients/registry.py | 56 ++- platformio/commands/package.py | 4 +- platformio/exception.py | 33 -- platformio/managers/package.py | 8 +- .../{downloader.py => package/download.py} | 59 ++- platformio/package/exception.py | 16 +- platformio/{ => package}/lockfile.py | 0 platformio/package/manager/__init__.py | 13 + platformio/package/manager/_download.py | 95 +++++ platformio/package/manager/_install.py | 282 +++++++++++++ platformio/package/manager/_registry.py | 190 +++++++++ platformio/package/manager/base.py | 233 +++++++++++ platformio/package/manager/library.py | 64 +++ platformio/package/manager/platform.py | 30 ++ platformio/package/manager/tool.py | 29 ++ platformio/package/meta.py | 382 ++++++++++++++++++ platformio/package/pack.py | 4 +- platformio/package/spec.py | 169 -------- platformio/{unpacker.py => package/unpack.py} | 76 ++-- platformio/{ => package}/vcsclient.py | 0 platformio/util.py | 2 +- tests/package/test_manager.py | 303 ++++++++++++++ tests/package/test_meta.py | 250 ++++++++++++ tests/package/test_spec.py | 119 ------ 27 files changed, 2074 insertions(+), 407 deletions(-) rename platformio/clients/{rest.py => http.py} (73%) rename platformio/{downloader.py => package/download.py} (67%) rename platformio/{ => package}/lockfile.py (100%) create mode 100644 platformio/package/manager/__init__.py create mode 100644 platformio/package/manager/_download.py create mode 100644 platformio/package/manager/_install.py create mode 100644 platformio/package/manager/_registry.py create mode 100644 platformio/package/manager/base.py create mode 100644 platformio/package/manager/library.py create mode 100644 platformio/package/manager/platform.py create mode 100644 platformio/package/manager/tool.py create mode 100644 platformio/package/meta.py delete mode 100644 platformio/package/spec.py rename platformio/{unpacker.py => package/unpack.py} (65%) rename platformio/{ => package}/vcsclient.py (100%) create mode 100644 tests/package/test_manager.py create mode 100644 tests/package/test_meta.py delete mode 100644 tests/package/test_spec.py diff --git a/platformio/app.py b/platformio/app.py index f4ae15b9d9..a1a4ee7351 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,8 +25,9 @@ import requests -from platformio import __version__, exception, fs, lockfile, proc +from platformio import __version__, exception, fs, proc from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data +from platformio.package.lockfile import LockFile from platformio.project.helpers import ( get_default_projects_dir, get_project_cache_dir, @@ -125,7 +126,7 @@ def __exit__(self, type_, value, traceback): def _lock_state_file(self): if not self.lock: return - self._lockfile = lockfile.LockFile(self.path) + self._lockfile = LockFile(self.path) try: self._lockfile.acquire() except IOError: @@ -143,6 +144,9 @@ def __del__(self): def as_dict(self): return self._storage + def keys(self): + return self._storage.keys() + def get(self, key, default=True): return self._storage.get(key, default) @@ -187,7 +191,7 @@ def __exit__(self, type_, value, traceback): def _lock_dbindex(self): if not self.cache_dir: os.makedirs(self.cache_dir) - self._lockfile = lockfile.LockFile(self.cache_dir) + self._lockfile = LockFile(self.cache_dir) try: self._lockfile.acquire() except: # pylint: disable=bare-except diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 31e34f7889..c29ef9f9bc 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -16,7 +16,7 @@ import time from platformio import __accounts_api__, app -from platformio.clients.rest import RESTClient +from platformio.clients.http import HTTPClient from platformio.exception import PlatformioException @@ -35,7 +35,7 @@ class AccountAlreadyAuthorized(AccountError): MESSAGE = "You are already authorized with {0} account." -class AccountClient(RESTClient): # pylint:disable=too-many-public-methods +class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 @@ -67,7 +67,7 @@ def send_auth_request(self, *args, **kwargs): token = self.fetch_authentication_token() headers["Authorization"] = "Bearer %s" % token kwargs["headers"] = headers - return self.send_request(*args, **kwargs) + return self.request_json_data(*args, **kwargs) def login(self, username, password): try: @@ -79,11 +79,11 @@ def login(self, username, password): app.get_state_item("account", {}).get("email", "") ) - result = self.send_request( + data = self.request_json_data( "post", "/v1/login", data={"username": username, "password": password}, ) - app.set_state_item("account", result) - return result + app.set_state_item("account", data) + return data def login_with_code(self, client_id, code, redirect_uri): try: @@ -95,7 +95,7 @@ def login_with_code(self, client_id, code, redirect_uri): app.get_state_item("account", {}).get("email", "") ) - result = self.send_request( + result = self.request_json_data( "post", "/v1/login/code", data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, @@ -107,7 +107,7 @@ def logout(self): refresh_token = self.get_refresh_token() self.delete_local_session() try: - self.send_request( + self.request_json_data( "post", "/v1/logout", data={"refresh_token": refresh_token}, ) except AccountError: @@ -133,7 +133,7 @@ def registration( app.get_state_item("account", {}).get("email", "") ) - return self.send_request( + return self.request_json_data( "post", "/v1/registration", data={ @@ -153,7 +153,9 @@ def auth_token(self, password, regenerate): ).get("auth_token") def forgot_password(self, username): - return self.send_request("post", "/v1/forgot", data={"username": username},) + return self.request_json_data( + "post", "/v1/forgot", data={"username": username}, + ) def get_profile(self): return self.send_auth_request("get", "/v1/profile",) @@ -276,15 +278,15 @@ def fetch_authentication_token(self): return auth.get("access_token") if auth.get("refresh_token"): try: - result = self.send_request( + data = self.request_json_data( "post", "/v1/login", headers={ "Authorization": "Bearer %s" % auth.get("refresh_token") }, ) - app.set_state_item("account", result) - return result.get("auth").get("access_token") + app.set_state_item("account", data) + return data.get("auth").get("access_token") except AccountError: self.delete_local_session() raise AccountNotAuthorized() diff --git a/platformio/clients/rest.py b/platformio/clients/http.py similarity index 73% rename from platformio/clients/rest.py rename to platformio/clients/http.py index 4921e2ccb8..e125776226 100644 --- a/platformio/clients/rest.py +++ b/platformio/clients/http.py @@ -19,11 +19,11 @@ from platformio.exception import PlatformioException -class RESTClientError(PlatformioException): +class HTTPClientError(PlatformioException): pass -class RESTClient(object): +class HTTPClient(object): def __init__(self, base_url): if base_url.endswith("/"): base_url = base_url[:-1] @@ -33,19 +33,29 @@ def __init__(self, base_url): retry = Retry( total=5, backoff_factor=1, - method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], - status_forcelist=[500, 502, 503, 504], + # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], + status_forcelist=[413, 429, 500, 502, 503, 504], ) adapter = requests.adapters.HTTPAdapter(max_retries=retry) self._session.mount(base_url, adapter) + def __del__(self): + if not self._session: + return + self._session.close() + self._session = None + def send_request(self, method, path, **kwargs): - # check internet before and resolve issue with 60 seconds timeout + # check Internet before and resolve issue with 60 seconds timeout + # print(self, method, path, kwargs) util.internet_on(raise_exception=True) try: - response = getattr(self._session, method)(self.base_url + path, **kwargs) + return getattr(self._session, method)(self.base_url + path, **kwargs) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - raise RESTClientError(e) + raise HTTPClientError(e) + + def request_json_data(self, *args, **kwargs): + response = self.send_request(*args, **kwargs) return self.raise_error_from_response(response) @staticmethod @@ -59,4 +69,4 @@ def raise_error_from_response(response, expected_codes=(200, 201, 202)): message = response.json()["message"] except (KeyError, ValueError): message = response.text - raise RESTClientError(message) + raise HTTPClientError(message) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 3d45584b06..b7d724b9a7 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -14,13 +14,18 @@ from platformio import __registry_api__, fs from platformio.clients.account import AccountClient -from platformio.clients.rest import RESTClient -from platformio.package.spec import PackageType +from platformio.clients.http import HTTPClient +from platformio.package.meta import PackageType + +try: + from urllib.parse import quote +except ImportError: + from urllib import quote # pylint: disable=too-many-arguments -class RegistryClient(RESTClient): +class RegistryClient(HTTPClient): def __init__(self): super(RegistryClient, self).__init__(base_url=__registry_api__) @@ -30,7 +35,7 @@ def send_auth_request(self, *args, **kwargs): token = AccountClient().fetch_authentication_token() headers["Authorization"] = "Bearer %s" % token kwargs["headers"] = headers - return self.send_request(*args, **kwargs) + return self.request_json_data(*args, **kwargs) def publish_package( self, archive_path, owner=None, released_at=None, private=False, notify=True @@ -41,7 +46,7 @@ def publish_package( account.get_account_info(offline=True).get("profile").get("username") ) with open(archive_path, "rb") as fp: - response = self.send_auth_request( + return self.send_auth_request( "post", "/v3/packages/%s/%s" % (owner, PackageType.from_archive(archive_path)), params={ @@ -57,7 +62,6 @@ def publish_package( }, data=fp, ) - return response def unpublish_package( # pylint: disable=redefined-builtin self, type, name, owner=None, version=None, undo=False @@ -70,10 +74,9 @@ def unpublish_package( # pylint: disable=redefined-builtin path = "/v3/packages/%s/%s/%s" % (owner, type, name) if version: path += "/" + version - response = self.send_auth_request( + return self.send_auth_request( "delete", path, params={"undo": 1 if undo else 0}, ) - return response def update_resource(self, urn, private): return self.send_auth_request( @@ -96,3 +99,40 @@ def list_resources(self, owner): return self.send_auth_request( "get", "/v3/resources", params={"owner": owner} if owner else None ) + + def list_packages(self, query=None, filters=None, page=None): + assert query or filters + search_query = [] + if filters: + valid_filters = ( + "authors", + "keywords", + "frameworks", + "platforms", + "headers", + "ids", + "names", + "owners", + "types", + ) + assert set(filters.keys()) <= set(valid_filters) + for name, values in filters.items(): + for value in set( + values if isinstance(values, (list, tuple)) else [values] + ): + search_query.append("%s:%s" % (name[:-1], value)) + if query: + search_query.append(query) + params = dict(query=quote(" ".join(search_query))) + if page: + params["page"] = int(page) + return self.request_json_data("get", "/v3/packages", params=params) + + def get_package(self, type_, owner, name, version=None): + return self.request_json_data( + "get", + "/v3/packages/{owner}/{type}/{name}".format( + type=type_, owner=owner, name=quote(name) + ), + params=dict(version=version) if version else None, + ) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 73eb551ad5..6ec78d3808 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -18,8 +18,8 @@ import click from platformio.clients.registry import RegistryClient +from platformio.package.meta import PackageSpec, PackageType from platformio.package.pack import PackagePacker -from platformio.package.spec import PackageSpec, PackageType def validate_datetime(ctx, param, value): # pylint: disable=unused-argument @@ -106,7 +106,7 @@ def package_unpublish(package, type, undo): # pylint: disable=redefined-builtin response = RegistryClient().unpublish_package( type=type, name=spec.name, - owner=spec.ownername, + owner=spec.owner, version=spec.requirements, undo=undo, ) diff --git a/platformio/exception.py b/platformio/exception.py index d291ad7fd0..c39b79577a 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -119,39 +119,6 @@ class PackageInstallError(PlatformIOPackageException): ) -class ExtractArchiveItemError(PlatformIOPackageException): - - MESSAGE = ( - "Could not extract `{0}` to `{1}`. Try to disable antivirus " - "tool or check this solution -> http://bit.ly/faq-package-manager" - ) - - -class UnsupportedArchiveType(PlatformIOPackageException): - - MESSAGE = "Can not unpack file '{0}'" - - -class FDUnrecognizedStatusCode(PlatformIOPackageException): - - MESSAGE = "Got an unrecognized status code '{0}' when downloaded {1}" - - -class FDSizeMismatch(PlatformIOPackageException): - - MESSAGE = ( - "The size ({0:d} bytes) of downloaded file '{1}' " - "is not equal to remote size ({2:d} bytes)" - ) - - -class FDSHASumMismatch(PlatformIOPackageException): - - MESSAGE = ( - "The 'sha1' sum '{0}' of downloaded file '{1}' is not equal to remote '{2}'" - ) - - # # Library # diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 92ba451582..346cce5954 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -26,12 +26,12 @@ from platformio import __version__, app, exception, fs, util from platformio.compat import hashlib_encode_data -from platformio.downloader import FileDownloader -from platformio.lockfile import LockFile +from platformio.package.download import FileDownloader from platformio.package.exception import ManifestException +from platformio.package.lockfile import LockFile from platformio.package.manifest.parser import ManifestParserFactory -from platformio.unpacker import FileUnpacker -from platformio.vcsclient import VCSClientFactory +from platformio.package.unpack import FileUnpacker +from platformio.package.vcsclient import VCSClientFactory # pylint: disable=too-many-arguments, too-many-return-statements diff --git a/platformio/downloader.py b/platformio/package/download.py similarity index 67% rename from platformio/downloader.py rename to platformio/package/download.py index ccbc5b361e..3c723c4baf 100644 --- a/platformio/downloader.py +++ b/platformio/package/download.py @@ -23,11 +23,7 @@ import requests from platformio import app, fs, util -from platformio.exception import ( - FDSHASumMismatch, - FDSizeMismatch, - FDUnrecognizedStatusCode, -) +from platformio.package.exception import PackageException class FileDownloader(object): @@ -41,7 +37,11 @@ def __init__(self, url, dest_dir=None): verify=sys.version_info >= (2, 7, 9), ) if self._request.status_code != 200: - raise FDUnrecognizedStatusCode(self._request.status_code, url) + raise PackageException( + "Got the unrecognized status code '{0}' when downloaded {1}".format( + self._request.status_code, url + ) + ) disposition = self._request.headers.get("content-disposition") if disposition and "filename=" in disposition: @@ -74,21 +74,21 @@ def get_size(self): def start(self, with_progress=True, silent=False): label = "Downloading" itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) - f = open(self._destination, "wb") + fp = open(self._destination, "wb") try: if not with_progress or self.get_size() == -1: if not silent: click.echo("%s..." % label) for chunk in itercontent: if chunk: - f.write(chunk) + fp.write(chunk) else: chunks = int(math.ceil(self.get_size() / float(io.DEFAULT_BUFFER_SIZE))) with click.progressbar(length=chunks, label=label) as pb: for _ in pb: - f.write(next(itercontent)) + fp.write(next(itercontent)) finally: - f.close() + fp.close() self._request.close() if self.get_lmtime(): @@ -96,15 +96,40 @@ def start(self, with_progress=True, silent=False): return True - def verify(self, sha1=None): + def verify(self, checksum=None): _dlsize = getsize(self._destination) if self.get_size() != -1 and _dlsize != self.get_size(): - raise FDSizeMismatch(_dlsize, self._fname, self.get_size()) - if not sha1: - return None - checksum = fs.calculate_file_hashsum("sha1", self._destination) - if sha1.lower() != checksum.lower(): - raise FDSHASumMismatch(checksum, self._fname, sha1) + raise PackageException( + ( + "The size ({0:d} bytes) of downloaded file '{1}' " + "is not equal to remote size ({2:d} bytes)" + ).format(_dlsize, self._fname, self.get_size()) + ) + if not checksum: + return True + + checksum_len = len(checksum) + hash_algo = None + if checksum_len == 32: + hash_algo = "md5" + elif checksum_len == 40: + hash_algo = "sha1" + elif checksum_len == 64: + hash_algo = "sha256" + + if not hash_algo: + raise PackageException( + "Could not determine checksum algorithm by %s" % checksum + ) + + dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination) + if checksum.lower() != dl_checksum.lower(): + raise PackageException( + "The checksum '{0}' of the downloaded file '{1}' " + "does not match to the remote '{2}'".format( + dl_checksum, self._fname, checksum + ) + ) return True def _preserve_filemtime(self, lmdate): diff --git a/platformio/package/exception.py b/platformio/package/exception.py index adadc0889f..f32c89ce8c 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio.exception import PlatformioException +from platformio import util +from platformio.exception import PlatformioException, UserSideException class PackageException(PlatformioException): @@ -44,3 +45,16 @@ def __str__(self): "https://docs.platformio.org/page/librarymanager/config.html" % self.messages ) + + +class MissingPackageManifestError(ManifestException): + + MESSAGE = "Could not find one of '{0}' manifest files in the package" + + +class UnknownPackageError(UserSideException): + + MESSAGE = ( + "Could not find a package with '{0}' requirements for your system '%s'" + % util.get_systype() + ) diff --git a/platformio/lockfile.py b/platformio/package/lockfile.py similarity index 100% rename from platformio/lockfile.py rename to platformio/package/lockfile.py diff --git a/platformio/package/manager/__init__.py b/platformio/package/manager/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/package/manager/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py new file mode 100644 index 0000000000..a052da0901 --- /dev/null +++ b/platformio/package/manager/_download.py @@ -0,0 +1,95 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import tempfile +import time + +import click + +from platformio import app, compat +from platformio.package.download import FileDownloader +from platformio.package.lockfile import LockFile + + +class PackageManagerDownloadMixin(object): + + DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month + + def compute_download_path(self, *args): + request_hash = hashlib.new("sha256") + for arg in args: + request_hash.update(compat.hashlib_encode_data(arg)) + dl_path = os.path.join(self.get_download_dir(), request_hash.hexdigest()) + return dl_path + + def get_download_usagedb_path(self): + return os.path.join(self.get_download_dir(), "usage.db") + + def set_download_utime(self, path, utime=None): + with app.State(self.get_download_usagedb_path(), lock=True) as state: + state[os.path.basename(path)] = int(time.time() if not utime else utime) + + def cleanup_expired_downloads(self): + with app.State(self.get_download_usagedb_path(), lock=True) as state: + # remove outdated + for fname in list(state.keys()): + if state[fname] > (time.time() - self.DOWNLOAD_CACHE_EXPIRE): + continue + del state[fname] + dl_path = os.path.join(self.get_download_dir(), fname) + if os.path.isfile(dl_path): + os.remove(dl_path) + + def download(self, url, checksum=None, silent=False): + dl_path = self.compute_download_path(url, checksum or "") + if os.path.isfile(dl_path): + self.set_download_utime(dl_path) + return dl_path + + with_progress = not silent and not app.is_disabled_progressbar() + tmp_path = tempfile.mkstemp(dir=self.get_download_dir())[1] + try: + with LockFile(dl_path): + try: + fd = FileDownloader(url) + fd.set_destination(tmp_path) + fd.start(with_progress=with_progress, silent=silent) + except IOError as e: + raise_error = not with_progress + if with_progress: + try: + fd = FileDownloader(url) + fd.set_destination(tmp_path) + fd.start(with_progress=False, silent=silent) + except IOError: + raise_error = True + if raise_error: + click.secho( + "Error: Please read http://bit.ly/package-manager-ioerror", + fg="red", + err=True, + ) + raise e + if checksum: + fd.verify(checksum) + os.rename(tmp_path, dl_path) + finally: + if os.path.isfile(tmp_path): + os.remove(tmp_path) + + assert os.path.isfile(dl_path) + self.set_download_utime(dl_path) + return dl_path diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py new file mode 100644 index 0000000000..a5aa1c3894 --- /dev/null +++ b/platformio/package/manager/_install.py @@ -0,0 +1,282 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import shutil +import tempfile + +import click + +from platformio import app, compat, fs, util +from platformio.package.exception import PackageException, UnknownPackageError +from platformio.package.lockfile import LockFile +from platformio.package.meta import PackageSourceItem, PackageSpec +from platformio.package.unpack import FileUnpacker +from platformio.package.vcsclient import VCSClientFactory + + +class PackageManagerInstallMixin(object): + + INSTALL_HISTORY = None # avoid circle dependencies + + @staticmethod + def unpack(src, dst): + with_progress = not app.is_disabled_progressbar() + try: + with FileUnpacker(src) as fu: + return fu.unpack(dst, with_progress=with_progress) + except IOError as e: + if not with_progress: + raise e + with FileUnpacker(src) as fu: + return fu.unpack(dst, with_progress=False) + + def install(self, spec, silent=False): + with LockFile(self.package_dir): + pkg = self._install(spec, silent=silent) + self.memcache_reset() + self.cleanup_expired_downloads() + return pkg + + def _install(self, spec, search_filters=None, silent=False): + spec = self.ensure_spec(spec) + + # avoid circle dependencies + if not self.INSTALL_HISTORY: + self.INSTALL_HISTORY = [] + if spec in self.INSTALL_HISTORY: + return None + self.INSTALL_HISTORY.append(spec) + + # check if package is already installed + pkg = self.get_package(spec) + if pkg: + if not silent: + click.secho( + "{name} @ {version} is already installed".format( + **pkg.metadata.as_dict() + ), + fg="yellow", + ) + return pkg + + if not silent: + msg = "Installing %s" % click.style(spec.humanize(), fg="cyan") + self.print_message(msg) + + if spec.url: + pkg = self.install_from_url(spec.url, spec, silent=silent) + else: + pkg = self.install_from_registry(spec, search_filters, silent=silent) + + if not pkg or not pkg.metadata: + raise PackageException( + "Could not install package '%s' for '%s' system" + % (spec.humanize(), util.get_systype()) + ) + + if not silent: + self.print_message( + click.style( + "{name} @ {version} has been successfully installed!".format( + **pkg.metadata.as_dict() + ), + fg="green", + ) + ) + + self.memcache_reset() + self.install_dependencies(pkg, silent) + return pkg + + def install_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageSourceItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message(click.style("Installing dependencies...", fg="yellow")) + for dependency in manifest.get("dependencies"): + if not self.install_dependency(dependency, silent) and not silent: + click.secho( + "Warning! Could not install dependency %s for package '%s'" + % (dependency, pkg.metadata.name), + fg="yellow", + ) + + def install_dependency(self, dependency, silent=False): + spec = PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + search_filters = { + key: value + for key, value in dependency.items() + if key in ("authors", "platforms", "frameworks") + } + return self._install(spec, search_filters=search_filters or None, silent=silent) + + def install_from_url(self, url, spec, checksum=None, silent=False): + spec = self.ensure_spec(spec) + tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir()) + vcs = None + try: + if url.startswith("file://"): + _url = url[7:] + if os.path.isfile(_url): + self.unpack(_url, tmp_dir) + else: + fs.rmtree(tmp_dir) + shutil.copytree(_url, tmp_dir, symlinks=True) + elif url.startswith(("http://", "https://")): + dl_path = self.download(url, checksum, silent=silent) + assert os.path.isfile(dl_path) + self.unpack(dl_path, tmp_dir) + else: + vcs = VCSClientFactory.newClient(tmp_dir, url) + assert vcs.export() + + root_dir = self.find_pkg_root(tmp_dir, spec) + pkg_item = PackageSourceItem( + root_dir, + self.build_metadata( + root_dir, spec, vcs.get_current_revision() if vcs else None + ), + ) + pkg_item.dump_meta() + return self._install_tmp_pkg(pkg_item) + finally: + if os.path.isdir(tmp_dir): + fs.rmtree(tmp_dir) + + def _install_tmp_pkg(self, tmp_pkg): + assert isinstance(tmp_pkg, PackageSourceItem) + # validate package version and declared requirements + if ( + tmp_pkg.metadata.spec.requirements + and tmp_pkg.metadata.version not in tmp_pkg.metadata.spec.requirements + ): + raise PackageException( + "Package version %s doesn't satisfy requirements %s based on %s" + % ( + tmp_pkg.metadata.version, + tmp_pkg.metadata.spec.requirements, + tmp_pkg.metadata, + ) + ) + dst_pkg = PackageSourceItem( + os.path.join(self.package_dir, tmp_pkg.get_safe_dirname()) + ) + + # what to do with existing package? + action = "overwrite" + if dst_pkg.metadata and dst_pkg.metadata.spec.url: + if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: + action = "detach-existing" + elif tmp_pkg.metadata.spec.url: + action = "detach-new" + elif dst_pkg.metadata and dst_pkg.metadata.version != tmp_pkg.metadata.version: + action = ( + "detach-existing" + if tmp_pkg.metadata.version > dst_pkg.metadata.version + else "detach-new" + ) + + def _cleanup_dir(path): + if os.path.isdir(path): + fs.rmtree(path) + + if action == "detach-existing": + target_dirname = "%s@%s" % ( + tmp_pkg.get_safe_dirname(), + dst_pkg.metadata.version, + ) + if dst_pkg.metadata.spec.url: + target_dirname = "%s@src-%s" % ( + tmp_pkg.get_safe_dirname(), + hashlib.md5( + compat.hashlib_encode_data(dst_pkg.metadata.spec.url) + ).hexdigest(), + ) + # move existing into the new place + pkg_dir = os.path.join(self.package_dir, target_dirname) + _cleanup_dir(pkg_dir) + shutil.move(dst_pkg.path, pkg_dir) + # move new source to the destination location + _cleanup_dir(dst_pkg.path) + shutil.move(tmp_pkg.path, dst_pkg.path) + return PackageSourceItem(dst_pkg.path) + + if action == "detach-new": + target_dirname = "%s@%s" % ( + tmp_pkg.get_safe_dirname(), + tmp_pkg.metadata.version, + ) + if tmp_pkg.metadata.spec.url: + target_dirname = "%s@src-%s" % ( + tmp_pkg.get_safe_dirname(), + hashlib.md5( + compat.hashlib_encode_data(tmp_pkg.metadata.spec.url) + ).hexdigest(), + ) + pkg_dir = os.path.join(self.package_dir, target_dirname) + _cleanup_dir(pkg_dir) + shutil.move(tmp_pkg.path, pkg_dir) + return PackageSourceItem(pkg_dir) + + # otherwise, overwrite existing + _cleanup_dir(dst_pkg.path) + shutil.move(tmp_pkg.path, dst_pkg.path) + return PackageSourceItem(dst_pkg.path) + + def uninstall(self, path_or_spec, silent=False): + with LockFile(self.package_dir): + pkg = ( + PackageSourceItem(path_or_spec) + if os.path.isdir(path_or_spec) + else self.get_package(path_or_spec) + ) + if not pkg or not pkg.metadata: + raise UnknownPackageError(path_or_spec) + + if not silent: + self.print_message( + "Uninstalling %s @ %s: \t" + % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), + nl=False, + ) + if os.path.islink(pkg.path): + os.unlink(pkg.path) + else: + fs.rmtree(pkg.path) + self.memcache_reset() + + # unfix detached-package with the same name + detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) + if ( + detached_pkg + and "@" in detached_pkg.path + and not os.path.isdir( + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) + ) + ): + shutil.move( + detached_pkg.path, + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), + ) + self.memcache_reset() + + if not silent: + click.echo("[%s]" % click.style("OK", fg="green")) + return True diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py new file mode 100644 index 0000000000..3f3ae813f9 --- /dev/null +++ b/platformio/package/manager/_registry.py @@ -0,0 +1,190 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import click + +from platformio.clients.http import HTTPClient +from platformio.clients.registry import RegistryClient +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageMetaData, PackageSpec + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +class RegistryFileMirrorsIterator(object): + + HTTP_CLIENT_INSTANCES = {} + + def __init__(self, download_url): + self.download_url = download_url + self._url_parts = urlparse(download_url) + self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) + self._visited_mirrors = [] + + def __iter__(self): + return self + + def __next__(self): + http = self.get_http_client() + response = http.send_request( + "head", + self._url_parts.path, + allow_redirects=False, + params=dict(bypass=",".join(self._visited_mirrors)) + if self._visited_mirrors + else None, + ) + stop_conditions = [ + response.status_code not in (302, 307), + not response.headers.get("Location"), + not response.headers.get("X-PIO-Mirror"), + response.headers.get("X-PIO-Mirror") in self._visited_mirrors, + ] + if any(stop_conditions): + raise StopIteration + self._visited_mirrors.append(response.headers.get("X-PIO-Mirror")) + return ( + response.headers.get("Location"), + response.headers.get("X-PIO-Content-SHA256"), + ) + + def get_http_client(self): + if self._base_url not in RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES: + RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[ + self._base_url + ] = HTTPClient(self._base_url) + return RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[self._base_url] + + +class PackageManageRegistryMixin(object): + def install_from_registry(self, spec, search_filters=None, silent=False): + packages = self.search_registry_packages(spec, search_filters) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1 and not silent: + self.print_multi_package_issue(packages, spec) + package, version = self.find_best_registry_version(packages, spec) + pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None + if not pkgfile: + raise UnknownPackageError(spec.humanize()) + + for url, checksum in RegistryFileMirrorsIterator(pkgfile["download_url"]): + try: + return self.install_from_url( + url, + PackageSpec( + owner=package["owner"]["username"], + id=package["id"], + name=package["name"], + ), + checksum or pkgfile["checksum"]["sha256"], + silent=silent, + ) + except Exception as e: # pylint: disable=broad-except + click.secho("Warning! Package Mirror: %s" % e, fg="yellow") + click.secho("Looking for another mirror...", fg="yellow") + + return None + + def get_registry_client_instance(self): + if not self._registry_client: + self._registry_client = RegistryClient() + return self._registry_client + + def search_registry_packages(self, spec, filters=None): + filters = filters or {} + if spec.id: + filters["ids"] = str(spec.id) + else: + filters["types"] = self.pkg_type + filters["names"] = '"%s"' % spec.name.lower() + if spec.owner: + filters["owners"] = spec.owner.lower() + return self.get_registry_client_instance().list_packages(filters=filters)[ + "items" + ] + + def fetch_registry_package_versions(self, owner, name): + return self.get_registry_client_instance().get_package( + self.pkg_type, owner, name + )["versions"] + + @staticmethod + def print_multi_package_issue(packages, spec): + click.secho( + "Warning! More than one package has been found by ", fg="yellow", nl=False + ) + click.secho(spec.humanize(), fg="cyan", nl=False) + click.secho(" requirements:", fg="yellow") + for item in packages: + click.echo( + " - {owner}/{name} @ {version}".format( + owner=click.style(item["owner"]["username"], fg="cyan"), + name=item["name"], + version=item["version"]["name"], + ) + ) + click.secho( + "Please specify detailed REQUIREMENTS using package owner and version " + "(showed above) to avoid project compatibility issues.", + fg="yellow", + ) + + def find_best_registry_version(self, packages, spec): + # find compatible version within the latest package versions + for package in packages: + version = self._pick_best_pkg_version([package["version"]], spec) + if version: + return (package, version) + + if not spec.requirements: + return None + + # if the custom version requirements, check ALL package versions + for package in packages: + version = self._pick_best_pkg_version( + self.fetch_registry_package_versions( + package["owner"]["username"], package["name"] + ), + spec, + ) + if version: + return (package, version) + time.sleep(1) + return None + + def _pick_best_pkg_version(self, versions, spec): + best = None + for version in versions: + semver = PackageMetaData.to_semver(version["name"]) + if spec.requirements and semver not in spec.requirements: + continue + if not any( + self.is_system_compatible(f.get("system")) for f in version["files"] + ): + continue + if not best or (semver > PackageMetaData.to_semver(best["name"])): + best = version + return best + + def _pick_compatible_pkg_file(self, version_files): + for item in version_files: + if self.is_system_compatible(item.get("system")): + return item + return None diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py new file mode 100644 index 0000000000..e48fc25039 --- /dev/null +++ b/platformio/package/manager/base.py @@ -0,0 +1,233 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from datetime import datetime + +import click +import semantic_version + +from platformio import fs, util +from platformio.commands import PlatformioCLI +from platformio.package.exception import ManifestException, MissingPackageManifestError +from platformio.package.manager._download import PackageManagerDownloadMixin +from platformio.package.manager._install import PackageManagerInstallMixin +from platformio.package.manager._registry import PackageManageRegistryMixin +from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.meta import ( + PackageMetaData, + PackageSourceItem, + PackageSpec, + PackageType, +) +from platformio.project.helpers import get_project_cache_dir + + +class BasePackageManager( + PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin +): + MEMORY_CACHE = {} + + def __init__(self, pkg_type, package_dir): + self.pkg_type = pkg_type + self.package_dir = self.ensure_dir_exists(package_dir) + self.MEMORY_CACHE = {} + self._download_dir = None + self._tmp_dir = None + self._registry_client = None + + def memcache_get(self, key, default=None): + return self.MEMORY_CACHE.get(key, default) + + def memcache_set(self, key, value): + self.MEMORY_CACHE[key] = value + + def memcache_reset(self): + self.MEMORY_CACHE.clear() + + @staticmethod + def is_system_compatible(value): + if not value or "*" in value: + return True + return util.items_in_list(value, util.get_systype()) + + @staticmethod + def generate_rand_version(): + return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S") + + @staticmethod + def ensure_dir_exists(path): + if not os.path.isdir(path): + os.makedirs(path) + assert os.path.isdir(path) + return path + + @staticmethod + def ensure_spec(spec): + return spec if isinstance(spec, PackageSpec) else PackageSpec(spec) + + @property + def manifest_names(self): + raise NotImplementedError + + def print_message(self, message, nl=True): + click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl) + + def get_download_dir(self): + if not self._download_dir: + self._download_dir = self.ensure_dir_exists( + os.path.join(get_project_cache_dir(), "downloads") + ) + return self._download_dir + + def get_tmp_dir(self): + if not self._tmp_dir: + self._tmp_dir = self.ensure_dir_exists( + os.path.join(get_project_cache_dir(), "tmp") + ) + return self._tmp_dir + + def find_pkg_root(self, path, spec): # pylint: disable=unused-argument + if self.manifest_exists(path): + return path + for root, _, _ in os.walk(path): + if self.manifest_exists(root): + return root + raise MissingPackageManifestError(", ".join(self.manifest_names)) + + def get_manifest_path(self, pkg_dir): + if not os.path.isdir(pkg_dir): + return None + for name in self.manifest_names: + manifest_path = os.path.join(pkg_dir, name) + if os.path.isfile(manifest_path): + return manifest_path + return None + + def manifest_exists(self, pkg_dir): + return self.get_manifest_path(pkg_dir) + + def load_manifest(self, src): + path = src.path if isinstance(src, PackageSourceItem) else src + cache_key = "load_manifest-%s" % path + result = self.memcache_get(cache_key) + if result: + return result + candidates = ( + [os.path.join(path, name) for name in self.manifest_names] + if os.path.isdir(path) + else [path] + ) + for item in candidates: + if not os.path.isfile(item): + continue + try: + result = ManifestParserFactory.new_from_file(item).as_dict() + self.memcache_set(cache_key, result) + return result + except ManifestException as e: + if not PlatformioCLI.in_silence(): + click.secho(str(e), fg="yellow") + raise MissingPackageManifestError(", ".join(self.manifest_names)) + + def build_legacy_spec(self, pkg_dir): + # find src manifest + src_manifest_name = ".piopkgmanager.json" + src_manifest_path = None + for name in os.listdir(pkg_dir): + if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)): + continue + src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name) + break + + if src_manifest_path: + src_manifest = fs.load_json(src_manifest_path) + return PackageSpec( + name=src_manifest.get("name"), + url=src_manifest.get("url"), + requirements=src_manifest.get("requirements"), + ) + + # fall back to a package manifest + manifest = self.load_manifest(pkg_dir) + return PackageSpec(name=manifest.get("name")) + + def build_metadata(self, pkg_dir, spec, vcs_revision=None): + manifest = self.load_manifest(pkg_dir) + metadata = PackageMetaData( + type=self.pkg_type, + name=manifest.get("name"), + version=manifest.get("version"), + spec=spec, + ) + if not metadata.name or spec.is_custom_name(): + metadata.name = spec.name + if vcs_revision: + metadata.version = "%s+sha.%s" % ( + metadata.version if metadata.version else "0.0.0", + vcs_revision, + ) + if not metadata.version: + metadata.version = self.generate_rand_version() + return metadata + + def get_installed(self): + result = [] + for name in os.listdir(self.package_dir): + pkg_dir = os.path.join(self.package_dir, name) + if not os.path.isdir(pkg_dir): + continue + pkg = PackageSourceItem(pkg_dir) + if not pkg.metadata: + try: + spec = self.build_legacy_spec(pkg_dir) + pkg.metadata = self.build_metadata(pkg_dir, spec) + except MissingPackageManifestError: + pass + if pkg.metadata: + result.append(pkg) + return result + + def get_package(self, spec): + def _ci_strings_are_equal(a, b): + if a == b: + return True + if not a or not b: + return False + return a.strip().lower() == b.strip().lower() + + spec = self.ensure_spec(spec) + best = None + for pkg in self.get_installed(): + skip_conditions = [ + spec.owner + and not _ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner), + spec.url and spec.url != pkg.metadata.spec.url, + spec.id and spec.id != pkg.metadata.spec.id, + not spec.id + and not spec.url + and not _ci_strings_are_equal(spec.name, pkg.metadata.name), + ] + if any(skip_conditions): + continue + if self.pkg_type == PackageType.TOOL: + # TODO: check "system" for pkg + pass + + assert isinstance(pkg.metadata.version, semantic_version.Version) + if spec.requirements and pkg.metadata.version not in spec.requirements: + continue + if not best or (pkg.metadata.version > best.metadata.version): + best = pkg + return best diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py new file mode 100644 index 0000000000..9fe924b9e7 --- /dev/null +++ b/platformio/package/manager/library.py @@ -0,0 +1,64 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +from platformio.package.exception import MissingPackageManifestError +from platformio.package.manager.base import BasePackageManager +from platformio.package.meta import PackageSpec, PackageType +from platformio.project.helpers import get_project_global_lib_dir + + +class LibraryPackageManager(BasePackageManager): + def __init__(self, package_dir=None): + super(LibraryPackageManager, self).__init__( + PackageType.LIBRARY, package_dir or get_project_global_lib_dir() + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.LIBRARY] + + def find_pkg_root(self, path, spec): + try: + return super(LibraryPackageManager, self).find_pkg_root(path, spec) + except MissingPackageManifestError: + pass + assert isinstance(spec, PackageSpec) + + root_dir = self.find_library_root(path) + + # automatically generate library manifest + with open(os.path.join(root_dir, "library.json"), "w") as fp: + json.dump( + dict(name=spec.name, version=self.generate_rand_version(),), + fp, + indent=2, + ) + + return root_dir + + @staticmethod + def find_library_root(path): + for root, dirs, files in os.walk(path): + if not files and len(dirs) == 1: + continue + for fname in files: + if not fname.endswith((".c", ".cpp", ".h", ".S")): + continue + if os.path.isdir(os.path.join(os.path.dirname(root), "src")): + return os.path.dirname(root) + return root + return path diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py new file mode 100644 index 0000000000..627bad4734 --- /dev/null +++ b/platformio/package/manager/platform.py @@ -0,0 +1,30 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.manager.base import BasePackageManager +from platformio.package.meta import PackageType +from platformio.project.config import ProjectConfig + + +class PlatformPackageManager(BasePackageManager): + def __init__(self, package_dir=None): + self.config = ProjectConfig.get_instance() + super(PlatformPackageManager, self).__init__( + PackageType.PLATFORM, + package_dir or self.config.get_optional_dir("platforms"), + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.PLATFORM] diff --git a/platformio/package/manager/tool.py b/platformio/package/manager/tool.py new file mode 100644 index 0000000000..db660303a0 --- /dev/null +++ b/platformio/package/manager/tool.py @@ -0,0 +1,29 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.manager.base import BasePackageManager +from platformio.package.meta import PackageType +from platformio.project.config import ProjectConfig + + +class ToolPackageManager(BasePackageManager): + def __init__(self, package_dir=None): + self.config = ProjectConfig.get_instance() + super(ToolPackageManager, self).__init__( + PackageType.TOOL, package_dir or self.config.get_optional_dir("packages"), + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.TOOL] diff --git a/platformio/package/meta.py b/platformio/package/meta.py new file mode 100644 index 0000000000..0f1214e13f --- /dev/null +++ b/platformio/package/meta.py @@ -0,0 +1,382 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import tarfile + +import semantic_version + +from platformio.compat import get_object_members, string_types +from platformio.package.manifest.parser import ManifestFileType + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +class PackageType(object): + LIBRARY = "library" + PLATFORM = "platform" + TOOL = "tool" + + @classmethod + def items(cls): + return get_object_members(cls) + + @classmethod + def get_manifest_map(cls): + return { + cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), + cls.LIBRARY: ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ), + cls.TOOL: (ManifestFileType.PACKAGE_JSON,), + } + + @classmethod + def from_archive(cls, path): + assert path.endswith("tar.gz") + manifest_map = cls.get_manifest_map() + with tarfile.open(path, mode="r:gz") as tf: + for t in sorted(cls.items().values()): + for manifest in manifest_map[t]: + try: + if tf.getmember(manifest): + return t + except KeyError: + pass + return None + + +class PackageSpec(object): + def __init__( # pylint: disable=redefined-builtin,too-many-arguments + self, raw=None, owner=None, id=None, name=None, requirements=None, url=None + ): + self.owner = owner + self.id = id + self.name = name + self._requirements = None + self.url = url + if requirements: + self.requirements = requirements + self._name_is_custom = False + self._parse(raw) + + def __eq__(self, other): + return all( + [ + self.owner == other.owner, + self.id == other.id, + self.name == other.name, + self.requirements == other.requirements, + self.url == other.url, + ] + ) + + def __repr__(self): + return ( + "PackageSpec ".format(**self.as_dict()) + ) + + @property + def requirements(self): + return self._requirements + + @requirements.setter + def requirements(self, value): + if not value: + self._requirements = None + return + self._requirements = ( + value + if isinstance(value, semantic_version.SimpleSpec) + else semantic_version.SimpleSpec(value) + ) + + def humanize(self): + if self.url: + result = self.url + elif self.id: + result = "id:%d" % self.id + else: + result = "" + if self.owner: + result = self.owner + "/" + result += self.name + if self.requirements: + result += " @ " + str(self.requirements) + return result + + def is_custom_name(self): + return self._name_is_custom + + def as_dict(self): + return dict( + owner=self.owner, + id=self.id, + name=self.name, + requirements=str(self.requirements) if self.requirements else None, + url=self.url, + ) + + def _parse(self, raw): + if raw is None: + return + if not isinstance(raw, string_types): + raw = str(raw) + raw = raw.strip() + + parsers = ( + self._parse_requirements, + self._parse_custom_name, + self._parse_id, + self._parse_owner, + self._parse_url, + ) + for parser in parsers: + if raw is None: + break + raw = parser(raw) + + # if name is not custom, parse it from URL + if not self.name and self.url: + self.name = self._parse_name_from_url(self.url) + elif raw: + # the leftover is a package name + self.name = raw + + def _parse_requirements(self, raw): + if "@" not in raw: + return raw + tokens = raw.rsplit("@", 1) + if any(s in tokens[1] for s in (":", "/")): + return raw + self.requirements = tokens[1].strip() + return tokens[0].strip() + + def _parse_custom_name(self, raw): + if "=" not in raw or raw.startswith("id="): + return raw + tokens = raw.split("=", 1) + if "/" in tokens[0]: + return raw + self.name = tokens[0].strip() + self._name_is_custom = True + return tokens[1].strip() + + def _parse_id(self, raw): + if raw.isdigit(): + self.id = int(raw) + return None + if raw.startswith("id="): + return self._parse_id(raw[3:]) + return raw + + def _parse_owner(self, raw): + if raw.count("/") != 1 or "@" in raw: + return raw + tokens = raw.split("/", 1) + self.owner = tokens[0].strip() + self.name = tokens[1].strip() + return None + + def _parse_url(self, raw): + if not any(s in raw for s in ("@", ":", "/")): + return raw + self.url = raw.strip() + parts = urlparse(self.url) + + # if local file or valid URL with scheme vcs+protocol:// + if parts.scheme == "file" or "+" in parts.scheme or self.url.startswith("git+"): + return None + + # parse VCS + git_conditions = [ + parts.path.endswith(".git"), + # Handle GitHub URL (https://github.com/user/package) + parts.netloc in ("github.com", "gitlab.com", "bitbucket.com") + and not parts.path.endswith((".zip", ".tar.gz")), + ] + hg_conditions = [ + # Handle Developer Mbed URL + # (https://developer.mbed.org/users/user/code/package/) + # (https://os.mbed.com/users/user/code/package/) + parts.netloc + in ("mbed.com", "os.mbed.com", "developer.mbed.org") + ] + if any(git_conditions): + self.url = "git+" + self.url + elif any(hg_conditions): + self.url = "hg+" + self.url + + return None + + @staticmethod + def _parse_name_from_url(url): + if url.endswith("/"): + url = url[:-1] + for c in ("#", "?"): + if c in url: + url = url[: url.index(c)] + + # parse real repository name from Github + parts = urlparse(url) + if parts.netloc == "github.com" and parts.path.count("/") > 2: + return parts.path.split("/")[2] + + name = os.path.basename(url) + if "." in name: + return name.split(".", 1)[0].strip() + return name + + +class PackageMetaData(object): + def __init__( # pylint: disable=redefined-builtin + self, type, name, version, spec=None + ): + assert type in PackageType.items().values() + if spec: + assert isinstance(spec, PackageSpec) + self.type = type + self.name = name + self._version = None + self.version = version + self.spec = spec + + def __repr__(self): + return ( + "PackageMetaData -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import tarfile - -from platformio.compat import get_object_members, string_types -from platformio.package.manifest.parser import ManifestFileType - - -class PackageType(object): - LIBRARY = "library" - PLATFORM = "platform" - TOOL = "tool" - - @classmethod - def items(cls): - return get_object_members(cls) - - @classmethod - def get_manifest_map(cls): - return { - cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), - cls.LIBRARY: ( - ManifestFileType.LIBRARY_JSON, - ManifestFileType.LIBRARY_PROPERTIES, - ManifestFileType.MODULE_JSON, - ), - cls.TOOL: (ManifestFileType.PACKAGE_JSON,), - } - - @classmethod - def from_archive(cls, path): - assert path.endswith("tar.gz") - manifest_map = cls.get_manifest_map() - with tarfile.open(path, mode="r:gz") as tf: - for t in sorted(cls.items().values()): - for manifest in manifest_map[t]: - try: - if tf.getmember(manifest): - return t - except KeyError: - pass - return None - - -class PackageSpec(object): - def __init__( # pylint: disable=redefined-builtin,too-many-arguments - self, raw=None, ownername=None, id=None, name=None, requirements=None, url=None - ): - self.ownername = ownername - self.id = id - self.name = name - self.requirements = requirements - self.url = url - - self._parse(raw) - - def __repr__(self): - return ( - "PackageSpec ".format( - ownername=self.ownername, - id=self.id, - name=self.name, - requirements=self.requirements, - url=self.url, - ) - ) - - def __eq__(self, other): - return all( - [ - self.ownername == other.ownername, - self.id == other.id, - self.name == other.name, - self.requirements == other.requirements, - self.url == other.url, - ] - ) - - def _parse(self, raw): - if raw is None: - return - if not isinstance(raw, string_types): - raw = str(raw) - raw = raw.strip() - - parsers = ( - self._parse_requirements, - self._parse_fixed_name, - self._parse_id, - self._parse_ownername, - self._parse_url, - ) - for parser in parsers: - if raw is None: - break - raw = parser(raw) - - # if name is not fixed, parse it from URL - if not self.name and self.url: - self.name = self._parse_name_from_url(self.url) - elif raw: - # the leftover is a package name - self.name = raw - - def _parse_requirements(self, raw): - if "@" not in raw: - return raw - tokens = raw.rsplit("@", 1) - if any(s in tokens[1] for s in (":", "/")): - return raw - self.requirements = tokens[1].strip() - return tokens[0].strip() - - def _parse_fixed_name(self, raw): - if "=" not in raw or raw.startswith("id="): - return raw - tokens = raw.split("=", 1) - if "/" in tokens[0]: - return raw - self.name = tokens[0].strip() - return tokens[1].strip() - - def _parse_id(self, raw): - if raw.isdigit(): - self.id = int(raw) - return None - if raw.startswith("id="): - return self._parse_id(raw[3:]) - return raw - - def _parse_ownername(self, raw): - if raw.count("/") != 1 or "@" in raw: - return raw - tokens = raw.split("/", 1) - self.ownername = tokens[0].strip() - self.name = tokens[1].strip() - return None - - def _parse_url(self, raw): - if not any(s in raw for s in ("@", ":", "/")): - return raw - self.url = raw.strip() - return None - - @staticmethod - def _parse_name_from_url(url): - if url.endswith("/"): - url = url[:-1] - for c in ("#", "?"): - if c in url: - url = url[: url.index(c)] - name = os.path.basename(url) - if "." in name: - return name.split(".", 1)[0].strip() - return name diff --git a/platformio/unpacker.py b/platformio/package/unpack.py similarity index 65% rename from platformio/unpacker.py rename to platformio/package/unpack.py index 7fce466dc1..a00873cd6a 100644 --- a/platformio/unpacker.py +++ b/platformio/package/unpack.py @@ -19,10 +19,19 @@ import click -from platformio import exception, util +from platformio import util +from platformio.package.exception import PackageException -class ArchiveBase(object): +class ExtractArchiveItemError(PackageException): + + MESSAGE = ( + "Could not extract `{0}` to `{1}`. Try to disable antivirus " + "tool or check this solution -> http://bit.ly/faq-package-manager" + ) + + +class BaseArchiver(object): def __init__(self, arhfileobj): self._afo = arhfileobj @@ -46,9 +55,9 @@ def close(self): self._afo.close() -class TARArchive(ArchiveBase): +class TARArchiver(BaseArchiver): def __init__(self, archpath): - super(TARArchive, self).__init__(tarfile_open(archpath)) + super(TARArchiver, self).__init__(tarfile_open(archpath)) def get_items(self): return self._afo.getmembers() @@ -79,7 +88,7 @@ def extract_item(self, item, dest_dir): self.is_link(item) and self.is_bad_link(item, dest_dir), ] if not any(bad_conds): - super(TARArchive, self).extract_item(item, dest_dir) + super(TARArchiver, self).extract_item(item, dest_dir) else: click.secho( "Blocked insecure item `%s` from TAR archive" % item.name, @@ -88,9 +97,9 @@ def extract_item(self, item, dest_dir): ) -class ZIPArchive(ArchiveBase): +class ZIPArchiver(BaseArchiver): def __init__(self, archpath): - super(ZIPArchive, self).__init__(ZipFile(archpath)) + super(ZIPArchiver, self).__init__(ZipFile(archpath)) @staticmethod def preserve_permissions(item, dest_dir): @@ -121,48 +130,59 @@ def after_extract(self, item, dest_dir): class FileUnpacker(object): - def __init__(self, archpath): - self.archpath = archpath - self._unpacker = None + def __init__(self, path): + self.path = path + self._archiver = None + + def _init_archiver(self): + magic_map = { + b"\x1f\x8b\x08": TARArchiver, + b"\x42\x5a\x68": TARArchiver, + b"\x50\x4b\x03\x04": ZIPArchiver, + } + magic_len = max(len(k) for k in magic_map) + with open(self.path, "rb") as fp: + data = fp.read(magic_len) + for magic, archiver in magic_map.items(): + if data.startswith(magic): + return archiver(self.path) + raise PackageException("Unknown archive type '%s'" % self.path) def __enter__(self): - if self.archpath.lower().endswith((".gz", ".bz2", ".tar")): - self._unpacker = TARArchive(self.archpath) - elif self.archpath.lower().endswith(".zip"): - self._unpacker = ZIPArchive(self.archpath) - if not self._unpacker: - raise exception.UnsupportedArchiveType(self.archpath) + self._archiver = self._init_archiver() return self def __exit__(self, *args): - if self._unpacker: - self._unpacker.close() + if self._archiver: + self._archiver.close() def unpack( - self, dest_dir=".", with_progress=True, check_unpacked=True, silent=False + self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False ): - assert self._unpacker + assert self._archiver + if not dest_dir: + dest_dir = os.getcwd() if not with_progress or silent: if not silent: click.echo("Unpacking...") - for item in self._unpacker.get_items(): - self._unpacker.extract_item(item, dest_dir) + for item in self._archiver.get_items(): + self._archiver.extract_item(item, dest_dir) else: - items = self._unpacker.get_items() + items = self._archiver.get_items() with click.progressbar(items, label="Unpacking") as pb: for item in pb: - self._unpacker.extract_item(item, dest_dir) + self._archiver.extract_item(item, dest_dir) if not check_unpacked: return True # check on disk - for item in self._unpacker.get_items(): - filename = self._unpacker.get_item_filename(item) + for item in self._archiver.get_items(): + filename = self._archiver.get_item_filename(item) item_path = os.path.join(dest_dir, filename) try: - if not self._unpacker.is_link(item) and not os.path.exists(item_path): - raise exception.ExtractArchiveItemError(filename, dest_dir) + if not self._archiver.is_link(item) and not os.path.exists(item_path): + raise ExtractArchiveItemError(filename, dest_dir) except NotImplementedError: pass return True diff --git a/platformio/vcsclient.py b/platformio/package/vcsclient.py similarity index 100% rename from platformio/vcsclient.py rename to platformio/package/vcsclient.py diff --git a/platformio/util.py b/platformio/util.py index c222310959..362545865f 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -371,7 +371,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): ] -@memoized(expire="5s") +@memoized(expire="10s") def _internet_on(): timeout = 2 socket.setdefaulttimeout(timeout) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py new file mode 100644 index 0000000000..2f7f4a6232 --- /dev/null +++ b/tests/package/test_manager.py @@ -0,0 +1,303 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +import pytest + +from platformio import fs, util +from platformio.package.exception import MissingPackageManifestError +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec +from platformio.package.pack import PackagePacker + + +def test_download(isolated_pio_core): + url = "https://github.com/platformio/platformio-core/archive/v4.3.4.zip" + checksum = "69d59642cb91e64344f2cdc1d3b98c5cd57679b5f6db7accc7707bd4c5d9664a" + lm = LibraryPackageManager() + archive_path = lm.download(url, checksum, silent=True) + assert fs.calculate_file_hashsum("sha256", archive_path) == checksum + lm.cleanup_expired_downloads() + assert os.path.isfile(archive_path) + # test outdated downloads + lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 1) + lm.cleanup_expired_downloads() + assert not os.path.isfile(archive_path) + # check that key is deleted from DB + with open(lm.get_download_usagedb_path()) as fp: + assert os.path.basename(archive_path) not in fp.read() + + +def test_find_pkg_root(isolated_pio_core, tmpdir_factory): + # has manifest + pkg_dir = tmpdir_factory.mktemp("package-has-manifest") + root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir() + root_dir.join("platform.json").write("") + pm = PlatformPackageManager() + found_dir = pm.find_pkg_root(str(pkg_dir), spec=None) + assert os.path.realpath(str(root_dir)) == os.path.realpath(found_dir) + + # does not have manifest + pkg_dir = tmpdir_factory.mktemp("package-does-not-have-manifest") + pkg_dir.join("nested").mkdir().join("folder").mkdir().join("readme.txt").write("") + pm = PlatformPackageManager() + with pytest.raises(MissingPackageManifestError): + pm.find_pkg_root(str(pkg_dir), spec=None) + + # library package without manifest, should find source root + pkg_dir = tmpdir_factory.mktemp("library-package-without-manifest") + root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir() + root_dir.join("src").mkdir().join("main.cpp").write("") + root_dir.join("include").mkdir().join("main.h").write("") + assert os.path.realpath(str(root_dir)) == os.path.realpath( + LibraryPackageManager.find_library_root(str(pkg_dir)) + ) + + # library manager should create "library.json" + lm = LibraryPackageManager() + spec = PackageSpec("custom-name@1.0.0") + pkg_root = lm.find_pkg_root(pkg_dir, spec) + manifest_path = os.path.join(pkg_root, "library.json") + assert os.path.realpath(str(root_dir)) == os.path.realpath(pkg_root) + assert os.path.isfile(manifest_path) + manifest = lm.load_manifest(pkg_root) + assert manifest["name"] == "custom-name" + assert "0.0.0" in str(manifest["version"]) + + +def test_build_legacy_spec(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + pm = PlatformPackageManager(str(storage_dir)) + # test src manifest + pkg1_dir = storage_dir.join("pkg-1").mkdir() + pkg1_dir.join(".pio").mkdir().join(".piopkgmanager.json").write( + """ +{ + "name": "StreamSpy-0.0.1.tar", + "url": "https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", + "requirements": null +} +""" + ) + assert pm.build_legacy_spec(str(pkg1_dir)) == PackageSpec( + name="StreamSpy-0.0.1.tar", + url="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", + ) + + # without src manifest + pkg2_dir = storage_dir.join("pkg-2").mkdir() + pkg2_dir.join("main.cpp").write("") + with pytest.raises(MissingPackageManifestError): + pm.build_legacy_spec(str(pkg2_dir)) + + # with package manifest + pkg3_dir = storage_dir.join("pkg-3").mkdir() + pkg3_dir.join("platform.json").write('{"name": "pkg3", "version": "1.2.0"}') + assert pm.build_legacy_spec(str(pkg3_dir)) == PackageSpec(name="pkg3") + + +def test_build_metadata(isolated_pio_core, tmpdir_factory): + pm = PlatformPackageManager() + vcs_revision = "a2ebfd7c0f" + pkg_dir = tmpdir_factory.mktemp("package") + + # test package without manifest + with pytest.raises(MissingPackageManifestError): + pm.load_manifest(str(pkg_dir)) + with pytest.raises(MissingPackageManifestError): + pm.build_metadata(str(pkg_dir), PackageSpec("MyLib")) + + # with manifest + pkg_dir.join("platform.json").write( + '{"name": "Dev-Platform", "version": "1.2.3-alpha.1"}' + ) + metadata = pm.build_metadata(str(pkg_dir), PackageSpec("owner/platform-name")) + assert metadata.name == "Dev-Platform" + assert str(metadata.version) == "1.2.3-alpha.1" + + # with vcs + metadata = pm.build_metadata( + str(pkg_dir), PackageSpec("owner/platform-name"), vcs_revision + ) + assert str(metadata.version) == ("1.2.3-alpha.1+sha." + vcs_revision) + assert metadata.version.build[1] == vcs_revision + + +def test_install_from_url(isolated_pio_core, tmpdir_factory): + tmp_dir = tmpdir_factory.mktemp("tmp") + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + + # install from local directory + src_dir = tmp_dir.join("local-lib-dir").mkdir() + src_dir.join("main.cpp").write("") + spec = PackageSpec("file://%s" % src_dir) + pkg = lm.install(spec, silent=True) + assert os.path.isfile(os.path.join(pkg.path, "main.cpp")) + manifest = lm.load_manifest(pkg) + assert manifest["name"] == "local-lib-dir" + assert manifest["version"].startswith("0.0.0+") + assert spec == pkg.metadata.spec + + # install from local archive + src_dir = tmp_dir.join("archive-src").mkdir() + root_dir = src_dir.mkdir("root") + root_dir.mkdir("src").join("main.cpp").write("#include ") + root_dir.join("library.json").write( + '{"name": "manifest-lib-name", "version": "2.0.0"}' + ) + tarball_path = PackagePacker(str(src_dir)).pack(str(tmp_dir)) + spec = PackageSpec("file://%s" % tarball_path) + pkg = lm.install(spec, silent=True) + assert os.path.isfile(os.path.join(pkg.path, "src", "main.cpp")) + assert pkg == lm.get_package(spec) + assert spec == pkg.metadata.spec + + # install from registry + src_dir = tmp_dir.join("registry-1").mkdir() + src_dir.join("library.properties").write( + """ +name = wifilib +version = 5.2.7 +""" + ) + spec = PackageSpec("company/wifilib @ ^5") + pkg = lm.install_from_url("file://%s" % src_dir, spec) + assert str(pkg.metadata.version) == "5.2.7" + + +def test_install_from_registry(isolated_pio_core, tmpdir_factory): + # Libraries + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) + # library with dependencies + lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert len(lm.get_installed()) == 3 + pkg = lm.get_package("AsyncTCP-esphome") + assert pkg.metadata.spec.owner == "ottowinter" + assert not lm.get_package("non-existing-package") + # mbed library + assert lm.install("wolfSSL", silent=True) + assert len(lm.get_installed()) == 4 + + # Tools + tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) + pkg = tm.install("tool-stlink @ ~1.10400.0", silent=True) + manifest = tm.load_manifest(pkg) + assert tm.is_system_compatible(manifest.get("system")) + assert util.get_systype() in manifest.get("system", []) + + +def test_get_installed(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + + # VCS package + ( + storage_dir.join("pkg-vcs") + .mkdir() + .join(".git") + .mkdir() + .join(".piopm") + .write( + """ +{ + "name": "pkg-via-vcs", + "spec": { + "id": null, + "name": "pkg-via-vcs", + "owner": null, + "requirements": null, + "url": "git+https://github.com/username/repo.git" + }, + "type": "library", + "version": "0.0.0+sha.1ea4d5e" +} +""" + ) + ) + + # package without metadata file + ( + storage_dir.join("foo@3.4.5") + .mkdir() + .join("library.json") + .write('{"name": "foo", "version": "3.4.5"}') + ) + + # package with metadata file + foo_dir = storage_dir.join("foo").mkdir() + foo_dir.join("library.json").write('{"name": "foo", "version": "3.6.0"}') + foo_dir.join(".piopm").write( + """ +{ + "name": "foo", + "spec": { + "name": "foo", + "owner": null, + "requirements": "^3" + }, + "type": "library", + "version": "3.6.0" +} +""" + ) + + # invalid package + storage_dir.join("invalid-package").mkdir().join("package.json").write( + '{"name": "tool-scons", "version": "4.0.0"}' + ) + + installed = lm.get_installed() + assert len(installed) == 3 + assert set(["pkg-via-vcs", "foo"]) == set(p.metadata.name for p in installed) + assert str(lm.get_package("foo").metadata.version) == "3.6.0" + + +def test_uninstall(isolated_pio_core, tmpdir_factory): + tmp_dir = tmpdir_factory.mktemp("tmp") + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + + # foo @ 1.0.0 + pkg_dir = tmp_dir.join("foo").mkdir() + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') + lm.install_from_url("file://%s" % pkg_dir, "foo") + # foo @ 1.3.0 + pkg_dir = tmp_dir.join("foo-1.3.0").mkdir() + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}') + lm.install_from_url("file://%s" % pkg_dir, "foo") + # bar + pkg_dir = tmp_dir.join("bar").mkdir() + pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}') + lm.install("file://%s" % pkg_dir, silent=True) + + assert len(lm.get_installed()) == 3 + assert os.path.isdir(os.path.join(str(storage_dir), "foo")) + assert os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) + + # check detaching + assert lm.uninstall("FOO", silent=True) + assert len(lm.get_installed()) == 2 + assert os.path.isdir(os.path.join(str(storage_dir), "foo")) + assert not os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) + + # uninstall the rest + assert lm.uninstall("foo", silent=True) + assert lm.uninstall("bar", silent=True) + + assert len(lm.get_installed()) == 0 diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py new file mode 100644 index 0000000000..d9d205c765 --- /dev/null +++ b/tests/package/test_meta.py @@ -0,0 +1,250 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import jsondiff +import semantic_version + +from platformio.package.meta import PackageMetaData, PackageSpec, PackageType + + +def test_spec_owner(): + assert PackageSpec("alice/foo library") == PackageSpec( + owner="alice", name="foo library" + ) + spec = PackageSpec(" Bob / BarUpper ") + assert spec != PackageSpec(owner="BOB", name="BARUPPER") + assert spec.owner == "Bob" + assert spec.name == "BarUpper" + + +def test_spec_id(): + assert PackageSpec(13) == PackageSpec(id=13) + assert PackageSpec("20") == PackageSpec(id=20) + spec = PackageSpec("id=199") + assert spec == PackageSpec(id=199) + assert isinstance(spec.id, int) + + +def test_spec_name(): + assert PackageSpec("foo") == PackageSpec(name="foo") + assert PackageSpec(" bar-24 ") == PackageSpec(name="bar-24") + + +def test_spec_requirements(): + assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") + assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") + assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") + spec = PackageSpec("id=20 @ !=1.2.3,<2.0") + assert isinstance(spec.requirements, semantic_version.SimpleSpec) + assert semantic_version.Version("1.3.0-beta.1") in spec.requirements + assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0") + + +def test_spec_local_urls(): + assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( + url="file:///tmp/foo.tar.gz", name="foo" + ) + assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec( + url="file:///tmp/bar.zip", name="customName" + ) + assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( + url="file:///tmp/some-lib/", name="some-lib" + ) + assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec( + url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1" + ) + + +def test_spec_external_urls(): + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/develop.zip", + name="platformio-core", + ) + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" + " @ !=2" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/" + "develop.zip?param=value", + name="platformio-core", + requirements="!=2", + ) + spec = PackageSpec( + "Custom-Name=" + "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" + ) + assert spec.is_custom_name() + assert spec.name == "Custom-Name" + assert spec == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/develop.tar.gz", + name="Custom-Name", + requirements="4.4.0", + ) + + +def test_spec_vcs_urls(): + assert PackageSpec("https://github.com/platformio/platformio-core") == PackageSpec( + name="platformio-core", url="git+https://github.com/platformio/platformio-core" + ) + assert PackageSpec("https://gitlab.com/username/reponame") == PackageSpec( + name="reponame", url="git+https://gitlab.com/username/reponame" + ) + assert PackageSpec( + "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) == PackageSpec( + name="wolfSSL", url="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) + assert PackageSpec( + "https://github.com/platformio/platformio-core.git#master" + ) == PackageSpec( + name="platformio-core", + url="git+https://github.com/platformio/platformio-core.git#master", + ) + assert PackageSpec( + "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0" + ) == PackageSpec( + name="core", + url="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", + requirements="4.4.0", + ) + assert PackageSpec( + "username@github.com:platformio/platformio-core.git" + ) == PackageSpec( + name="platformio-core", + url="git+username@github.com:platformio/platformio-core.git", + ) + assert PackageSpec( + "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5" + ) == PackageSpec( + name="pkg", + url="git+git@github.com:platformio/platformio-core.git", + requirements="^1.2.3,!=5", + ) + + +def test_spec_as_dict(): + assert not jsondiff.diff( + PackageSpec("bob/foo@1.2.3").as_dict(), + { + "owner": "bob", + "id": None, + "name": "foo", + "requirements": "1.2.3", + "url": None, + }, + ) + assert not jsondiff.diff( + PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" + " @ !=2" + ).as_dict(), + { + "owner": None, + "id": None, + "name": "platformio-core", + "requirements": "!=2", + "url": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value", + }, + ) + + +def test_metadata_as_dict(): + metadata = PackageMetaData(PackageType.LIBRARY, "foo", "1.2.3") + # test setter + metadata.version = "0.1.2+12345" + assert metadata.version == semantic_version.Version("0.1.2+12345") + assert not jsondiff.diff( + metadata.as_dict(), + { + "type": PackageType.LIBRARY, + "name": "foo", + "version": "0.1.2+12345", + "spec": None, + }, + ) + + assert not jsondiff.diff( + PackageMetaData( + PackageType.TOOL, + "toolchain", + "2.0.5", + PackageSpec("platformio/toolchain@~2.0.0"), + ).as_dict(), + { + "type": PackageType.TOOL, + "name": "toolchain", + "version": "2.0.5", + "spec": { + "owner": "platformio", + "id": None, + "name": "toolchain", + "requirements": "~2.0.0", + "url": None, + }, + }, + ) + + +def test_metadata_dump(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + metadata = PackageMetaData( + PackageType.TOOL, + "toolchain", + "2.0.5", + PackageSpec("platformio/toolchain@~2.0.0"), + ) + + dst = pkg_dir.join(".piopm") + metadata.dump(str(dst)) + assert os.path.isfile(str(dst)) + contents = dst.read() + assert all(s in contents for s in ("null", '"~2.0.0"')) + + +def test_metadata_load(tmpdir_factory): + contents = """ +{ + "name": "foo", + "spec": { + "name": "foo", + "owner": "username", + "requirements": "!=3.4.5" + }, + "type": "platform", + "version": "0.1.3" +} +""" + pkg_dir = tmpdir_factory.mktemp("package") + dst = pkg_dir.join(".piopm") + dst.write(contents) + metadata = PackageMetaData.load(str(dst)) + assert metadata.version == semantic_version.Version("0.1.3") + assert metadata == PackageMetaData( + PackageType.PLATFORM, + "foo", + "0.1.3", + spec=PackageSpec(owner="username", name="foo", requirements="!=3.4.5"), + ) + + piopm_path = pkg_dir.join(".piopm") + metadata = PackageMetaData( + PackageType.LIBRARY, "mylib", version="1.2.3", spec=PackageSpec("mylib") + ) + metadata.dump(str(piopm_path)) + restored_metadata = PackageMetaData.load(str(piopm_path)) + assert metadata == restored_metadata diff --git a/tests/package/test_spec.py b/tests/package/test_spec.py deleted file mode 100644 index dce89d7f92..0000000000 --- a/tests/package/test_spec.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from platformio.package.spec import PackageSpec - - -def test_ownername(): - assert PackageSpec("alice/foo library") == PackageSpec( - ownername="alice", name="foo library" - ) - assert PackageSpec(" bob / bar ") == PackageSpec(ownername="bob", name="bar") - - -def test_id(): - assert PackageSpec(13) == PackageSpec(id=13) - assert PackageSpec("20") == PackageSpec(id=20) - assert PackageSpec("id=199") == PackageSpec(id=199) - - -def test_name(): - assert PackageSpec("foo") == PackageSpec(name="foo") - assert PackageSpec(" bar-24 ") == PackageSpec(name="bar-24") - - -def test_requirements(): - assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") - assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") - assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") - assert PackageSpec("id=20 @ !=1.2.3,<2.0") == PackageSpec( - id=20, requirements="!=1.2.3,<2.0" - ) - - -def test_local_urls(): - assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( - url="file:///tmp/foo.tar.gz", name="foo" - ) - assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec( - url="file:///tmp/bar.zip", name="customName" - ) - assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( - url="file:///tmp/some-lib/", name="some-lib" - ) - assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec( - url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1" - ) - - -def test_external_urls(): - assert PackageSpec( - "https://github.com/platformio/platformio-core/archive/develop.zip" - ) == PackageSpec( - url="https://github.com/platformio/platformio-core/archive/develop.zip", - name="develop", - ) - assert PackageSpec( - "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" - " @ !=2" - ) == PackageSpec( - url="https://github.com/platformio/platformio-core/archive/" - "develop.zip?param=value", - name="develop", - requirements="!=2", - ) - assert PackageSpec( - "platformio-core=" - "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" - ) == PackageSpec( - url="https://github.com/platformio/platformio-core/archive/develop.tar.gz", - name="platformio-core", - requirements="4.4.0", - ) - - -def test_vcs_urls(): - assert PackageSpec( - "https://github.com/platformio/platformio-core.git" - ) == PackageSpec( - name="platformio-core", url="https://github.com/platformio/platformio-core.git", - ) - assert PackageSpec( - "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" - ) == PackageSpec( - name="wolfSSL", url="https://os.mbed.com/users/wolfSSL/code/wolfSSL/", - ) - assert PackageSpec( - "git+https://github.com/platformio/platformio-core.git#master" - ) == PackageSpec( - name="platformio-core", - url="git+https://github.com/platformio/platformio-core.git#master", - ) - assert PackageSpec( - "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0" - ) == PackageSpec( - name="core", - url="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", - requirements="4.4.0", - ) - assert PackageSpec("git@github.com:platformio/platformio-core.git") == PackageSpec( - name="platformio-core", url="git@github.com:platformio/platformio-core.git", - ) - assert PackageSpec( - "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5" - ) == PackageSpec( - name="pkg", - url="git+git@github.com:platformio/platformio-core.git", - requirements="^1.2.3,!=5", - ) From a1970bbfe316693fedbc4d569fbe8c82794570a7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 14:38:28 +0300 Subject: [PATCH 139/223] Allow a forced package installation with removing existing package --- platformio/package/manager/_download.py | 3 +- platformio/package/manager/_install.py | 50 ++++++++++++++++--------- platformio/package/manager/_registry.py | 2 +- platformio/package/manager/base.py | 19 +++++++++- platformio/package/meta.py | 11 +++++- tests/package/test_manager.py | 23 ++++++++++-- 6 files changed, 82 insertions(+), 26 deletions(-) diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index a052da0901..2ab8c14334 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -14,6 +14,7 @@ import hashlib import os +import shutil import tempfile import time @@ -85,7 +86,7 @@ def download(self, url, checksum=None, silent=False): raise e if checksum: fd.verify(checksum) - os.rename(tmp_path, dl_path) + shutil.copyfile(tmp_path, dl_path) finally: if os.path.isfile(tmp_path): os.remove(tmp_path) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index a5aa1c3894..58b1de1ebf 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -21,7 +21,6 @@ from platformio import app, compat, fs, util from platformio.package.exception import PackageException, UnknownPackageError -from platformio.package.lockfile import LockFile from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -43,25 +42,33 @@ def unpack(src, dst): with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=False) - def install(self, spec, silent=False): - with LockFile(self.package_dir): - pkg = self._install(spec, silent=silent) + def install(self, spec, silent=False, force=False): + try: + self.lock() + pkg = self._install(spec, silent=silent, force=force) self.memcache_reset() self.cleanup_expired_downloads() return pkg + finally: + self.unlock() - def _install(self, spec, search_filters=None, silent=False): + def _install(self, spec, search_filters=None, silent=False, force=False): spec = self.ensure_spec(spec) # avoid circle dependencies if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = [] + self.INSTALL_HISTORY = {} if spec in self.INSTALL_HISTORY: - return None - self.INSTALL_HISTORY.append(spec) + return self.INSTALL_HISTORY[spec] # check if package is already installed pkg = self.get_package(spec) + + # if a forced installation + if pkg and force: + self.uninstall(pkg, silent=silent) + pkg = None + if pkg: if not silent: click.secho( @@ -99,6 +106,7 @@ def _install(self, spec, search_filters=None, silent=False): self.memcache_reset() self.install_dependencies(pkg, silent) + self.INSTALL_HISTORY[spec] = pkg return pkg def install_dependencies(self, pkg, silent=False): @@ -240,15 +248,18 @@ def _cleanup_dir(path): shutil.move(tmp_pkg.path, dst_pkg.path) return PackageSourceItem(dst_pkg.path) - def uninstall(self, path_or_spec, silent=False): - with LockFile(self.package_dir): - pkg = ( - PackageSourceItem(path_or_spec) - if os.path.isdir(path_or_spec) - else self.get_package(path_or_spec) - ) + def uninstall(self, pkg, silent=False): + try: + self.lock() + + if not isinstance(pkg, PackageSourceItem): + pkg = ( + PackageSourceItem(pkg) + if os.path.isdir(pkg) + else self.get_package(pkg) + ) if not pkg or not pkg.metadata: - raise UnknownPackageError(path_or_spec) + raise UnknownPackageError(pkg) if not silent: self.print_message( @@ -276,7 +287,10 @@ def uninstall(self, path_or_spec, silent=False): os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), ) self.memcache_reset() + finally: + self.unlock() + + if not silent: + click.echo("[%s]" % click.style("OK", fg="green")) - if not silent: - click.echo("[%s]" % click.style("OK", fg="green")) return True diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 3f3ae813f9..cdd46cd067 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -37,7 +37,7 @@ def __init__(self, download_url): self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) self._visited_mirrors = [] - def __iter__(self): + def __iter__(self): # pylint: disable=non-iterator-returned return self def __next__(self): diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index e48fc25039..93599200b7 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -21,6 +21,7 @@ from platformio import fs, util from platformio.commands import PlatformioCLI from platformio.package.exception import ManifestException, MissingPackageManifestError +from platformio.package.lockfile import LockFile from platformio.package.manager._download import PackageManagerDownloadMixin from platformio.package.manager._install import PackageManagerInstallMixin from platformio.package.manager._registry import PackageManageRegistryMixin @@ -34,7 +35,7 @@ from platformio.project.helpers import get_project_cache_dir -class BasePackageManager( +class BasePackageManager( # pylint: disable=too-many-public-methods PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin ): MEMORY_CACHE = {} @@ -43,10 +44,26 @@ def __init__(self, pkg_type, package_dir): self.pkg_type = pkg_type self.package_dir = self.ensure_dir_exists(package_dir) self.MEMORY_CACHE = {} + + self._lockfile = None self._download_dir = None self._tmp_dir = None self._registry_client = None + def lock(self): + if self._lockfile: + return + self._lockfile = LockFile(self.package_dir) + self._lockfile.acquire() + + def unlock(self): + if hasattr(self, "_lockfile") and self._lockfile: + self._lockfile.release() + self._lockfile = None + + def __del__(self): + self.unlock() + def memcache_get(self, key, default=None): return self.MEMORY_CACHE.get(key, default) diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 0f1214e13f..6cd2904b3f 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -16,10 +16,11 @@ import os import re import tarfile +from binascii import crc32 import semantic_version -from platformio.compat import get_object_members, string_types +from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType try: @@ -89,6 +90,14 @@ def __eq__(self, other): ] ) + def __hash__(self): + return crc32( + hashlib_encode_data( + "%s-%s-%s-%s-%s" + % (self.owner, self.id, self.name, self.requirements, self.url) + ) + ) + def __repr__(self): return ( "PackageSpec 5 + + def test_get_installed(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") lm = LibraryPackageManager(str(storage_dir)) @@ -276,7 +291,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): # foo @ 1.0.0 pkg_dir = tmp_dir.join("foo").mkdir() pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') - lm.install_from_url("file://%s" % pkg_dir, "foo") + foo_1_0_0_pkg = lm.install_from_url("file://%s" % pkg_dir, "foo") # foo @ 1.3.0 pkg_dir = tmp_dir.join("foo-1.3.0").mkdir() pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}') @@ -284,7 +299,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): # bar pkg_dir = tmp_dir.join("bar").mkdir() pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}') - lm.install("file://%s" % pkg_dir, silent=True) + bar_pkg = lm.install("file://%s" % pkg_dir, silent=True) assert len(lm.get_installed()) == 3 assert os.path.isdir(os.path.join(str(storage_dir), "foo")) @@ -297,7 +312,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): assert not os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) # uninstall the rest - assert lm.uninstall("foo", silent=True) - assert lm.uninstall("bar", silent=True) + assert lm.uninstall(foo_1_0_0_pkg.path, silent=True) + assert lm.uninstall(bar_pkg, silent=True) assert len(lm.get_installed()) == 0 From 41c2d64ef0786b7dc511a21140fc98272f318f2e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 15:36:28 +0300 Subject: [PATCH 140/223] Fix "PermissionError: [WinError 32] The process cannot access the file" on Windows --- platformio/package/manager/_download.py | 6 ++++-- platformio/package/manager/_registry.py | 4 ++++ tests/package/test_manager.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 2ab8c14334..01be36aa81 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -61,7 +61,7 @@ def download(self, url, checksum=None, silent=False): return dl_path with_progress = not silent and not app.is_disabled_progressbar() - tmp_path = tempfile.mkstemp(dir=self.get_download_dir())[1] + tmp_fd, tmp_path = tempfile.mkstemp(dir=self.get_download_dir()) try: with LockFile(dl_path): try: @@ -86,9 +86,11 @@ def download(self, url, checksum=None, silent=False): raise e if checksum: fd.verify(checksum) - shutil.copyfile(tmp_path, dl_path) + os.close(tmp_fd) + os.rename(tmp_path, dl_path) finally: if os.path.isfile(tmp_path): + os.close(tmp_fd) os.remove(tmp_path) assert os.path.isfile(dl_path) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index cdd46cd067..a14bde98d2 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -64,6 +64,10 @@ def __next__(self): response.headers.get("X-PIO-Content-SHA256"), ) + def next(self): + """ For Python 2 compatibility """ + return self.__next__() + def get_http_client(self): if self._base_url not in RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES: RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[ diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 7d3cb07024..2d8a7b1419 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -71,7 +71,7 @@ def test_find_pkg_root(isolated_pio_core, tmpdir_factory): # library manager should create "library.json" lm = LibraryPackageManager() spec = PackageSpec("custom-name@1.0.0") - pkg_root = lm.find_pkg_root(pkg_dir, spec) + pkg_root = lm.find_pkg_root(str(pkg_dir), spec) manifest_path = os.path.join(pkg_root, "library.json") assert os.path.realpath(str(root_dir)) == os.path.realpath(pkg_root) assert os.path.isfile(manifest_path) From 6ac538fba4e75a1be5d44028459ff168c7ab866a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 15:49:10 +0300 Subject: [PATCH 141/223] Remove unused import --- platformio/package/manager/_download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 01be36aa81..83de9f3711 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -14,7 +14,6 @@ import hashlib import os -import shutil import tempfile import time From a01b3a247361c3af01e9201db492a08f140bcc2f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 19:58:59 +0300 Subject: [PATCH 142/223] Do not raise exception when package is not found (404), return None --- platformio/clients/http.py | 4 +++- platformio/clients/registry.py | 23 ++++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/platformio/clients/http.py b/platformio/clients/http.py index e125776226..3070f74918 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -69,4 +69,6 @@ def raise_error_from_response(response, expected_codes=(200, 201, 202)): message = response.json()["message"] except (KeyError, ValueError): message = response.text - raise HTTPClientError(message) + exc = HTTPClientError(message) + exc.response = response + raise exc diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index b7d724b9a7..f8130c60bc 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -14,7 +14,7 @@ from platformio import __registry_api__, fs from platformio.clients.account import AccountClient -from platformio.clients.http import HTTPClient +from platformio.clients.http import HTTPClient, HTTPClientError from platformio.package.meta import PackageType try: @@ -120,7 +120,7 @@ def list_packages(self, query=None, filters=None, page=None): for value in set( values if isinstance(values, (list, tuple)) else [values] ): - search_query.append("%s:%s" % (name[:-1], value)) + search_query.append('%s:"%s"' % (name[:-1], value)) if query: search_query.append(query) params = dict(query=quote(" ".join(search_query))) @@ -129,10 +129,15 @@ def list_packages(self, query=None, filters=None, page=None): return self.request_json_data("get", "/v3/packages", params=params) def get_package(self, type_, owner, name, version=None): - return self.request_json_data( - "get", - "/v3/packages/{owner}/{type}/{name}".format( - type=type_, owner=owner, name=quote(name) - ), - params=dict(version=version) if version else None, - ) + try: + return self.request_json_data( + "get", + "/v3/packages/{owner}/{type}/{name}".format( + type=type_, owner=owner.lower(), name=quote(name.lower()) + ), + params=dict(version=version) if version else None, + ) + except HTTPClientError as e: + if e.response.status_code == 404: + return None + raise e From 2dd69e21c00f7e843521b7a998063d60ae803d43 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Aug 2020 20:17:07 +0300 Subject: [PATCH 143/223] Implement package removing with dependencies --- platformio/clients/http.py | 14 ++-- platformio/package/manager/_install.py | 69 +++--------------- platformio/package/manager/_registry.py | 29 +++++--- platformio/package/manager/_uninstall.py | 93 ++++++++++++++++++++++++ platformio/package/manager/base.py | 16 ++-- tests/package/test_manager.py | 28 ++++++- 6 files changed, 167 insertions(+), 82 deletions(-) create mode 100644 platformio/package/manager/_uninstall.py diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 3070f74918..47f6c16260 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -20,7 +20,13 @@ class HTTPClientError(PlatformioException): - pass + def __init__(self, message, response=None): + super(HTTPClientError, self).__init__() + self.message = message + self.response = response + + def __str__(self): # pragma: no cover + return self.message class HTTPClient(object): @@ -52,7 +58,7 @@ def send_request(self, method, path, **kwargs): try: return getattr(self._session, method)(self.base_url + path, **kwargs) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - raise HTTPClientError(e) + raise HTTPClientError(str(e)) def request_json_data(self, *args, **kwargs): response = self.send_request(*args, **kwargs) @@ -69,6 +75,4 @@ def raise_error_from_response(response, expected_codes=(200, 201, 202)): message = response.json()["message"] except (KeyError, ValueError): message = response.text - exc = HTTPClientError(message) - exc.response = response - raise exc + raise HTTPClientError(message, response) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 58b1de1ebf..ea409f2ecf 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -20,7 +20,7 @@ import click from platformio import app, compat, fs, util -from platformio.package.exception import PackageException, UnknownPackageError +from platformio.package.exception import PackageException from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -28,7 +28,7 @@ class PackageManagerInstallMixin(object): - INSTALL_HISTORY = None # avoid circle dependencies + _INSTALL_HISTORY = None # avoid circle dependencies @staticmethod def unpack(src, dst): @@ -56,10 +56,10 @@ def _install(self, spec, search_filters=None, silent=False, force=False): spec = self.ensure_spec(spec) # avoid circle dependencies - if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = {} - if spec in self.INSTALL_HISTORY: - return self.INSTALL_HISTORY[spec] + if not self._INSTALL_HISTORY: + self._INSTALL_HISTORY = {} + if spec in self._INSTALL_HISTORY: + return self._INSTALL_HISTORY[spec] # check if package is already installed pkg = self.get_package(spec) @@ -105,11 +105,11 @@ def _install(self, spec, search_filters=None, silent=False, force=False): ) self.memcache_reset() - self.install_dependencies(pkg, silent) - self.INSTALL_HISTORY[spec] = pkg + self._install_dependencies(pkg, silent) + self._INSTALL_HISTORY[spec] = pkg return pkg - def install_dependencies(self, pkg, silent=False): + def _install_dependencies(self, pkg, silent=False): assert isinstance(pkg, PackageSourceItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): @@ -117,14 +117,14 @@ def install_dependencies(self, pkg, silent=False): if not silent: self.print_message(click.style("Installing dependencies...", fg="yellow")) for dependency in manifest.get("dependencies"): - if not self.install_dependency(dependency, silent) and not silent: + if not self._install_dependency(dependency, silent) and not silent: click.secho( "Warning! Could not install dependency %s for package '%s'" % (dependency, pkg.metadata.name), fg="yellow", ) - def install_dependency(self, dependency, silent=False): + def _install_dependency(self, dependency, silent=False): spec = PackageSpec( name=dependency.get("name"), requirements=dependency.get("version") ) @@ -247,50 +247,3 @@ def _cleanup_dir(path): _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) return PackageSourceItem(dst_pkg.path) - - def uninstall(self, pkg, silent=False): - try: - self.lock() - - if not isinstance(pkg, PackageSourceItem): - pkg = ( - PackageSourceItem(pkg) - if os.path.isdir(pkg) - else self.get_package(pkg) - ) - if not pkg or not pkg.metadata: - raise UnknownPackageError(pkg) - - if not silent: - self.print_message( - "Uninstalling %s @ %s: \t" - % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), - nl=False, - ) - if os.path.islink(pkg.path): - os.unlink(pkg.path) - else: - fs.rmtree(pkg.path) - self.memcache_reset() - - # unfix detached-package with the same name - detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) - if ( - detached_pkg - and "@" in detached_pkg.path - and not os.path.isdir( - os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) - ) - ): - shutil.move( - detached_pkg.path, - os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), - ) - self.memcache_reset() - finally: - self.unlock() - - if not silent: - click.echo("[%s]" % click.style("OK", fg="green")) - - return True diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index a14bde98d2..d5c9ddadce 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -78,12 +78,19 @@ def get_http_client(self): class PackageManageRegistryMixin(object): def install_from_registry(self, spec, search_filters=None, silent=False): - packages = self.search_registry_packages(spec, search_filters) - if not packages: - raise UnknownPackageError(spec.humanize()) - if len(packages) > 1 and not silent: - self.print_multi_package_issue(packages, spec) - package, version = self.find_best_registry_version(packages, spec) + if spec.owner and spec.name and not search_filters: + package = self.fetch_registry_package(spec.owner, spec.name) + if not package: + raise UnknownPackageError(spec.humanize()) + version = self._pick_best_pkg_version(package["versions"], spec) + else: + packages = self.search_registry_packages(spec, search_filters) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1 and not silent: + self.print_multi_package_issue(packages, spec) + package, version = self.find_best_registry_version(packages, spec) + pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None if not pkgfile: raise UnknownPackageError(spec.humanize()) @@ -117,17 +124,17 @@ def search_registry_packages(self, spec, filters=None): filters["ids"] = str(spec.id) else: filters["types"] = self.pkg_type - filters["names"] = '"%s"' % spec.name.lower() + filters["names"] = spec.name.lower() if spec.owner: filters["owners"] = spec.owner.lower() return self.get_registry_client_instance().list_packages(filters=filters)[ "items" ] - def fetch_registry_package_versions(self, owner, name): + def fetch_registry_package(self, owner, name): return self.get_registry_client_instance().get_package( self.pkg_type, owner, name - )["versions"] + ) @staticmethod def print_multi_package_issue(packages, spec): @@ -163,9 +170,9 @@ def find_best_registry_version(self, packages, spec): # if the custom version requirements, check ALL package versions for package in packages: version = self._pick_best_pkg_version( - self.fetch_registry_package_versions( + self.fetch_registry_package( package["owner"]["username"], package["name"] - ), + ).get("versions"), spec, ) if version: diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py new file mode 100644 index 0000000000..e754eab2a0 --- /dev/null +++ b/platformio/package/manager/_uninstall.py @@ -0,0 +1,93 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil + +import click + +from platformio import fs +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageSourceItem, PackageSpec + + +class PackageManagerUninstallMixin(object): + def uninstall(self, pkg, silent=False, skip_dependencies=False): + try: + self.lock() + return self._uninstall(pkg, silent, skip_dependencies) + finally: + self.unlock() + + def _uninstall(self, pkg, silent=False, skip_dependencies=False): + if not isinstance(pkg, PackageSourceItem): + pkg = ( + PackageSourceItem(pkg) if os.path.isdir(pkg) else self.get_package(pkg) + ) + if not pkg or not pkg.metadata: + raise UnknownPackageError(pkg) + + if not silent: + self.print_message( + "Removing %s @ %s: \t" + % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), + nl=False, + ) + + # firstly, remove dependencies + if not skip_dependencies: + self._uninstall_dependencies(pkg, silent) + + if os.path.islink(pkg.path): + os.unlink(pkg.path) + else: + fs.rmtree(pkg.path) + self.memcache_reset() + + # unfix detached-package with the same name + detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) + if ( + detached_pkg + and "@" in detached_pkg.path + and not os.path.isdir( + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) + ) + ): + shutil.move( + detached_pkg.path, + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), + ) + self.memcache_reset() + + if not silent: + click.echo("[%s]" % click.style("OK", fg="green")) + + return True + + def _uninstall_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageSourceItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message(click.style("Removing dependencies...", fg="yellow")) + for dependency in manifest.get("dependencies"): + pkg = self.get_package( + PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + ) + if not pkg: + continue + self._uninstall(pkg, silent=silent) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index 93599200b7..ca06583350 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -25,6 +25,7 @@ from platformio.package.manager._download import PackageManagerDownloadMixin from platformio.package.manager._install import PackageManagerInstallMixin from platformio.package.manager._registry import PackageManageRegistryMixin +from platformio.package.manager._uninstall import PackageManagerUninstallMixin from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import ( PackageMetaData, @@ -36,14 +37,17 @@ class BasePackageManager( # pylint: disable=too-many-public-methods - PackageManagerDownloadMixin, PackageManageRegistryMixin, PackageManagerInstallMixin + PackageManagerDownloadMixin, + PackageManageRegistryMixin, + PackageManagerInstallMixin, + PackageManagerUninstallMixin, ): - MEMORY_CACHE = {} + _MEMORY_CACHE = {} def __init__(self, pkg_type, package_dir): self.pkg_type = pkg_type self.package_dir = self.ensure_dir_exists(package_dir) - self.MEMORY_CACHE = {} + self._MEMORY_CACHE = {} self._lockfile = None self._download_dir = None @@ -65,13 +69,13 @@ def __del__(self): self.unlock() def memcache_get(self, key, default=None): - return self.MEMORY_CACHE.get(key, default) + return self._MEMORY_CACHE.get(key, default) def memcache_set(self, key, value): - self.MEMORY_CACHE[key] = value + self._MEMORY_CACHE[key] = value def memcache_reset(self): - self.MEMORY_CACHE.clear() + self._MEMORY_CACHE.clear() @staticmethod def is_system_compatible(value): diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 2d8a7b1419..5898ae2b71 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -18,7 +18,10 @@ import pytest from platformio import fs, util -from platformio.package.exception import MissingPackageManifestError +from platformio.package.exception import ( + MissingPackageManifestError, + UnknownPackageError, +) from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager @@ -193,14 +196,24 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory): # mbed library assert lm.install("wolfSSL", silent=True) assert len(lm.get_installed()) == 4 + # case sensitive author name + assert lm.install("DallasTemperature", silent=True) + assert lm.get_package("OneWire").metadata.version.major >= 2 + assert len(lm.get_installed()) == 6 # Tools tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) - pkg = tm.install("tool-stlink @ ~1.10400.0", silent=True) + pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True) manifest = tm.load_manifest(pkg) assert tm.is_system_compatible(manifest.get("system")) assert util.get_systype() in manifest.get("system", []) + # Test unknown + with pytest.raises(UnknownPackageError): + tm.install("unknown-package-tool @ 9.1.1", silent=True) + with pytest.raises(UnknownPackageError): + tm.install("owner/unknown-package-tool", silent=True) + def test_install_force(isolated_pio_core, tmpdir_factory): lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) @@ -316,3 +329,14 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): assert lm.uninstall(bar_pkg, silent=True) assert len(lm.get_installed()) == 0 + + # test uninstall dependencies + assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert len(lm.get_installed()) == 3 + assert lm.uninstall("AsyncMqttClient-esphome", silent=True, skip_dependencies=True) + assert len(lm.get_installed()) == 2 + + lm = LibraryPackageManager(str(storage_dir)) + assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert lm.uninstall("AsyncMqttClient-esphome", silent=True) + assert len(lm.get_installed()) == 0 From 893ca1b328523650a9832ba2dbfaf14994d61584 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Aug 2020 13:27:05 +0300 Subject: [PATCH 144/223] Switch library manager to the new package manager --- platformio/builder/tools/piolib.py | 58 ++- platformio/clients/http.py | 5 +- platformio/commands/lib/__init__.py | 13 + .../commands/{lib.py => lib/command.py} | 178 ++++---- platformio/commands/lib/helpers.py | 90 ++++ platformio/commands/system/command.py | 4 +- platformio/commands/update.py | 8 +- platformio/compat.py | 8 + platformio/exception.py | 9 - platformio/maintenance.py | 41 +- platformio/managers/lib.py | 374 ---------------- platformio/managers/package.py | 8 +- platformio/package/manager/_install.py | 39 +- platformio/package/manager/_legacy.py | 57 +++ platformio/package/manager/_registry.py | 50 ++- platformio/package/manager/_uninstall.py | 7 +- platformio/package/manager/_update.py | 166 +++++++ platformio/package/manager/base.py | 76 +--- platformio/package/manager/library.py | 2 +- platformio/package/manager/platform.py | 2 +- platformio/package/manager/tool.py | 2 +- platformio/package/manifest/schema.py | 2 +- platformio/package/meta.py | 69 ++- platformio/package/vcsclient.py | 34 +- tests/commands/test_ci.py | 2 +- tests/commands/test_lib.py | 414 ++++++------------ tests/commands/test_lib_complex.py | 348 +++++++++++++++ tests/package/test_manager.py | 85 ++++ tests/package/test_meta.py | 50 ++- tests/test_maintenance.py | 4 +- 30 files changed, 1304 insertions(+), 901 deletions(-) create mode 100644 platformio/commands/lib/__init__.py rename platformio/commands/{lib.py => lib/command.py} (81%) create mode 100644 platformio/commands/lib/helpers.py delete mode 100644 platformio/managers/lib.py create mode 100644 platformio/package/manager/_legacy.py create mode 100644 platformio/package/manager/_update.py create mode 100644 tests/commands/test_lib_complex.py diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index fbd8949cc0..24229b1c75 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -34,11 +34,13 @@ from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool from platformio.compat import WINDOWS, hashlib_encode_data, string_types -from platformio.managers.lib import LibraryManager +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.library import LibraryPackageManager from platformio.package.manifest.parser import ( ManifestParserError, ManifestParserFactory, ) +from platformio.package.meta import PackageSourceItem from platformio.project.options import ProjectOptions @@ -851,34 +853,36 @@ def process_extra_options(self): pass def install_dependencies(self): - def _is_builtin(uri): + def _is_builtin(spec): for lb in self.env.GetLibBuilders(): - if lb.name == uri: + if lb.name == spec: return True return False - not_found_uri = [] - for uri in self.dependencies: + not_found_specs = [] + for spec in self.dependencies: # check if built-in library - if _is_builtin(uri): + if _is_builtin(spec): continue found = False for storage_dir in self.env.GetLibSourceDirs(): - lm = LibraryManager(storage_dir) - if lm.get_package_dir(*lm.parse_pkg_uri(uri)): + lm = LibraryPackageManager(storage_dir) + if lm.get_package(spec): found = True break if not found: - not_found_uri.append(uri) + not_found_specs.append(spec) did_install = False - lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))) - for uri in not_found_uri: + lm = LibraryPackageManager( + self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) + ) + for spec in not_found_specs: try: - lm.install(uri) + lm.install(spec) did_install = True - except (exception.LibNotFound, exception.InternetIsOffline) as e: + except (UnknownPackageError, exception.InternetIsOffline) as e: click.secho("Warning! %s" % e, fg="yellow") # reset cache @@ -886,17 +890,17 @@ def _is_builtin(uri): DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None) def process_dependencies(self): # pylint: disable=too-many-branches - for uri in self.dependencies: + for spec in self.dependencies: found = False for storage_dir in self.env.GetLibSourceDirs(): if found: break - lm = LibraryManager(storage_dir) - lib_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri)) - if not lib_dir: + lm = LibraryPackageManager(storage_dir) + pkg = lm.get_package(spec) + if not pkg: continue for lb in self.env.GetLibBuilders(): - if lib_dir != lb.path: + if pkg.path != lb.path: continue if lb not in self.depbuilders: self.depend_recursive(lb) @@ -908,7 +912,7 @@ def process_dependencies(self): # pylint: disable=too-many-branches # look for built-in libraries by a name # which don't have package manifest for lb in self.env.GetLibBuilders(): - if lb.name != uri: + if lb.name != spec: continue if lb not in self.depbuilders: self.depend_recursive(lb) @@ -1000,10 +1004,6 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches def ConfigureProjectLibBuilder(env): - def _get_vcs_info(lb): - path = LibraryManager.get_src_manifest_path(lb.path) - return fs.load_json(path) if path else None - def _correct_found_libs(lib_builders): # build full dependency graph found_lbs = [lb for lb in lib_builders if lb.dependent] @@ -1019,15 +1019,13 @@ def _print_deps_tree(root, level=0): margin = "| " * (level) for lb in root.depbuilders: title = "<%s>" % lb.name - vcs_info = _get_vcs_info(lb) - if lb.version: - title += " %s" % lb.version - if vcs_info and vcs_info.get("version"): - title += " #%s" % vcs_info.get("version") + pkg = PackageSourceItem(lb.path) + if pkg.metadata: + title += " %s" % pkg.metadata.version click.echo("%s|-- %s" % (margin, title), nl=False) if int(ARGUMENTS.get("PIOVERBOSE", 0)): - if vcs_info: - click.echo(" [%s]" % vcs_info.get("url"), nl=False) + if pkg.metadata and pkg.metadata.spec.external: + click.echo(" [%s]" % pkg.metadata.spec.url, nl=False) click.echo(" (", nl=False) click.echo(lb.path, nl=False) click.echo(")", nl=False) diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 47f6c16260..974017b733 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -30,7 +30,9 @@ def __str__(self): # pragma: no cover class HTTPClient(object): - def __init__(self, base_url): + def __init__( + self, base_url, + ): if base_url.endswith("/"): base_url = base_url[:-1] self.base_url = base_url @@ -51,6 +53,7 @@ def __del__(self): self._session.close() self._session = None + @util.throttle(500) def send_request(self, method, path, **kwargs): # check Internet before and resolve issue with 60 seconds timeout # print(self, method, path, kwargs) diff --git a/platformio/commands/lib/__init__.py b/platformio/commands/lib/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/commands/lib/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/lib.py b/platformio/commands/lib/command.py similarity index 81% rename from platformio/commands/lib.py rename to platformio/commands/lib/command.py index 5bd38aee9d..33249f3edf 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib/command.py @@ -18,16 +18,21 @@ import time import click -import semantic_version from tabulate import tabulate from platformio import exception, fs, util from platformio.commands import PlatformioCLI +from platformio.commands.lib.helpers import ( + get_builtin_libs, + is_builtin_lib, + save_project_libdeps, +) from platformio.compat import dump_json_to_unicode -from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.proc import is_ci from platformio.project.config import ProjectConfig -from platformio.project.exception import InvalidProjectConfError from platformio.project.helpers import get_project_dir, is_platformio_project try: @@ -124,89 +129,106 @@ def cli(ctx, **options): @cli.command("install", short_help="Install library") @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") @click.option( - "--save", + "--save/--no-save", is_flag=True, - help="Save installed libraries into the `platformio.ini` dependency list", + default=True, + help="Save installed libraries into the `platformio.ini` dependency list" + " (enabled by default)", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option( - "--interactive", is_flag=True, help="Allow to make a choice for all prompts" + "--interactive", + is_flag=True, + help="Deprecated! Please use a strict dependency specification (owner/libname)", ) @click.option( "-f", "--force", is_flag=True, help="Reinstall/redownload library if exists" ) @click.pass_context -def lib_install( # pylint: disable=too-many-arguments +def lib_install( # pylint: disable=too-many-arguments,unused-argument ctx, libraries, save, silent, interactive, force ): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, []) - installed_manifests = {} + installed_pkgs = {} for storage_dir in storage_dirs: if not silent and (libraries or storage_dir in storage_libdeps): print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) + lm = LibraryPackageManager(storage_dir) + if libraries: - for library in libraries: - pkg_dir = lm.install( - library, silent=silent, interactive=interactive, force=force - ) - installed_manifests[library] = lm.load_manifest(pkg_dir) + installed_pkgs = { + library: lm.install(library, silent=silent, force=force) + for library in libraries + } + elif storage_dir in storage_libdeps: builtin_lib_storages = None for library in storage_libdeps[storage_dir]: try: - pkg_dir = lm.install( - library, silent=silent, interactive=interactive, force=force - ) - installed_manifests[library] = lm.load_manifest(pkg_dir) - except exception.LibNotFound as e: + lm.install(library, silent=silent, force=force) + except UnknownPackageError as e: if builtin_lib_storages is None: builtin_lib_storages = get_builtin_libs() if not silent or not is_builtin_lib(builtin_lib_storages, library): click.secho("Warning! %s" % e, fg="yellow") - if not save or not libraries: - return + if save and installed_pkgs: + _save_deps(ctx, installed_pkgs) + + +def _save_deps(ctx, pkgs, action="add"): + specs = [] + for library, pkg in pkgs.items(): + spec = PackageSpec(library) + if spec.external: + specs.append(spec) + else: + specs.append( + PackageSpec( + owner=pkg.metadata.spec.owner, + name=pkg.metadata.spec.name, + requirements=spec.requirements + or ( + ("^%s" % pkg.metadata.version) + if not pkg.metadata.version.build + else pkg.metadata.version + ), + ) + ) input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] for input_dir in input_dirs: - config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini")) - config.validate(project_environments) - for env in config.envs(): - if project_environments and env not in project_environments: - continue - config.expand_interpolations = False - try: - lib_deps = config.get("env:" + env, "lib_deps") - except InvalidProjectConfError: - lib_deps = [] - for library in libraries: - if library in lib_deps: - continue - manifest = installed_manifests[library] - try: - assert library.lower() == manifest["name"].lower() - assert semantic_version.Version(manifest["version"]) - lib_deps.append("{name}@^{version}".format(**manifest)) - except (AssertionError, ValueError): - lib_deps.append(library) - config.set("env:" + env, "lib_deps", lib_deps) - config.save() + if not is_platformio_project(input_dir): + continue + save_project_libdeps(input_dir, specs, project_environments, action=action) -@cli.command("uninstall", short_help="Uninstall libraries") +@cli.command("uninstall", short_help="Remove libraries") @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]") +@click.option( + "--save/--no-save", + is_flag=True, + default=True, + help="Remove libraries from the `platformio.ini` dependency list and save changes" + " (enabled by default)", +) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.pass_context -def lib_uninstall(ctx, libraries): +def lib_uninstall(ctx, libraries, save, silent): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] + uninstalled_pkgs = {} for storage_dir in storage_dirs: print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) - for library in libraries: - lm.uninstall(library) + lm = LibraryPackageManager(storage_dir) + uninstalled_pkgs = { + library: lm.uninstall(library, silent=silent) for library in libraries + } + + if save and uninstalled_pkgs: + _save_deps(ctx, uninstalled_pkgs, action="remove") @cli.command("update", short_help="Update installed libraries") @@ -220,42 +242,53 @@ def lib_uninstall(ctx, libraries): @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option("--json-output", is_flag=True) @click.pass_context -def lib_update(ctx, libraries, only_check, dry_run, json_output): +def lib_update( # pylint: disable=too-many-arguments + ctx, libraries, only_check, dry_run, silent, json_output +): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] only_check = dry_run or only_check json_result = {} for storage_dir in storage_dirs: if not json_output: print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) - - _libraries = libraries - if not _libraries: - _libraries = [manifest["__pkg_dir"] for manifest in lm.get_installed()] + lm = LibraryPackageManager(storage_dir) + _libraries = libraries or lm.get_installed() if only_check and json_output: result = [] for library in _libraries: - pkg_dir = library if os.path.isdir(library) else None - requirements = None - url = None - if not pkg_dir: - name, requirements, url = lm.parse_pkg_uri(library) - pkg_dir = lm.get_package_dir(name, requirements, url) - if not pkg_dir: + spec = None + pkg = None + if isinstance(library, PackageSourceItem): + pkg = library + else: + spec = PackageSpec(library) + pkg = lm.get_package(spec) + if not pkg: continue - latest = lm.outdated(pkg_dir, requirements) - if not latest: + outdated = lm.outdated(pkg, spec) + if not outdated.is_outdated(allow_incompatible=True): continue - manifest = lm.load_manifest(pkg_dir) - manifest["versionLatest"] = latest + manifest = lm.legacy_load_manifest(pkg) + manifest["versionWanted"] = ( + str(outdated.wanted) if outdated.wanted else None + ) + manifest["versionLatest"] = ( + str(outdated.latest) if outdated.latest else None + ) result.append(manifest) json_result[storage_dir] = result else: for library in _libraries: - lm.update(library, only_check=only_check) + spec = ( + None + if isinstance(library, PackageSourceItem) + else PackageSpec(library) + ) + lm.update(library, spec=spec, only_check=only_check, silent=silent) if json_output: return click.echo( @@ -276,8 +309,8 @@ def lib_list(ctx, json_output): for storage_dir in storage_dirs: if not json_output: print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) - items = lm.get_installed() + lm = LibraryPackageManager(storage_dir) + items = lm.legacy_get_installed() if json_output: json_result[storage_dir] = items elif items: @@ -301,6 +334,7 @@ def lib_list(ctx, json_output): @click.option("--json-output", is_flag=True) @click.option("--page", type=click.INT, default=1) @click.option("--id", multiple=True) +@click.option("-o", "--owner", multiple=True) @click.option("-n", "--name", multiple=True) @click.option("-a", "--author", multiple=True) @click.option("-k", "--keyword", multiple=True) @@ -404,12 +438,8 @@ def lib_builtin(storage, json_output): @click.argument("library", metavar="[LIBRARY]") @click.option("--json-output", is_flag=True) def lib_show(library, json_output): - lm = LibraryManager() - name, requirements, _ = lm.parse_pkg_uri(library) - lib_id = lm.search_lib_id( - {"name": name, "requirements": requirements}, - silent=json_output, - interactive=not json_output, + lib_id = LibraryPackageManager().reveal_registry_package_id( + library, silent=json_output ) lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") if json_output: diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py new file mode 100644 index 0000000000..a5cc07e35a --- /dev/null +++ b/platformio/commands/lib/helpers.py @@ -0,0 +1,90 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio.compat import ci_strings_are_equal +from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.meta import PackageSpec +from platformio.project.config import ProjectConfig +from platformio.project.exception import InvalidProjectConfError + + +def get_builtin_libs(storage_names=None): + # pylint: disable=import-outside-toplevel + from platformio.package.manager.library import LibraryPackageManager + + items = [] + storage_names = storage_names or [] + pm = PlatformManager() + for manifest in pm.get_installed(): + p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + for storage in p.get_lib_storages(): + if storage_names and storage["name"] not in storage_names: + continue + lm = LibraryPackageManager(storage["path"]) + items.append( + { + "name": storage["name"], + "path": storage["path"], + "items": lm.legacy_get_installed(), + } + ) + return items + + +def is_builtin_lib(storages, name): + for storage in storages or []: + if any(lib.get("name") == name for lib in storage["items"]): + return True + return False + + +def ignore_deps_by_specs(deps, specs): + result = [] + for dep in deps: + depspec = PackageSpec(dep) + if depspec.external: + result.append(dep) + continue + ignore_conditions = [] + for spec in specs: + if depspec.owner: + ignore_conditions.append( + ci_strings_are_equal(depspec.owner, spec.owner) + and ci_strings_are_equal(depspec.name, spec.name) + ) + else: + ignore_conditions.append(ci_strings_are_equal(depspec.name, spec.name)) + if not any(ignore_conditions): + result.append(dep) + return result + + +def save_project_libdeps(project_dir, specs, environments=None, action="add"): + config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) + config.validate(environments) + for env in config.envs(): + if environments and env not in environments: + continue + config.expand_interpolations = False + lib_deps = [] + try: + lib_deps = ignore_deps_by_specs(config.get("env:" + env, "lib_deps"), specs) + except InvalidProjectConfError: + pass + if action == "add": + lib_deps.extend(spec.as_dependency() for spec in specs) + config.set("env:" + env, "lib_deps", lib_deps) + config.save() diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index a684e14a93..af4071e072 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -26,9 +26,9 @@ install_completion_code, uninstall_completion_code, ) -from platformio.managers.lib import LibraryManager from platformio.managers.package import PackageManager from platformio.managers.platform import PlatformManager +from platformio.package.manager.library import LibraryPackageManager from platformio.project.config import ProjectConfig @@ -73,7 +73,7 @@ def system_info(json_output): } data["global_lib_nums"] = { "title": "Global Libraries", - "value": len(LibraryManager().get_installed()), + "value": len(LibraryPackageManager().get_installed()), } data["dev_platform_nums"] = { "title": "Development Platforms", diff --git a/platformio/commands/update.py b/platformio/commands/update.py index 1bac4f777f..bf8291655a 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -15,11 +15,11 @@ import click from platformio import app -from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY -from platformio.commands.lib import lib_update as cmd_lib_update +from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY +from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.managers.core import update_core_packages -from platformio.managers.lib import LibraryManager +from platformio.package.manager.library import LibraryPackageManager @click.command( @@ -55,5 +55,5 @@ def cli(ctx, core_packages, only_check, dry_run): click.echo() click.echo("Library Manager") click.echo("===============") - ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryManager().package_dir] + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryPackageManager().package_dir] ctx.invoke(cmd_lib_update, only_check=only_check) diff --git a/platformio/compat.py b/platformio/compat.py index 7f749fc96a..59362d01b3 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -50,6 +50,14 @@ def get_object_members(obj, ignore_private=True): } +def ci_strings_are_equal(a, b): + if a == b: + return True + if not a or not b: + return False + return a.strip().lower() == b.strip().lower() + + if PY2: import imp diff --git a/platformio/exception.py b/platformio/exception.py index c39b79577a..9ab0e4d810 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -124,15 +124,6 @@ class PackageInstallError(PlatformIOPackageException): # -class LibNotFound(PlatformioException): - - MESSAGE = ( - "Library `{0}` has not been found in PlatformIO Registry.\n" - "You can ignore this message, if `{0}` is a built-in library " - "(included in framework, SDK). E.g., SPI, Wire, etc." - ) - - class NotGlobalLibDir(UserSideException): MESSAGE = ( diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 0c8ee2dfa0..b0e64f523b 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -21,13 +21,13 @@ from platformio import __version__, app, exception, fs, telemetry, util from platformio.commands import PlatformioCLI -from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY -from platformio.commands.lib import lib_update as cmd_lib_update +from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY +from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version from platformio.managers.core import update_core_packages -from platformio.managers.lib import LibraryManager from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.library import LibraryPackageManager from platformio.proc import is_container @@ -240,7 +240,7 @@ def check_platformio_upgrade(): click.echo("") -def check_internal_updates(ctx, what): +def check_internal_updates(ctx, what): # pylint: disable=too-many-branches last_check = app.get_state_item("last_check", {}) interval = int(app.get_setting("check_%s_interval" % what)) * 3600 * 24 if (time() - interval) < last_check.get(what + "_update", 0): @@ -251,20 +251,27 @@ def check_internal_updates(ctx, what): util.internet_on(raise_exception=True) - pm = PlatformManager() if what == "platforms" else LibraryManager() outdated_items = [] - for manifest in pm.get_installed(): - if manifest["name"] in outdated_items: - continue - conds = [ - pm.outdated(manifest["__pkg_dir"]), - what == "platforms" - and PlatformFactory.newPlatform( - manifest["__pkg_dir"] - ).are_outdated_packages(), - ] - if any(conds): - outdated_items.append(manifest["name"]) + pm = PlatformManager() if what == "platforms" else LibraryPackageManager() + if isinstance(pm, PlatformManager): + for manifest in pm.get_installed(): + if manifest["name"] in outdated_items: + continue + conds = [ + pm.outdated(manifest["__pkg_dir"]), + what == "platforms" + and PlatformFactory.newPlatform( + manifest["__pkg_dir"] + ).are_outdated_packages(), + ] + if any(conds): + outdated_items.append(manifest["name"]) + else: + for pkg in pm.get_installed(): + if pkg.metadata.name in outdated_items: + continue + if pm.outdated(pkg).is_outdated(): + outdated_items.append(pkg.metadata.name) if not outdated_items: return diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py deleted file mode 100644 index 6e6b1b7dd0..0000000000 --- a/platformio/managers/lib.py +++ /dev/null @@ -1,374 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=too-many-arguments, too-many-locals, too-many-branches -# pylint: disable=too-many-return-statements - -import json -from glob import glob -from os.path import isdir, join - -import click -import semantic_version - -from platformio import app, exception, util -from platformio.compat import glob_escape -from platformio.managers.package import BasePkgManager -from platformio.managers.platform import PlatformFactory, PlatformManager -from platformio.package.exception import ManifestException -from platformio.package.manifest.parser import ManifestParserFactory -from platformio.project.config import ProjectConfig - - -class LibraryManager(BasePkgManager): - - FILE_CACHE_VALID = "30d" # 1 month - - def __init__(self, package_dir=None): - self.config = ProjectConfig.get_instance() - super(LibraryManager, self).__init__( - package_dir or self.config.get_optional_dir("globallib") - ) - - @property - def manifest_names(self): - return [".library.json", "library.json", "library.properties", "module.json"] - - def get_manifest_path(self, pkg_dir): - path = BasePkgManager.get_manifest_path(self, pkg_dir) - if path: - return path - - # if library without manifest, returns first source file - src_dir = join(glob_escape(pkg_dir)) - if isdir(join(pkg_dir, "src")): - src_dir = join(src_dir, "src") - chs_files = glob(join(src_dir, "*.[chS]")) - if chs_files: - return chs_files[0] - cpp_files = glob(join(src_dir, "*.cpp")) - if cpp_files: - return cpp_files[0] - - return None - - def max_satisfying_repo_version(self, versions, requirements=None): - def _cmp_dates(datestr1, datestr2): - date1 = util.parse_date(datestr1) - date2 = util.parse_date(datestr2) - if date1 == date2: - return 0 - return -1 if date1 < date2 else 1 - - semver_spec = None - try: - semver_spec = ( - semantic_version.SimpleSpec(requirements) if requirements else None - ) - except ValueError: - pass - - item = {} - - for v in versions: - semver_new = self.parse_semver_version(v["name"]) - if semver_spec: - # pylint: disable=unsupported-membership-test - if not semver_new or semver_new not in semver_spec: - continue - if not item or self.parse_semver_version(item["name"]) < semver_new: - item = v - elif requirements: - if requirements == v["name"]: - return v - - else: - if not item or _cmp_dates(item["released"], v["released"]) == -1: - item = v - return item - - def get_latest_repo_version(self, name, requirements, silent=False): - item = self.max_satisfying_repo_version( - util.get_api_result( - "/lib/info/%d" - % self.search_lib_id( - {"name": name, "requirements": requirements}, silent=silent - ), - cache_valid="1h", - )["versions"], - requirements, - ) - return item["name"] if item else None - - def _install_from_piorepo(self, name, requirements): - assert name.startswith("id="), name - version = self.get_latest_repo_version(name, requirements) - if not version: - raise exception.UndefinedPackageVersion( - requirements or "latest", util.get_systype() - ) - dl_data = util.get_api_result( - "/lib/download/" + str(name[3:]), dict(version=version), cache_valid="30d" - ) - assert dl_data - - return self._install_from_url( - name, - dl_data["url"].replace("http://", "https://") - if app.get_setting("strict_ssl") - else dl_data["url"], - requirements, - ) - - def search_lib_id( # pylint: disable=too-many-branches - self, filters, silent=False, interactive=False - ): - assert isinstance(filters, dict) - assert "name" in filters - - # try to find ID within installed packages - lib_id = self._get_lib_id_from_installed(filters) - if lib_id: - return lib_id - - # looking in PIO Library Registry - if not silent: - click.echo( - "Looking for %s library in registry" - % click.style(filters["name"], fg="cyan") - ) - query = [] - for key in filters: - if key not in ("name", "authors", "frameworks", "platforms"): - continue - values = filters[key] - if not isinstance(values, list): - values = [v.strip() for v in values.split(",") if v] - for value in values: - query.append( - '%s:"%s"' % (key[:-1] if key.endswith("s") else key, value) - ) - - lib_info = None - result = util.get_api_result( - "/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h" - ) - if result["total"] == 1: - lib_info = result["items"][0] - elif result["total"] > 1: - if silent and not interactive: - lib_info = result["items"][0] - else: - click.secho( - "Conflict: More than one library has been found " - "by request %s:" % json.dumps(filters), - fg="yellow", - err=True, - ) - # pylint: disable=import-outside-toplevel - from platformio.commands.lib import print_lib_item - - for item in result["items"]: - print_lib_item(item) - - if not interactive: - click.secho( - "Automatically chose the first available library " - "(use `--interactive` option to make a choice)", - fg="yellow", - err=True, - ) - lib_info = result["items"][0] - else: - deplib_id = click.prompt( - "Please choose library ID", - type=click.Choice([str(i["id"]) for i in result["items"]]), - ) - for item in result["items"]: - if item["id"] == int(deplib_id): - lib_info = item - break - - if not lib_info: - if list(filters) == ["name"]: - raise exception.LibNotFound(filters["name"]) - raise exception.LibNotFound(str(filters)) - if not silent: - click.echo( - "Found: %s" - % click.style( - "https://platformio.org/lib/show/{id}/{name}".format(**lib_info), - fg="blue", - ) - ) - return int(lib_info["id"]) - - def _get_lib_id_from_installed(self, filters): - if filters["name"].startswith("id="): - return int(filters["name"][3:]) - package_dir = self.get_package_dir( - filters["name"], filters.get("requirements", filters.get("version")) - ) - if not package_dir: - return None - manifest = self.load_manifest(package_dir) - if "id" not in manifest: - return None - - for key in ("frameworks", "platforms"): - if key not in filters: - continue - if key not in manifest: - return None - if not util.items_in_list( - util.items_to_list(filters[key]), util.items_to_list(manifest[key]) - ): - return None - - if "authors" in filters: - if "authors" not in manifest: - return None - manifest_authors = manifest["authors"] - if not isinstance(manifest_authors, list): - manifest_authors = [manifest_authors] - manifest_authors = [ - a["name"] - for a in manifest_authors - if isinstance(a, dict) and "name" in a - ] - filter_authors = filters["authors"] - if not isinstance(filter_authors, list): - filter_authors = [filter_authors] - if not set(filter_authors) <= set(manifest_authors): - return None - - return int(manifest["id"]) - - def install( # pylint: disable=arguments-differ - self, - name, - requirements=None, - silent=False, - after_update=False, - interactive=False, - force=False, - ): - _name, _requirements, _url = self.parse_pkg_uri(name, requirements) - if not _url: - name = "id=%d" % self.search_lib_id( - {"name": _name, "requirements": _requirements}, - silent=silent, - interactive=interactive, - ) - requirements = _requirements - pkg_dir = BasePkgManager.install( - self, - name, - requirements, - silent=silent, - after_update=after_update, - force=force, - ) - - if not pkg_dir: - return None - - manifest = None - try: - manifest = ManifestParserFactory.new_from_dir(pkg_dir).as_dict() - except ManifestException: - pass - if not manifest or not manifest.get("dependencies"): - return pkg_dir - - if not silent: - click.secho("Installing dependencies", fg="yellow") - - builtin_lib_storages = None - for filters in manifest["dependencies"]: - assert "name" in filters - - # avoid circle dependencies - if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = [] - history_key = str(filters) - if history_key in self.INSTALL_HISTORY: - continue - self.INSTALL_HISTORY.append(history_key) - - if any(s in filters.get("version", "") for s in ("\\", "/")): - self.install( - "{name}={version}".format(**filters), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force, - ) - else: - try: - lib_id = self.search_lib_id(filters, silent, interactive) - except exception.LibNotFound as e: - if builtin_lib_storages is None: - builtin_lib_storages = get_builtin_libs() - if not silent or is_builtin_lib( - builtin_lib_storages, filters["name"] - ): - click.secho("Warning! %s" % e, fg="yellow") - continue - - if filters.get("version"): - self.install( - lib_id, - filters.get("version"), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force, - ) - else: - self.install( - lib_id, - silent=silent, - after_update=after_update, - interactive=interactive, - force=force, - ) - return pkg_dir - - -def get_builtin_libs(storage_names=None): - items = [] - storage_names = storage_names or [] - pm = PlatformManager() - for manifest in pm.get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) - for storage in p.get_lib_storages(): - if storage_names and storage["name"] not in storage_names: - continue - lm = LibraryManager(storage["path"]) - items.append( - { - "name": storage["name"], - "path": storage["path"], - "items": lm.get_installed(), - } - ) - return items - - -def is_builtin_lib(storages, name): - for storage in storages or []: - if any(l.get("name") == name for l in storage["items"]): - return True - return False diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 346cce5954..071d67886d 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -482,7 +482,7 @@ def _install_from_url(self, name, url, requirements=None, sha1=None, track=False self.unpack(dlpath, tmp_dir) os.remove(dlpath) else: - vcs = VCSClientFactory.newClient(tmp_dir, url) + vcs = VCSClientFactory.new(tmp_dir, url) assert vcs.export() src_manifest_dir = vcs.storage_dir src_manifest["version"] = vcs.get_current_revision() @@ -628,9 +628,7 @@ def outdated(self, pkg_dir, requirements=None): if "__src_url" in manifest: try: - vcs = VCSClientFactory.newClient( - pkg_dir, manifest["__src_url"], silent=True - ) + vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"], silent=True) except (AttributeError, exception.PlatformioException): return None if not vcs.can_be_updated: @@ -800,7 +798,7 @@ def update(self, package, requirements=None, only_check=False): return True if "__src_url" in manifest: - vcs = VCSClientFactory.newClient(pkg_dir, manifest["__src_url"]) + vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"]) assert vcs.update() self._update_src_manifest( dict(version=vcs.get_current_revision()), vcs.storage_dir diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index ea409f2ecf..cb565f108c 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -20,7 +20,7 @@ import click from platformio import app, compat, fs, util -from platformio.package.exception import PackageException +from platformio.package.exception import MissingPackageManifestError, PackageException from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -83,7 +83,7 @@ def _install(self, spec, search_filters=None, silent=False, force=False): msg = "Installing %s" % click.style(spec.humanize(), fg="cyan") self.print_message(msg) - if spec.url: + if spec.external: pkg = self.install_from_url(spec.url, spec, silent=silent) else: pkg = self.install_from_registry(spec, search_filters, silent=silent) @@ -152,7 +152,7 @@ def install_from_url(self, url, spec, checksum=None, silent=False): assert os.path.isfile(dl_path) self.unpack(dl_path, tmp_dir) else: - vcs = VCSClientFactory.newClient(tmp_dir, url) + vcs = VCSClientFactory.new(tmp_dir, url) assert vcs.export() root_dir = self.find_pkg_root(tmp_dir, spec) @@ -189,12 +189,20 @@ def _install_tmp_pkg(self, tmp_pkg): # what to do with existing package? action = "overwrite" - if dst_pkg.metadata and dst_pkg.metadata.spec.url: + if tmp_pkg.metadata.spec.has_custom_name(): + action = "overwrite" + dst_pkg = PackageSourceItem( + os.path.join(self.package_dir, tmp_pkg.metadata.spec.name) + ) + elif dst_pkg.metadata and dst_pkg.metadata.spec.external: if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: action = "detach-existing" - elif tmp_pkg.metadata.spec.url: + elif tmp_pkg.metadata.spec.external: action = "detach-new" - elif dst_pkg.metadata and dst_pkg.metadata.version != tmp_pkg.metadata.version: + elif dst_pkg.metadata and ( + dst_pkg.metadata.version != tmp_pkg.metadata.version + or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner + ): action = ( "detach-existing" if tmp_pkg.metadata.version > dst_pkg.metadata.version @@ -231,7 +239,7 @@ def _cleanup_dir(path): tmp_pkg.get_safe_dirname(), tmp_pkg.metadata.version, ) - if tmp_pkg.metadata.spec.url: + if tmp_pkg.metadata.spec.external: target_dirname = "%s@src-%s" % ( tmp_pkg.get_safe_dirname(), hashlib.md5( @@ -247,3 +255,20 @@ def _cleanup_dir(path): _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) return PackageSourceItem(dst_pkg.path) + + def get_installed(self): + result = [] + for name in os.listdir(self.package_dir): + pkg_dir = os.path.join(self.package_dir, name) + if not os.path.isdir(pkg_dir): + continue + pkg = PackageSourceItem(pkg_dir) + if not pkg.metadata: + try: + spec = self.build_legacy_spec(pkg_dir) + pkg.metadata = self.build_metadata(pkg_dir, spec) + except MissingPackageManifestError: + pass + if pkg.metadata: + result.append(pkg) + return result diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py new file mode 100644 index 0000000000..22478eff19 --- /dev/null +++ b/platformio/package/manager/_legacy.py @@ -0,0 +1,57 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import fs +from platformio.package.meta import PackageSourceItem, PackageSpec + + +class PackageManagerLegacyMixin(object): + def build_legacy_spec(self, pkg_dir): + # find src manifest + src_manifest_name = ".piopkgmanager.json" + src_manifest_path = None + for name in os.listdir(pkg_dir): + if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)): + continue + src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name) + break + + if src_manifest_path: + src_manifest = fs.load_json(src_manifest_path) + return PackageSpec( + name=src_manifest.get("name"), + url=src_manifest.get("url"), + requirements=src_manifest.get("requirements"), + ) + + # fall back to a package manifest + manifest = self.load_manifest(pkg_dir) + return PackageSpec(name=manifest.get("name")) + + def legacy_load_manifest(self, pkg): + assert isinstance(pkg, PackageSourceItem) + manifest = self.load_manifest(pkg) + manifest["__pkg_dir"] = pkg.path + for key in ("name", "version"): + if not manifest.get(key): + manifest[key] = str(getattr(pkg.metadata, key)) + if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external: + manifest["__src_url"] = pkg.metadata.spec.url + manifest["version"] = str(pkg.metadata.version) + return manifest + + def legacy_get_installed(self): + return [self.legacy_load_manifest(pkg) for pkg in self.get_installed()] diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index d5c9ddadce..41ca58b376 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -79,10 +79,10 @@ def get_http_client(self): class PackageManageRegistryMixin(object): def install_from_registry(self, spec, search_filters=None, silent=False): if spec.owner and spec.name and not search_filters: - package = self.fetch_registry_package(spec.owner, spec.name) + package = self.fetch_registry_package(spec) if not package: raise UnknownPackageError(spec.humanize()) - version = self._pick_best_pkg_version(package["versions"], spec) + version = self.pick_best_registry_version(package["versions"], spec) else: packages = self.search_registry_packages(spec, search_filters) if not packages: @@ -131,10 +131,33 @@ def search_registry_packages(self, spec, filters=None): "items" ] - def fetch_registry_package(self, owner, name): - return self.get_registry_client_instance().get_package( - self.pkg_type, owner, name - ) + def fetch_registry_package(self, spec): + result = None + if spec.owner and spec.name: + result = self.get_registry_client_instance().get_package( + self.pkg_type, spec.owner, spec.name + ) + if not result and (spec.id or (spec.name and not spec.owner)): + packages = self.search_registry_packages(spec) + if packages: + result = self.get_registry_client_instance().get_package( + self.pkg_type, packages[0]["owner"]["username"], packages[0]["name"] + ) + if not result: + raise UnknownPackageError(spec.humanize()) + return result + + def reveal_registry_package_id(self, spec, silent=False): + spec = self.ensure_spec(spec) + if spec.id: + return spec.id + packages = self.search_registry_packages(spec) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1 and not silent: + self.print_multi_package_issue(packages, spec) + click.echo("") + return packages[0]["id"] @staticmethod def print_multi_package_issue(packages, spec): @@ -160,7 +183,7 @@ def print_multi_package_issue(packages, spec): def find_best_registry_version(self, packages, spec): # find compatible version within the latest package versions for package in packages: - version = self._pick_best_pkg_version([package["version"]], spec) + version = self.pick_best_registry_version([package["version"]], spec) if version: return (package, version) @@ -169,9 +192,13 @@ def find_best_registry_version(self, packages, spec): # if the custom version requirements, check ALL package versions for package in packages: - version = self._pick_best_pkg_version( + version = self.pick_best_registry_version( self.fetch_registry_package( - package["owner"]["username"], package["name"] + PackageSpec( + id=package["id"], + owner=package["owner"]["username"], + name=package["name"], + ) ).get("versions"), spec, ) @@ -180,11 +207,12 @@ def find_best_registry_version(self, packages, spec): time.sleep(1) return None - def _pick_best_pkg_version(self, versions, spec): + def pick_best_registry_version(self, versions, spec=None): + assert not spec or isinstance(spec, PackageSpec) best = None for version in versions: semver = PackageMetaData.to_semver(version["name"]) - if spec.requirements and semver not in spec.requirements: + if spec and spec.requirements and semver not in spec.requirements: continue if not any( self.is_system_compatible(f.get("system")) for f in version["files"] diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index e754eab2a0..813ada6de3 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -31,10 +31,7 @@ def uninstall(self, pkg, silent=False, skip_dependencies=False): self.unlock() def _uninstall(self, pkg, silent=False, skip_dependencies=False): - if not isinstance(pkg, PackageSourceItem): - pkg = ( - PackageSourceItem(pkg) if os.path.isdir(pkg) else self.get_package(pkg) - ) + pkg = self.get_package(pkg) if not pkg or not pkg.metadata: raise UnknownPackageError(pkg) @@ -73,7 +70,7 @@ def _uninstall(self, pkg, silent=False, skip_dependencies=False): if not silent: click.echo("[%s]" % click.style("OK", fg="green")) - return True + return pkg def _uninstall_dependencies(self, pkg, silent=False): assert isinstance(pkg, PackageSourceItem) diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py new file mode 100644 index 0000000000..87d5e7f4b1 --- /dev/null +++ b/platformio/package/manager/_update.py @@ -0,0 +1,166 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click + +from platformio import util +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import ( + PackageOutdatedResult, + PackageSourceItem, + PackageSpec, +) +from platformio.package.vcsclient import VCSBaseException, VCSClientFactory + + +class PackageManagerUpdateMixin(object): + def outdated(self, pkg, spec=None): + assert isinstance(pkg, PackageSourceItem) + assert not spec or isinstance(spec, PackageSpec) + assert os.path.isdir(pkg.path) and pkg.metadata + + # skip detached package to a specific version + detached_conditions = [ + "@" in pkg.path, + pkg.metadata.spec and not pkg.metadata.spec.external, + not spec, + ] + if all(detached_conditions): + return PackageOutdatedResult(current=pkg.metadata.version, detached=True) + + latest = None + wanted = None + if pkg.metadata.spec.external: + latest = self._fetch_vcs_latest_version(pkg) + else: + try: + reg_pkg = self.fetch_registry_package(pkg.metadata.spec) + latest = ( + self.pick_best_registry_version(reg_pkg["versions"]) or {} + ).get("name") + if spec: + wanted = ( + self.pick_best_registry_version(reg_pkg["versions"], spec) or {} + ).get("name") + if not wanted: # wrong library + latest = None + except UnknownPackageError: + pass + + return PackageOutdatedResult( + current=pkg.metadata.version, latest=latest, wanted=wanted + ) + + def _fetch_vcs_latest_version(self, pkg): + vcs = None + try: + vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url, silent=True) + except VCSBaseException: + return None + if not vcs.can_be_updated: + return None + return str( + self.build_metadata( + pkg.path, pkg.metadata.spec, vcs_revision=vcs.get_latest_revision() + ).version + ) + + def update(self, pkg, spec=None, only_check=False, silent=False): + pkg = self.get_package(pkg) + if not pkg or not pkg.metadata: + raise UnknownPackageError(pkg) + + if not silent: + click.echo( + "{} {:<45} {:<30}".format( + "Checking" if only_check else "Updating", + click.style(pkg.metadata.spec.humanize(), fg="cyan"), + "%s (%s)" % (pkg.metadata.version, spec.requirements) + if spec and spec.requirements + else str(pkg.metadata.version), + ), + nl=False, + ) + if not util.internet_on(): + if not silent: + click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) + return pkg + + outdated = self.outdated(pkg, spec) + if not silent: + self.print_outdated_state(outdated) + + up_to_date = any( + [ + outdated.detached, + not outdated.latest, + outdated.latest and outdated.current == outdated.latest, + outdated.wanted and outdated.current == outdated.wanted, + ] + ) + if only_check or up_to_date: + return pkg + + try: + self.lock() + return self._update(pkg, outdated, silent=silent) + finally: + self.unlock() + + @staticmethod + def print_outdated_state(outdated): + if outdated.detached: + return click.echo("[%s]" % (click.style("Detached", fg="yellow"))) + if not outdated.latest or outdated.current == outdated.latest: + return click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) + if outdated.wanted and outdated.current == outdated.wanted: + return click.echo( + "[%s]" + % (click.style("Incompatible (%s)" % outdated.latest, fg="yellow")) + ) + return click.echo( + "[%s]" % (click.style(str(outdated.wanted or outdated.latest), fg="red")) + ) + + def _update(self, pkg, outdated, silent=False): + if pkg.metadata.spec.external: + vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url) + assert vcs.update() + pkg.metadata.version = self._fetch_vcs_latest_version(pkg) + pkg.dump_meta() + return pkg + + new_pkg = self.install( + PackageSpec( + id=pkg.metadata.spec.id, + owner=pkg.metadata.spec.owner, + name=pkg.metadata.spec.name, + requirements=outdated.wanted or outdated.latest, + ), + silent=silent, + ) + if new_pkg: + old_pkg = self.get_package( + PackageSpec( + id=pkg.metadata.spec.id, + owner=pkg.metadata.spec.owner, + name=pkg.metadata.name, + requirements=pkg.metadata.version, + ) + ) + if old_pkg: + self.uninstall(old_pkg, silent=silent, skip_dependencies=True) + return new_pkg diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index ca06583350..ee2928c5cb 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -18,14 +18,17 @@ import click import semantic_version -from platformio import fs, util +from platformio import util from platformio.commands import PlatformioCLI +from platformio.compat import ci_strings_are_equal from platformio.package.exception import ManifestException, MissingPackageManifestError from platformio.package.lockfile import LockFile from platformio.package.manager._download import PackageManagerDownloadMixin from platformio.package.manager._install import PackageManagerInstallMixin +from platformio.package.manager._legacy import PackageManagerLegacyMixin from platformio.package.manager._registry import PackageManageRegistryMixin from platformio.package.manager._uninstall import PackageManagerUninstallMixin +from platformio.package.manager._update import PackageManagerUpdateMixin from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import ( PackageMetaData, @@ -41,6 +44,8 @@ class BasePackageManager( # pylint: disable=too-many-public-methods PackageManageRegistryMixin, PackageManagerInstallMixin, PackageManagerUninstallMixin, + PackageManagerUpdateMixin, + PackageManagerLegacyMixin, ): _MEMORY_CACHE = {} @@ -83,10 +88,6 @@ def is_system_compatible(value): return True return util.items_in_list(value, util.get_systype()) - @staticmethod - def generate_rand_version(): - return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S") - @staticmethod def ensure_dir_exists(path): if not os.path.isdir(path): @@ -162,27 +163,9 @@ def load_manifest(self, src): click.secho(str(e), fg="yellow") raise MissingPackageManifestError(", ".join(self.manifest_names)) - def build_legacy_spec(self, pkg_dir): - # find src manifest - src_manifest_name = ".piopkgmanager.json" - src_manifest_path = None - for name in os.listdir(pkg_dir): - if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)): - continue - src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name) - break - - if src_manifest_path: - src_manifest = fs.load_json(src_manifest_path) - return PackageSpec( - name=src_manifest.get("name"), - url=src_manifest.get("url"), - requirements=src_manifest.get("requirements"), - ) - - # fall back to a package manifest - manifest = self.load_manifest(pkg_dir) - return PackageSpec(name=manifest.get("name")) + @staticmethod + def generate_rand_version(): + return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S") def build_metadata(self, pkg_dir, spec, vcs_revision=None): manifest = self.load_manifest(pkg_dir) @@ -192,7 +175,7 @@ def build_metadata(self, pkg_dir, spec, vcs_revision=None): version=manifest.get("version"), spec=spec, ) - if not metadata.name or spec.is_custom_name(): + if not metadata.name or spec.has_custom_name(): metadata.name = spec.name if vcs_revision: metadata.version = "%s+sha.%s" % ( @@ -203,42 +186,27 @@ def build_metadata(self, pkg_dir, spec, vcs_revision=None): metadata.version = self.generate_rand_version() return metadata - def get_installed(self): - result = [] - for name in os.listdir(self.package_dir): - pkg_dir = os.path.join(self.package_dir, name) - if not os.path.isdir(pkg_dir): - continue - pkg = PackageSourceItem(pkg_dir) - if not pkg.metadata: - try: - spec = self.build_legacy_spec(pkg_dir) - pkg.metadata = self.build_metadata(pkg_dir, spec) - except MissingPackageManifestError: - pass - if pkg.metadata: - result.append(pkg) - return result - def get_package(self, spec): - def _ci_strings_are_equal(a, b): - if a == b: - return True - if not a or not b: - return False - return a.strip().lower() == b.strip().lower() + if isinstance(spec, PackageSourceItem): + return spec + + if not isinstance(spec, PackageSpec) and os.path.isdir(spec): + for pkg in self.get_installed(): + if spec == pkg.path: + return pkg + return None spec = self.ensure_spec(spec) best = None for pkg in self.get_installed(): skip_conditions = [ spec.owner - and not _ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner), - spec.url and spec.url != pkg.metadata.spec.url, + and not ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner), + spec.external and spec.url != pkg.metadata.spec.url, spec.id and spec.id != pkg.metadata.spec.id, not spec.id - and not spec.url - and not _ci_strings_are_equal(spec.name, pkg.metadata.name), + and not spec.external + and not ci_strings_are_equal(spec.name, pkg.metadata.name), ] if any(skip_conditions): continue diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 9fe924b9e7..1375e84ee2 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -21,7 +21,7 @@ from platformio.project.helpers import get_project_global_lib_dir -class LibraryPackageManager(BasePackageManager): +class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): super(LibraryPackageManager, self).__init__( PackageType.LIBRARY, package_dir or get_project_global_lib_dir() diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 627bad4734..c79e7d1084 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -17,7 +17,7 @@ from platformio.project.config import ProjectConfig -class PlatformPackageManager(BasePackageManager): +class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): self.config = ProjectConfig.get_instance() super(PlatformPackageManager, self).__init__( diff --git a/platformio/package/manager/tool.py b/platformio/package/manager/tool.py index db660303a0..ae11179888 100644 --- a/platformio/package/manager/tool.py +++ b/platformio/package/manager/tool.py @@ -17,7 +17,7 @@ from platformio.project.config import ProjectConfig -class ToolPackageManager(BasePackageManager): +class ToolPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): self.config = ProjectConfig.get_instance() super(ToolPackageManager, self).__init__( diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index e19e6f259a..b886fb5a75 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -250,7 +250,7 @@ def validate_license(self, value): def load_spdx_licenses(): r = requests.get( "https://raw.githubusercontent.com/spdx/license-list-data" - "/v3.9/json/licenses.json" + "/v3.10/json/licenses.json" ) r.raise_for_status() return r.json() diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 6cd2904b3f..af1e0baaea 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -65,7 +65,44 @@ def from_archive(cls, path): return None -class PackageSpec(object): +class PackageOutdatedResult(object): + def __init__(self, current, latest=None, wanted=None, detached=False): + self.current = current + self.latest = latest + self.wanted = wanted + self.detached = detached + + def __repr__(self): + return ( + "PackageOutdatedResult ".format( + current=self.current, + latest=self.latest, + wanted=self.wanted, + detached=self.detached, + ) + ) + + def __setattr__(self, name, value): + if ( + value + and name in ("current", "latest", "wanted") + and not isinstance(value, semantic_version.Version) + ): + value = semantic_version.Version(str(value)) + return super(PackageOutdatedResult, self).__setattr__(name, value) + + def is_outdated(self, allow_incompatible=False): + if self.detached or not self.latest or self.current == self.latest: + return False + if allow_incompatible: + return self.current != self.latest + if self.wanted: + return self.current != self.wanted + return True + + +class PackageSpec(object): # pylint: disable=too-many-instance-attributes def __init__( # pylint: disable=redefined-builtin,too-many-arguments self, raw=None, owner=None, id=None, name=None, requirements=None, url=None ): @@ -74,6 +111,7 @@ def __init__( # pylint: disable=redefined-builtin,too-many-arguments self.name = name self._requirements = None self.url = url + self.raw = raw if requirements: self.requirements = requirements self._name_is_custom = False @@ -104,6 +142,10 @@ def __repr__(self): "requirements={requirements} url={url}>".format(**self.as_dict()) ) + @property + def external(self): + return bool(self.url) + @property def requirements(self): return self._requirements @@ -116,24 +158,24 @@ def requirements(self, value): self._requirements = ( value if isinstance(value, semantic_version.SimpleSpec) - else semantic_version.SimpleSpec(value) + else semantic_version.SimpleSpec(str(value)) ) def humanize(self): + result = "" if self.url: result = self.url - elif self.id: - result = "id:%d" % self.id - else: - result = "" + elif self.name: if self.owner: result = self.owner + "/" result += self.name + elif self.id: + result = "id:%d" % self.id if self.requirements: result += " @ " + str(self.requirements) return result - def is_custom_name(self): + def has_custom_name(self): return self._name_is_custom def as_dict(self): @@ -145,6 +187,19 @@ def as_dict(self): url=self.url, ) + def as_dependency(self): + if self.url: + return self.raw or self.url + result = "" + if self.name: + result = "%s/%s" % (self.owner, self.name) if self.owner else self.name + elif self.id: + result = str(self.id) + assert result + if self.requirements: + result = "%s@%s" % (result, self.requirements) + return result + def _parse(self, raw): if raw is None: return diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index 5629196628..2e9bb23889 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -17,7 +17,11 @@ from subprocess import CalledProcessError, check_call from sys import modules -from platformio.exception import PlatformioException, UserSideException +from platformio.package.exception import ( + PackageException, + PlatformioException, + UserSideException, +) from platformio.proc import exec_command try: @@ -26,9 +30,13 @@ from urlparse import urlparse +class VCSBaseException(PackageException): + pass + + class VCSClientFactory(object): @staticmethod - def newClient(src_dir, remote_url, silent=False): + def new(src_dir, remote_url, silent=False): result = urlparse(remote_url) type_ = result.scheme tag = None @@ -41,12 +49,15 @@ def newClient(src_dir, remote_url, silent=False): if "#" in remote_url: remote_url, tag = remote_url.rsplit("#", 1) if not type_: - raise PlatformioException("VCS: Unknown repository type %s" % remote_url) - obj = getattr(modules[__name__], "%sClient" % type_.title())( - src_dir, remote_url, tag, silent - ) - assert isinstance(obj, VCSClientBase) - return obj + raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) + try: + obj = getattr(modules[__name__], "%sClient" % type_.title())( + src_dir, remote_url, tag, silent + ) + assert isinstance(obj, VCSClientBase) + return obj + except (AttributeError, AssertionError): + raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) class VCSClientBase(object): @@ -101,7 +112,7 @@ def run_cmd(self, args, **kwargs): check_call(args, **kwargs) return True except CalledProcessError as e: - raise PlatformioException("VCS: Could not process command %s" % e.cmd) + raise VCSBaseException("VCS: Could not process command %s" % e.cmd) def get_cmd_output(self, args, **kwargs): args = [self.command] + args @@ -110,7 +121,7 @@ def get_cmd_output(self, args, **kwargs): result = exec_command(args, **kwargs) if result["returncode"] == 0: return result["out"].strip() - raise PlatformioException( + raise VCSBaseException( "VCS: Could not receive an output from `%s` command (%s)" % (args, result) ) @@ -227,7 +238,6 @@ def export(self): return self.run_cmd(args) def update(self): - args = ["update"] return self.run_cmd(args) @@ -239,4 +249,4 @@ def get_current_revision(self): line = line.strip() if line.startswith("Revision:"): return line.split(":", 1)[1].strip() - raise PlatformioException("Could not detect current SVN revision") + raise VCSBaseException("Could not detect current SVN revision") diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index f3308a6a46..0ea22dd6a2 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -15,7 +15,7 @@ from os.path import isfile, join from platformio.commands.ci import cli as cmd_ci -from platformio.commands.lib import cli as cmd_lib +from platformio.commands.lib.command import cli as cmd_lib def test_ci_empty(clirunner): diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index f51b9dc250..1880d67163 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -13,332 +13,184 @@ # limitations under the License. import json -import re - -from platformio import exception -from platformio.commands import PlatformioCLI -from platformio.commands.lib import cli as cmd_lib - -PlatformioCLI.leftover_args = ["--json-output"] # hook for click - - -def test_search(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) - validate_cliresult(result) - match = re.search(r"Found\s+(\d+)\slibraries:", result.output) - assert int(match.group(1)) > 2 - - result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) +import os + +import semantic_version + +from platformio.clients.registry import RegistryClient +from platformio.commands.lib.command import cli as cmd_lib +from platformio.package.meta import PackageType +from platformio.package.vcsclient import VCSClientFactory +from platformio.project.config import ProjectConfig + + +def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): + regclient = RegistryClient() + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write( + """ +[env] +lib_deps = ArduinoJson + +[env:one] +board = devkit + +[env:two] +framework = foo +lib_deps = + CustomLib + ArduinoJson @ 5.10.1 +""" + ) + result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "install", "64"]) validate_cliresult(result) - match = re.search(r"Found\s+(\d+)\slibraries:", result.output) - assert int(match.group(1)) > 1 - + aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson") + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert config.get("env:one", "lib_deps") == [ + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"] + ] + assert config.get("env:two", "lib_deps") == [ + "CustomLib", + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + ] -def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core): + # ensure "build" version without NPM spec result = clirunner.invoke( cmd_lib, - [ - "-g", - "install", - "64", - "ArduinoJson@~5.10.0", - "547@2.2.4", - "AsyncMqttClient@<=0.8.2", - "Adafruit PN532@1.2.0", - ], + ["-d", str(project_dir), "-e", "one", "install", "mbed-sam-grove/LinkedList"], ) validate_cliresult(result) - - # install unknown library - result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"]) - assert result.exit_code != 0 - assert isinstance(result.exception, exception.LibNotFound) - - items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] - items2 = [ - "ArduinoJson", - "ArduinoJson@5.10.1", - "NeoPixelBus", - "AsyncMqttClient", - "ESPAsyncTCP", - "AsyncTCP", - "Adafruit PN532", - "Adafruit BusIO", + ll_pkg_data = regclient.get_package( + PackageType.LIBRARY, "mbed-sam-grove", "LinkedList" + ) + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert config.get("env:one", "lib_deps") == [ + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], ] - assert set(items1) == set(items2) - -def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core): + # check external package via Git repo result = clirunner.invoke( cmd_lib, [ - "-g", + "-d", + str(project_dir), + "-e", + "one", "install", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", - "SomeLib=http://dl.platformio.org/libraries/archives/0/9540.tar.gz", - "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", ], ) validate_cliresult(result) - - # incorrect requirements - result = clirunner.invoke( - cmd_lib, - [ - "-g", - "install", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3", - ], + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert len(config.get("env:one", "lib_deps")) == 3 + assert config.get("env:one", "lib_deps")[2] == ( + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3" ) - assert result.exit_code != 0 - - items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] - items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire", "ESP32WebServer"] - assert set(items1) >= set(items2) - -def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core): + # test uninstalling result = clirunner.invoke( - cmd_lib, - [ - "-g", - "install", - "https://github.com/gioblu/PJON.git#3.0", - "https://github.com/gioblu/PJON.git#6.2", - "https://github.com/bblanchon/ArduinoJson.git", - "https://gitlab.com/ivankravets/rs485-nodeproto.git", - "https://github.com/platformio/platformio-libmirror.git", - # "https://developer.mbed.org/users/simon/code/TextLCD/", - "knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163", - ], + cmd_lib, ["-d", str(project_dir), "uninstall", "ArduinoJson"] ) validate_cliresult(result) - items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] - items2 = [ - "PJON", - "PJON@src-79de467ebe19de18287becff0a1fb42d", - "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "rs485-nodeproto", - "platformio-libmirror", - "PubSubClient", + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert len(config.get("env:one", "lib_deps")) == 2 + assert len(config.get("env:two", "lib_deps")) == 1 + assert config.get("env:one", "lib_deps") == [ + "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", ] - assert set(items1) >= set(items2) - -def test_install_duplicates(clirunner, validate_cliresult, without_internet): - # registry - result = clirunner.invoke( - cmd_lib, - ["-g", "install", "http://dl.platformio.org/libraries/archives/0/9540.tar.gz"], - ) + # test list + result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"]) validate_cliresult(result) - assert "is already installed" in result.output - - # archive - result = clirunner.invoke( - cmd_lib, - [ - "-g", - "install", - "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", - ], + assert "Version: 0.8.3+sha." in result.stdout + assert ( + "Source: git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3" + in result.stdout ) - validate_cliresult(result) - assert "is already installed" in result.output - - # repository result = clirunner.invoke( - cmd_lib, - ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"], + cmd_lib, ["-d", str(project_dir), "list", "--json-output"] ) validate_cliresult(result) - assert "is already installed" in result.output - - -def test_global_lib_list(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["-g", "list"]) - validate_cliresult(result) - assert all( - [ - n in result.output - for n in ( - "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", - "Version: 5.10.1", - "Source: git+https://github.com/gioblu/PJON.git#3.0", - "Version: 1fb26fd", - ) - ] + data = {} + for key, value in json.loads(result.stdout).items(): + data[os.path.basename(key)] = value + ame_lib = next( + item for item in data["one"] if item["name"] == "AsyncMqttClient-esphome" ) - - result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) - assert all( - [ - n in result.output - for n in ( - "__pkg_dir", - '"__src_url": "git+https://gitlab.com/ivankravets/rs485-nodeproto.git"', - '"version": "5.10.1"', - ) - ] + ame_vcs = VCSClientFactory.new(ame_lib["__pkg_dir"], ame_lib["__src_url"]) + assert data["two"] == [] + assert "__pkg_dir" in data["one"][0] + assert ( + ame_lib["__src_url"] + == "git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3" ) - items1 = [i["name"] for i in json.loads(result.output)] - items2 = [ - "ESP32WebServer", - "ArduinoJson", - "ArduinoJson", - "ArduinoJson", - "ArduinoJson", - "AsyncMqttClient", - "AsyncTCP", - "SomeLib", - "ESPAsyncTCP", - "NeoPixelBus", - "OneWire", - "PJON", - "PJON", - "PubSubClient", - "Adafruit PN532", - "Adafruit BusIO", - "platformio-libmirror", - "rs485-nodeproto", - ] - assert sorted(items1) == sorted(items2) + assert ame_lib["version"] == ("0.8.3+sha.%s" % ame_vcs.get_current_revision()) - versions1 = [ - "{name}@{version}".format(**item) for item in json.loads(result.output) - ] - versions2 = [ - "ArduinoJson@5.8.2", - "ArduinoJson@5.10.1", - "AsyncMqttClient@0.8.2", - "NeoPixelBus@2.2.4", - "PJON@07fe9aa", - "PJON@1fb26fd", - "PubSubClient@bef5814", - "Adafruit PN532@1.2.0", - ] - assert set(versions1) >= set(versions2) - -def test_global_lib_update_check(clirunner, validate_cliresult): +def test_update(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("test-updates") result = clirunner.invoke( - cmd_lib, ["-g", "update", "--only-check", "--json-output"] + cmd_lib, + ["-d", str(storage_dir), "install", "ArduinoJson @ 5.10.1", "Blynk @ ~0.5.0"], ) validate_cliresult(result) - output = json.loads(result.output) - assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([l["name"] for l in output]) - - -def test_global_lib_update(clirunner, validate_cliresult): - # update library using package directory result = clirunner.invoke( - cmd_lib, ["-g", "update", "NeoPixelBus", "--only-check", "--json-output"] + cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "--json-output"] ) validate_cliresult(result) - oudated = json.loads(result.output) - assert len(oudated) == 1 - assert "__pkg_dir" in oudated[0] - result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]]) - validate_cliresult(result) - assert "Uninstalling NeoPixelBus @ 2.2.4" in result.output - - # update rest libraries - result = clirunner.invoke(cmd_lib, ["-g", "update"]) - validate_cliresult(result) - assert result.output.count("[Detached]") == 5 - assert result.output.count("[Up-to-date]") == 12 - - # update unknown library - result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) - assert result.exit_code != 0 - assert isinstance(result.exception, exception.UnknownPackage) - - -def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): - # uninstall using package directory - result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) - validate_cliresult(result) - items = json.loads(result.output) - items = sorted(items, key=lambda item: item["__pkg_dir"]) - result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) - validate_cliresult(result) - assert ("Uninstalling %s" % items[0]["name"]) in result.output - - # uninstall the rest libraries + outdated = json.loads(result.stdout) + assert len(outdated) == 2 + # ArduinoJson + assert outdated[0]["version"] == "5.10.1" + assert outdated[0]["versionWanted"] is None + assert semantic_version.Version( + outdated[0]["versionLatest"] + ) > semantic_version.Version("6.16.0") + # Blynk + assert outdated[1]["version"] == "0.5.4" + assert outdated[1]["versionWanted"] is None + assert semantic_version.Version( + outdated[1]["versionLatest"] + ) > semantic_version.Version("0.6.0") + + # check with spec result = clirunner.invoke( cmd_lib, [ - "-g", - "uninstall", - "OneWire", - "https://github.com/bblanchon/ArduinoJson.git", - "ArduinoJson@!=5.6.7", - "Adafruit PN532", + "-d", + str(storage_dir), + "update", + "--dry-run", + "--json-output", + "ArduinoJson @ ^5", ], ) validate_cliresult(result) - - items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] - items2 = [ - "rs485-nodeproto", - "platformio-libmirror", - "PubSubClient", - "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "ESPAsyncTCP", - "ESP32WebServer", - "NeoPixelBus", - "PJON", - "AsyncMqttClient", - "ArduinoJson", - "SomeLib_ID54", - "PJON@src-79de467ebe19de18287becff0a1fb42d", - "AsyncTCP", - ] - assert set(items1) == set(items2) - - # uninstall unknown library - result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"]) - assert result.exit_code != 0 - assert isinstance(result.exception, exception.UnknownPackage) - - -def test_lib_show(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["show", "64"]) - validate_cliresult(result) - assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")]) - result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) - validate_cliresult(result) - assert "OneWire" in result.output - - -def test_lib_builtin(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["builtin"]) + outdated = json.loads(result.stdout) + assert outdated[0]["version"] == "5.10.1" + assert outdated[0]["versionWanted"] == "5.13.4" + assert semantic_version.Version( + outdated[0]["versionLatest"] + ) > semantic_version.Version("6.16.0") + # update with spec + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "update", "--silent", "ArduinoJson @ ^5.10.1"] + ) validate_cliresult(result) - result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"]) + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "list", "--json-output"] + ) validate_cliresult(result) + items = json.loads(result.stdout) + assert len(items) == 2 + assert items[0]["version"] == "5.13.4" + assert items[1]["version"] == "0.5.4" - -def test_lib_stats(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["stats"]) - validate_cliresult(result) - assert all( - [ - s in result.output - for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show") - ] + # Check incompatible + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "ArduinoJson @ ^5"] ) - - result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) validate_cliresult(result) - assert set( - [ - "dlweek", - "added", - "updated", - "topkeywords", - "dlmonth", - "dlday", - "lastkeywords", - ] - ) == set(json.loads(result.output).keys()) + assert "Incompatible" in result.stdout diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py new file mode 100644 index 0000000000..3f0f37255d --- /dev/null +++ b/tests/commands/test_lib_complex.py @@ -0,0 +1,348 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re + +from platformio import exception +from platformio.commands import PlatformioCLI +from platformio.commands.lib.command import cli as cmd_lib +from platformio.package.exception import UnknownPackageError + +PlatformioCLI.leftover_args = ["--json-output"] # hook for click + + +def test_search(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) + validate_cliresult(result) + match = re.search(r"Found\s+(\d+)\slibraries:", result.output) + assert int(match.group(1)) > 2 + + result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) + validate_cliresult(result) + match = re.search(r"Found\s+(\d+)\slibraries:", result.output) + assert int(match.group(1)) > 1 + + +def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "64", + "ArduinoJson@~5.10.0", + "547@2.2.4", + "AsyncMqttClient@<=0.8.2", + "Adafruit PN532@1.2.0", + ], + ) + validate_cliresult(result) + + # install unknown library + result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "ArduinoJson@5.10.1", + "NeoPixelBus", + "AsyncMqttClient", + "ESPAsyncTCP", + "AsyncTCP", + "Adafruit PN532", + "Adafruit BusIO", + ] + assert set(items1) == set(items2) + + +def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", + "SomeLib=https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz", + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + ], + ) + validate_cliresult(result) + + # incorrect requirements + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3", + ], + ) + assert result.exit_code != 0 + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "SomeLib", + "OneWire", + "ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8", + ] + assert set(items1) >= set(items2) + + +def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/gioblu/PJON.git#3.0", + "https://github.com/gioblu/PJON.git#6.2", + "https://github.com/bblanchon/ArduinoJson.git", + "https://github.com/platformio/platformio-libmirror.git", + # "https://developer.mbed.org/users/simon/code/TextLCD/", + "https://github.com/knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163", + ], + ) + validate_cliresult(result) + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "PJON@src-1204e8bbd80de05e54e171b3a07bcc3f", + "PJON@src-79de467ebe19de18287becff0a1fb42d", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377", + "PubSubClient@src-98ec699a461a31615982e5adaaefadda", + ] + assert set(items1) >= set(items2) + + +def test_install_duplicates(clirunner, validate_cliresult, without_internet): + # registry + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz", + ], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + # archive + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + ], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + # repository + result = clirunner.invoke( + cmd_lib, + ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + +def test_global_lib_list(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["-g", "list"]) + validate_cliresult(result) + assert all( + [ + n in result.output + for n in ( + "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + "Version: 5.10.1", + "Source: git+https://github.com/gioblu/PJON.git#3.0", + "Version: 3.0.0+sha.1fb26fd", + ) + ] + ) + + result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) + assert all( + [ + n in result.output + for n in ( + "__pkg_dir", + '"__src_url": "git+https://github.com/gioblu/PJON.git#6.2"', + '"version": "5.10.1"', + ) + ] + ) + items1 = [i["name"] for i in json.loads(result.output)] + items2 = [ + "Adafruit BusIO", + "Adafruit PN532", + "ArduinoJson", + "ArduinoJson", + "ArduinoJson", + "ArduinoJson", + "AsyncMqttClient", + "AsyncTCP", + "DallasTemperature", + "ESP32WebServer", + "ESPAsyncTCP", + "NeoPixelBus", + "OneWire", + "PJON", + "PJON", + "platformio-libmirror", + "PubSubClient", + ] + assert sorted(items1) == sorted(items2) + + versions1 = [ + "{name}@{version}".format(**item) for item in json.loads(result.output) + ] + versions2 = [ + "ArduinoJson@5.8.2", + "ArduinoJson@5.10.1", + "AsyncMqttClient@0.8.2", + "NeoPixelBus@2.2.4", + "PJON@6.2.0+sha.07fe9aa", + "PJON@3.0.0+sha.1fb26fd", + "PubSubClient@2.6.0+sha.bef5814", + "Adafruit PN532@1.2.0", + ] + assert set(versions1) >= set(versions2) + + +def test_global_lib_update_check(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["-g", "update", "--dry-run", "--json-output"]) + validate_cliresult(result) + output = json.loads(result.output) + assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([lib["name"] for lib in output]) + + +def test_global_lib_update(clirunner, validate_cliresult): + # update library using package directory + result = clirunner.invoke( + cmd_lib, ["-g", "update", "NeoPixelBus", "--dry-run", "--json-output"] + ) + validate_cliresult(result) + oudated = json.loads(result.output) + assert len(oudated) == 1 + assert "__pkg_dir" in oudated[0] + result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]]) + validate_cliresult(result) + assert "Removing NeoPixelBus @ 2.2.4" in result.output + + # update rest libraries + result = clirunner.invoke(cmd_lib, ["-g", "update"]) + validate_cliresult(result) + assert result.output.count("[Detached]") == 1 + assert result.output.count("[Up-to-date]") == 15 + + # update unknown library + result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + +def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): + # uninstall using package directory + result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) + validate_cliresult(result) + items = json.loads(result.output) + items = sorted(items, key=lambda item: item["__pkg_dir"]) + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) + validate_cliresult(result) + assert ("Removing %s" % items[0]["name"]) in result.output + + # uninstall the rest libraries + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "uninstall", + "OneWire", + "https://github.com/bblanchon/ArduinoJson.git", + "ArduinoJson@!=5.6.7", + "Adafruit PN532", + ], + ) + validate_cliresult(result) + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "AsyncMqttClient", + "AsyncTCP", + "ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8", + "ESPAsyncTCP", + "NeoPixelBus", + "PJON@src-1204e8bbd80de05e54e171b3a07bcc3f", + "PJON@src-79de467ebe19de18287becff0a1fb42d", + "platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377", + "PubSubClient@src-98ec699a461a31615982e5adaaefadda", + "SomeLib", + ] + assert set(items1) == set(items2) + + # uninstall unknown library + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + +def test_lib_show(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["show", "64"]) + validate_cliresult(result) + assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")]) + result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) + validate_cliresult(result) + assert "OneWire" in result.output + + +def test_lib_builtin(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["builtin"]) + validate_cliresult(result) + result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"]) + validate_cliresult(result) + + +def test_lib_stats(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["stats"]) + validate_cliresult(result) + assert all( + [ + s in result.output + for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show") + ] + ) + + result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) + validate_cliresult(result) + assert set( + [ + "dlweek", + "added", + "updated", + "topkeywords", + "dlmonth", + "dlday", + "lastkeywords", + ] + ) == set(json.loads(result.output).keys()) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 5898ae2b71..131346af77 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -16,6 +16,7 @@ import time import pytest +import semantic_version from platformio import fs, util from platformio.package.exception import ( @@ -201,6 +202,12 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory): assert lm.get_package("OneWire").metadata.version.major >= 2 assert len(lm.get_installed()) == 6 + # test conflicted names + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("conflicted-storage"))) + lm.install("4@2.6.1", silent=True) + lm.install("5357@2.6.1", silent=True) + assert len(lm.get_installed()) == 2 + # Tools tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True) @@ -340,3 +347,81 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) assert lm.uninstall("AsyncMqttClient-esphome", silent=True) assert len(lm.get_installed()) == 0 + + +def test_registry(isolated_pio_core): + lm = LibraryPackageManager() + + # reveal ID + assert lm.reveal_registry_package_id(PackageSpec(id=13)) == 13 + assert lm.reveal_registry_package_id(PackageSpec(name="OneWire"), silent=True) == 1 + with pytest.raises(UnknownPackageError): + lm.reveal_registry_package_id(PackageSpec(name="/non-existing-package/")) + + # fetch package data + assert lm.fetch_registry_package(PackageSpec(id=1))["name"] == "OneWire" + assert lm.fetch_registry_package(PackageSpec(name="ArduinoJson"))["id"] == 64 + assert ( + lm.fetch_registry_package( + PackageSpec(id=13, owner="adafruit", name="Renamed library") + )["name"] + == "Adafruit GFX Library" + ) + with pytest.raises(UnknownPackageError): + lm.fetch_registry_package( + PackageSpec(owner="unknown<>owner", name="/non-existing-package/") + ) + with pytest.raises(UnknownPackageError): + lm.fetch_registry_package(PackageSpec(name="/non-existing-package/")) + + +def test_update_with_metadata(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + pkg = lm.install("ArduinoJson @ 5.10.1", silent=True) + + # tesy latest + outdated = lm.outdated(pkg) + assert str(outdated.current) == "5.10.1" + assert outdated.wanted is None + assert outdated.latest > outdated.current + assert outdated.latest > semantic_version.Version("5.99.99") + + # test wanted + outdated = lm.outdated(pkg, PackageSpec("ArduinoJson@~5")) + assert str(outdated.current) == "5.10.1" + assert str(outdated.wanted) == "5.13.4" + assert outdated.latest > semantic_version.Version("6.16.0") + + # update to the wanted 5.x + new_pkg = lm.update("ArduinoJson@^5", PackageSpec("ArduinoJson@^5"), silent=True) + assert str(new_pkg.metadata.version) == "5.13.4" + # check that old version is removed + assert len(lm.get_installed()) == 1 + + # update to the latest + lm = LibraryPackageManager(str(storage_dir)) + pkg = lm.update("ArduinoJson", silent=True) + assert pkg.metadata.version == outdated.latest + + +def test_update_without_metadata(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + storage_dir.join("legacy-package").mkdir().join("library.json").write( + '{"name": "AsyncMqttClient-esphome", "version": "0.8.2"}' + ) + storage_dir.join("legacy-dep").mkdir().join("library.json").write( + '{"name": "AsyncTCP-esphome", "version": "1.1.1"}' + ) + lm = LibraryPackageManager(str(storage_dir)) + pkg = lm.get_package("AsyncMqttClient-esphome") + outdated = lm.outdated(pkg) + assert len(lm.get_installed()) == 2 + assert str(pkg.metadata.version) == "0.8.2" + assert outdated.latest > semantic_version.Version("0.8.2") + + # update + lm = LibraryPackageManager(str(storage_dir)) + new_pkg = lm.update(pkg, silent=True) + assert len(lm.get_installed()) == 3 + assert new_pkg.metadata.spec.owner == "ottowinter" diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index d9d205c765..d7d4b820e2 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -17,7 +17,27 @@ import jsondiff import semantic_version -from platformio.package.meta import PackageMetaData, PackageSpec, PackageType +from platformio.package.meta import ( + PackageMetaData, + PackageOutdatedResult, + PackageSpec, + PackageType, +) + + +def test_outdated_result(): + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0") + assert result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.5.4") + assert result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.2.3") + assert not result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", detached=True) + assert not result.is_outdated() + assert not result.is_outdated(allow_incompatible=True) def test_spec_owner(): @@ -45,9 +65,16 @@ def test_spec_name(): def test_spec_requirements(): assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") + assert PackageSpec( + name="foo", requirements=semantic_version.Version("1.2.3") + ) == PackageSpec(name="foo", requirements="1.2.3") assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") + assert PackageSpec( + name="hello", requirements=semantic_version.SimpleSpec("~1.2.3") + ) == PackageSpec(name="hello", requirements="~1.2.3") spec = PackageSpec("id=20 @ !=1.2.3,<2.0") + assert not spec.external assert isinstance(spec.requirements, semantic_version.SimpleSpec) assert semantic_version.Version("1.3.0-beta.1") in spec.requirements assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0") @@ -88,7 +115,8 @@ def test_spec_external_urls(): "Custom-Name=" "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" ) - assert spec.is_custom_name() + assert spec.external + assert spec.has_custom_name() assert spec.name == "Custom-Name" assert spec == PackageSpec( url="https://github.com/platformio/platformio-core/archive/develop.tar.gz", @@ -163,6 +191,24 @@ def test_spec_as_dict(): ) +def test_spec_as_dependency(): + assert PackageSpec("owner/pkgname").as_dependency() == "owner/pkgname" + assert PackageSpec(owner="owner", name="pkgname").as_dependency() == "owner/pkgname" + assert PackageSpec("bob/foo @ ^1.2.3").as_dependency() == "bob/foo@^1.2.3" + assert ( + PackageSpec( + "https://github.com/o/r/a/develop.zip?param=value @ !=2" + ).as_dependency() + == "https://github.com/o/r/a/develop.zip?param=value @ !=2" + ) + assert ( + PackageSpec( + "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ).as_dependency() + == "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) + + def test_metadata_as_dict(): metadata = PackageMetaData(PackageType.LIBRARY, "foo", "1.2.3") # test setter diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 34d4ce6865..07fbabf80f 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -88,7 +88,9 @@ def test_check_and_update_libraries(clirunner, isolated_pio_core, validate_clire validate_cliresult(result) assert "There are the new updates for libraries (ArduinoJson)" in result.output assert "Please wait while updating libraries" in result.output - assert re.search(r"Updating ArduinoJson\s+@ 6.12.0\s+\[[\d\.]+\]", result.output) + assert re.search( + r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[[\d\.]+\]", result.output + ) # check updated version result = clirunner.invoke(cli_pio, ["lib", "-g", "list", "--json-output"]) From d5451756fd2f5f1c5979695516c292a11760fffd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Aug 2020 20:09:10 +0300 Subject: [PATCH 145/223] Minor improvements --- platformio/builder/tools/piolib.py | 2 + platformio/builder/tools/pioplatform.py | 2 +- platformio/commands/lib/command.py | 6 +- platformio/package/manager/_download.py | 4 +- platformio/package/manager/_install.py | 40 ++++-------- platformio/package/manager/_registry.py | 13 ++-- platformio/package/manager/_uninstall.py | 12 ++-- platformio/package/manager/_update.py | 12 ++-- platformio/package/manager/base.py | 77 ++++++++++++++++-------- tests/commands/test_lib_complex.py | 1 - 10 files changed, 90 insertions(+), 79 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 24229b1c75..3a7b3aad73 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -1022,6 +1022,8 @@ def _print_deps_tree(root, level=0): pkg = PackageSourceItem(lb.path) if pkg.metadata: title += " %s" % pkg.metadata.version + elif lb.version: + title += " %s" % lb.version click.echo("%s|-- %s" % (margin, title), nl=False) if int(ARGUMENTS.get("PIOVERBOSE", 0)): if pkg.metadata and pkg.metadata.spec.external: diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index e280372b7d..8c047365d5 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -139,7 +139,7 @@ def _get_configuration_data(): ) def _get_plaform_data(): - data = ["PLATFORM: %s %s" % (platform.title, platform.version)] + data = ["PLATFORM: %s (%s)" % (platform.title, platform.version)] if platform.src_version: data.append("#" + platform.src_version) if int(ARGUMENTS.get("PIOVERBOSE", 0)) and platform.src_url: diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 33249f3edf..ec5fd8e668 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -283,12 +283,14 @@ def lib_update( # pylint: disable=too-many-arguments json_result[storage_dir] = result else: for library in _libraries: - spec = ( + to_spec = ( None if isinstance(library, PackageSourceItem) else PackageSpec(library) ) - lm.update(library, spec=spec, only_check=only_check, silent=silent) + lm.update( + library, to_spec=to_spec, only_check=only_check, silent=silent + ) if json_output: return click.echo( diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 83de9f3711..34295287c4 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -17,8 +17,6 @@ import tempfile import time -import click - from platformio import app, compat from platformio.package.download import FileDownloader from platformio.package.lockfile import LockFile @@ -77,7 +75,7 @@ def download(self, url, checksum=None, silent=False): except IOError: raise_error = True if raise_error: - click.secho( + self.print_message( "Error: Please read http://bit.ly/package-manager-ioerror", fg="red", err=True, diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index cb565f108c..c63a504db0 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -20,7 +20,7 @@ import click from platformio import app, compat, fs, util -from platformio.package.exception import MissingPackageManifestError, PackageException +from platformio.package.exception import PackageException from platformio.package.meta import PackageSourceItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -71,7 +71,7 @@ def _install(self, spec, search_filters=None, silent=False, force=False): if pkg: if not silent: - click.secho( + self.print_message( "{name} @ {version} is already installed".format( **pkg.metadata.as_dict() ), @@ -80,8 +80,9 @@ def _install(self, spec, search_filters=None, silent=False, force=False): return pkg if not silent: - msg = "Installing %s" % click.style(spec.humanize(), fg="cyan") - self.print_message(msg) + self.print_message( + "Installing %s" % click.style(spec.humanize(), fg="cyan") + ) if spec.external: pkg = self.install_from_url(spec.url, spec, silent=silent) @@ -96,12 +97,10 @@ def _install(self, spec, search_filters=None, silent=False, force=False): if not silent: self.print_message( - click.style( - "{name} @ {version} has been successfully installed!".format( - **pkg.metadata.as_dict() - ), - fg="green", - ) + "{name} @ {version} has been successfully installed!".format( + **pkg.metadata.as_dict() + ), + fg="green", ) self.memcache_reset() @@ -115,10 +114,10 @@ def _install_dependencies(self, pkg, silent=False): if not manifest.get("dependencies"): return if not silent: - self.print_message(click.style("Installing dependencies...", fg="yellow")) + self.print_message("Installing dependencies...") for dependency in manifest.get("dependencies"): if not self._install_dependency(dependency, silent) and not silent: - click.secho( + self.print_message( "Warning! Could not install dependency %s for package '%s'" % (dependency, pkg.metadata.name), fg="yellow", @@ -255,20 +254,3 @@ def _cleanup_dir(path): _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) return PackageSourceItem(dst_pkg.path) - - def get_installed(self): - result = [] - for name in os.listdir(self.package_dir): - pkg_dir = os.path.join(self.package_dir, name) - if not os.path.isdir(pkg_dir): - continue - pkg = PackageSourceItem(pkg_dir) - if not pkg.metadata: - try: - spec = self.build_legacy_spec(pkg_dir) - pkg.metadata = self.build_metadata(pkg_dir, spec) - except MissingPackageManifestError: - pass - if pkg.metadata: - result.append(pkg) - return result diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 41ca58b376..0d1e45e105 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -108,8 +108,8 @@ def install_from_registry(self, spec, search_filters=None, silent=False): silent=silent, ) except Exception as e: # pylint: disable=broad-except - click.secho("Warning! Package Mirror: %s" % e, fg="yellow") - click.secho("Looking for another mirror...", fg="yellow") + self.print_message("Warning! Package Mirror: %s" % e, fg="yellow") + self.print_message("Looking for another mirror...", fg="yellow") return None @@ -159,9 +159,8 @@ def reveal_registry_package_id(self, spec, silent=False): click.echo("") return packages[0]["id"] - @staticmethod - def print_multi_package_issue(packages, spec): - click.secho( + def print_multi_package_issue(self, packages, spec): + self.print_message( "Warning! More than one package has been found by ", fg="yellow", nl=False ) click.secho(spec.humanize(), fg="cyan", nl=False) @@ -174,9 +173,9 @@ def print_multi_package_issue(packages, spec): version=item["version"]["name"], ) ) - click.secho( + self.print_message( "Please specify detailed REQUIREMENTS using package owner and version " - "(showed above) to avoid project compatibility issues.", + "(showed above) to avoid name conflicts", fg="yellow", ) diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 813ada6de3..603ad38218 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -23,17 +23,17 @@ class PackageManagerUninstallMixin(object): - def uninstall(self, pkg, silent=False, skip_dependencies=False): + def uninstall(self, spec, silent=False, skip_dependencies=False): try: self.lock() - return self._uninstall(pkg, silent, skip_dependencies) + return self._uninstall(spec, silent, skip_dependencies) finally: self.unlock() - def _uninstall(self, pkg, silent=False, skip_dependencies=False): - pkg = self.get_package(pkg) + def _uninstall(self, spec, silent=False, skip_dependencies=False): + pkg = self.get_package(spec) if not pkg or not pkg.metadata: - raise UnknownPackageError(pkg) + raise UnknownPackageError(spec) if not silent: self.print_message( @@ -78,7 +78,7 @@ def _uninstall_dependencies(self, pkg, silent=False): if not manifest.get("dependencies"): return if not silent: - self.print_message(click.style("Removing dependencies...", fg="yellow")) + self.print_message("Removing dependencies...", fg="yellow") for dependency in manifest.get("dependencies"): pkg = self.get_package( PackageSpec( diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index 87d5e7f4b1..d120e030c4 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -78,18 +78,18 @@ def _fetch_vcs_latest_version(self, pkg): ).version ) - def update(self, pkg, spec=None, only_check=False, silent=False): - pkg = self.get_package(pkg) + def update(self, from_spec, to_spec=None, only_check=False, silent=False): + pkg = self.get_package(from_spec) if not pkg or not pkg.metadata: - raise UnknownPackageError(pkg) + raise UnknownPackageError(from_spec) if not silent: click.echo( "{} {:<45} {:<30}".format( "Checking" if only_check else "Updating", click.style(pkg.metadata.spec.humanize(), fg="cyan"), - "%s (%s)" % (pkg.metadata.version, spec.requirements) - if spec and spec.requirements + "%s (%s)" % (pkg.metadata.version, to_spec.requirements) + if to_spec and to_spec.requirements else str(pkg.metadata.version), ), nl=False, @@ -99,7 +99,7 @@ def update(self, pkg, spec=None, only_check=False, silent=False): click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) return pkg - outdated = self.outdated(pkg, spec) + outdated = self.outdated(pkg, to_spec) if not silent: self.print_outdated_state(outdated) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index ee2928c5cb..58c35d47b3 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -103,8 +103,11 @@ def ensure_spec(spec): def manifest_names(self): raise NotImplementedError - def print_message(self, message, nl=True): - click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl) + def print_message(self, message, **kwargs): + click.echo( + "%s: " % str(self.__class__.__name__).replace("Package", " "), nl=False + ) + click.secho(message, **kwargs) def get_download_dir(self): if not self._download_dir: @@ -160,7 +163,7 @@ def load_manifest(self, src): return result except ManifestException as e: if not PlatformioCLI.in_silence(): - click.secho(str(e), fg="yellow") + self.print_message(str(e), fg="yellow") raise MissingPackageManifestError(", ".join(self.manifest_names)) @staticmethod @@ -186,37 +189,63 @@ def build_metadata(self, pkg_dir, spec, vcs_revision=None): metadata.version = self.generate_rand_version() return metadata + def get_installed(self): + result = [] + for name in os.listdir(self.package_dir): + pkg_dir = os.path.join(self.package_dir, name) + if not os.path.isdir(pkg_dir): + continue + pkg = PackageSourceItem(pkg_dir) + if not pkg.metadata: + try: + spec = self.build_legacy_spec(pkg_dir) + pkg.metadata = self.build_metadata(pkg_dir, spec) + except MissingPackageManifestError: + pass + if pkg.metadata: + result.append(pkg) + return result + def get_package(self, spec): if isinstance(spec, PackageSourceItem): return spec - - if not isinstance(spec, PackageSpec) and os.path.isdir(spec): - for pkg in self.get_installed(): - if spec == pkg.path: - return pkg - return None - spec = self.ensure_spec(spec) best = None for pkg in self.get_installed(): - skip_conditions = [ - spec.owner - and not ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner), - spec.external and spec.url != pkg.metadata.spec.url, - spec.id and spec.id != pkg.metadata.spec.id, - not spec.id - and not spec.external - and not ci_strings_are_equal(spec.name, pkg.metadata.name), - ] - if any(skip_conditions): + if not self._test_pkg_with_spec(pkg, spec): continue - if self.pkg_type == PackageType.TOOL: - # TODO: check "system" for pkg - pass - assert isinstance(pkg.metadata.version, semantic_version.Version) if spec.requirements and pkg.metadata.version not in spec.requirements: continue if not best or (pkg.metadata.version > best.metadata.version): best = pkg return best + + def _test_pkg_with_spec(self, pkg, spec): + # "id" mismatch + if spec.id and spec.id != pkg.metadata.spec.id: + return False + + # "owner" mismatch + if spec.owner and not ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner): + return False + + # external "URL" mismatch + if spec.external: + # local folder mismatch + if spec.url == pkg.path or ( + spec.url.startswith("file://") and pkg.path == spec.url[7:] + ): + return True + if spec.url != pkg.metadata.spec.url: + return False + + # "name" mismatch + elif not spec.id and not ci_strings_are_equal(spec.name, pkg.metadata.name): + return False + + if self.pkg_type == PackageType.TOOL: + # TODO: check "system" for pkg + pass + + return True diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index 3f0f37255d..a71330db06 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -15,7 +15,6 @@ import json import re -from platformio import exception from platformio.commands import PlatformioCLI from platformio.commands.lib.command import cli as cmd_lib from platformio.package.exception import UnknownPackageError From 38ec51720039144cc298dd66e8b51ea149d6443d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Aug 2020 21:09:42 +0300 Subject: [PATCH 146/223] Update history --- HISTORY.rst | 26 ++++++++++++++++++++++---- docs | 2 +- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b068fd59b5..147c4b5f4f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,16 +11,34 @@ PlatformIO Core 4 **A professional collaborative platform for embedded development** -* New `Account Management System `__ +* Integration with the new `Account Management System `__ - Manage own organizations - Manage organization teams - Manage resource access -* Registry Package Management +* Integration with the new **PlatformIO Trusted Registry** - - Publish a personal or organization package using `platformio package publish `__ command - - Remove a pushed package from the registry using `platformio package unpublish `__ command + - Enterprise-grade package storage with high availability (multi replicas) + - Secure, fast, and reliable global content delivery network (CDN) + - Universal support for all embedded packages: + + * Libraries + * Development platforms + * Toolchains + + - Built-in fine-grained access control (role based, teams, organizations) + - Command Line Interface: + + * `platformio package publish `__ – publish a personal or organization package + * `platformio package unpublish `__ – remove a pushed package from the registry + * Grant package access to the team members or maintainers + +* New **Package Management System** + + - Integrated PlatformIO Core with the new PlatformIO Trusted Registry + - Strict dependency declaration using owner name (resolves name conflicts) (`issue #1824 `_) + - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) * New `Custom Targets `__ diff --git a/docs b/docs index 13df46f9cf..e8ee370a33 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 13df46f9cf4e3f822f6f642c9c9b3085a0a93193 +Subproject commit e8ee370a338270b453d4d97bd286f537d5f06456 From fd7dba1d746d6a7fb10a04e7bc62e6fde4e5266e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Aug 2020 17:50:44 +0300 Subject: [PATCH 147/223] Package Manifest: increase package author.name field to the 100 chars --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index b886fb5a75..8befab521a 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -84,7 +84,7 @@ def _deserialize( # pylint: disable=arguments-differ class AuthorSchema(StrictSchema): - name = fields.Str(required=True, validate=validate.Length(min=1, max=50)) + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) email = fields.Email(validate=validate.Length(min=1, max=50)) maintainer = fields.Bool(default=False) url = fields.Url(validate=validate.Length(min=1, max=255)) From 64ff6a0ff5e5d7d3d10bee8c3addd0a4395e3a46 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Aug 2020 18:30:04 +0300 Subject: [PATCH 148/223] Switch legacy core package manager to the new --- platformio/__init__.py | 10 + platformio/builder/tools/pioide.py | 2 +- platformio/builder/tools/piomisc.py | 2 +- platformio/commands/check/tools/clangtidy.py | 2 +- platformio/commands/check/tools/cppcheck.py | 2 +- platformio/commands/check/tools/pvsstudio.py | 2 +- platformio/commands/debug/command.py | 2 +- platformio/commands/home/command.py | 2 +- platformio/commands/remote/command.py | 2 +- platformio/commands/update.py | 2 +- platformio/maintenance.py | 24 +- platformio/managers/platform.py | 2 +- platformio/package/manager/_registry.py | 2 + .../{managers => package/manager}/core.py | 109 +++----- platformio/util.py | 2 +- tests/test_managers.py | 234 ------------------ 16 files changed, 83 insertions(+), 318 deletions(-) rename platformio/{managers => package/manager}/core.py (57%) delete mode 100644 tests/test_managers.py diff --git a/platformio/__init__.py b/platformio/__init__.py index 8da7e6e0c2..0a1bc48c77 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -40,3 +40,13 @@ __accounts_api__ = "https://api.accounts.platformio.org" __registry_api__ = "https://api.registry.platformio.org" __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" + +__core_packages__ = { + "contrib-piohome": "~3.2.3", + "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), + "tool-unity": "~1.20500.0", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~3.30102.0", + "tool-cppcheck": "~1.190.0", + "tool-clangtidy": "~1.100000.0", + "tool-pvs-studio": "~7.7.0", +} diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index acb36ae4ed..6a3d343d5c 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -20,7 +20,7 @@ from SCons.Defaults import processDefines # pylint: disable=import-error from platformio.compat import glob_escape -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command, where_is_program diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index aa5158fee0..799b192fbc 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -25,7 +25,7 @@ from platformio import fs, util from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index f16104527f..05be67b4ab 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -17,7 +17,7 @@ from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class ClangtidyCheckTool(CheckToolBase): diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 3412971428..931b16edd9 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -19,7 +19,7 @@ from platformio import proc from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class CppcheckCheckTool(CheckToolBase): diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index 871ec4bc03..ce5d93ec62 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -22,7 +22,7 @@ from platformio import proc, util from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-attributes diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 2528611161..78a43eef09 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -24,7 +24,7 @@ from platformio import app, exception, fs, proc, util from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError -from platformio.managers.core import inject_contrib_pysite +from platformio.package.manager.core import inject_contrib_pysite from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project, load_project_ide_data diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 32d2806335..dd733bb601 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -22,7 +22,7 @@ from platformio import exception from platformio.compat import WINDOWS -from platformio.managers.core import get_core_package_dir, inject_contrib_pysite +from platformio.package.manager.core import get_core_package_dir, inject_contrib_pysite @click.command("home", short_help="PIO Home") diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index f9e24c299c..66c1069051 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -29,7 +29,7 @@ from platformio.commands.run.command import cli as cmd_run from platformio.commands.test.command import cli as cmd_test from platformio.compat import PY2 -from platformio.managers.core import inject_contrib_pysite +from platformio.package.manager.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError diff --git a/platformio/commands/update.py b/platformio/commands/update.py index bf8291655a..b1e15a4345 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -18,7 +18,7 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update -from platformio.managers.core import update_core_packages +from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager diff --git a/platformio/maintenance.py b/platformio/maintenance.py index b0e64f523b..cf2e0698e6 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -25,9 +25,11 @@ from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.managers.core import update_core_packages from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec from platformio.proc import is_container @@ -90,7 +92,8 @@ def __init__(self, from_version, to_version): ) self._upgraders = [ - (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms) + (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms), + (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata), ] def run(self, ctx): @@ -110,6 +113,22 @@ def _update_dev_platforms(ctx): ctx.invoke(cmd_platform_update) return True + @staticmethod + def _update_pkg_metadata(_): + pm = ToolPackageManager() + for pkg in pm.get_installed(): + if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id: + continue + result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name)) + if len(result) != 1: + continue + result = result[0] + pkg.metadata.spec = PackageSpec( + id=result["id"], owner=result["owner"]["username"], name=result["name"], + ) + pkg.dump_meta() + return True + def after_upgrade(ctx): terminal_width, _ = click.get_terminal_size() @@ -160,7 +179,6 @@ def after_upgrade(ctx): ) else: raise exception.UpgradeError("Auto upgrading...") - click.echo("") # PlatformIO banner click.echo("*" * terminal_width) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index ada4f4accd..8548bba63a 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -30,8 +30,8 @@ DebugSupportError, ) from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module -from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager +from platformio.package.manager.core import get_core_package_dir from platformio.project.config import ProjectConfig try: diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 0d1e45e105..7dc09964eb 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -119,6 +119,7 @@ def get_registry_client_instance(self): return self._registry_client def search_registry_packages(self, spec, filters=None): + assert isinstance(spec, PackageSpec) filters = filters or {} if spec.id: filters["ids"] = str(spec.id) @@ -132,6 +133,7 @@ def search_registry_packages(self, spec, filters=None): ] def fetch_registry_package(self, spec): + assert isinstance(spec, PackageSpec) result = None if spec.owner and spec.name: result = self.get_registry_client_instance().get_package( diff --git a/platformio/managers/core.py b/platformio/package/manager/core.py similarity index 57% rename from platformio/managers/core.py rename to platformio/package/manager/core.py index 27bee8c21d..2b872ab6d8 100644 --- a/platformio/managers/core.py +++ b/platformio/package/manager/core.py @@ -17,89 +17,58 @@ import subprocess import sys -from platformio import exception, util +from platformio import __core_packages__, exception, util from platformio.compat import PY2 -from platformio.managers.package import PackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec from platformio.proc import get_pythonexe_path -from platformio.project.config import ProjectConfig - -CORE_PACKAGES = { - "contrib-piohome": "~3.2.3", - "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), - "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", - "tool-cppcheck": "~1.190.0", - "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.7.0", -} - -# pylint: disable=arguments-differ,signature-differs - - -class CorePackageManager(PackageManager): - def __init__(self): - config = ProjectConfig.get_instance() - packages_dir = config.get_optional_dir("packages") - super(CorePackageManager, self).__init__( - packages_dir, - [ - "https://dl.bintray.com/platformio/dl-packages/manifest.json", - "http%s://dl.platformio.org/packages/manifest.json" - % ("" if sys.version_info < (2, 7, 9) else "s"), - ], - ) - - def install( # pylint: disable=keyword-arg-before-vararg - self, name, requirements=None, *args, **kwargs - ): - PackageManager.install(self, name, requirements, *args, **kwargs) - self.cleanup_packages() - return self.get_package_dir(name, requirements) - - def update(self, *args, **kwargs): - result = PackageManager.update(self, *args, **kwargs) - self.cleanup_packages() - return result - - def cleanup_packages(self): - self.cache_reset() - best_pkg_versions = {} - for name, requirements in CORE_PACKAGES.items(): - pkg_dir = self.get_package_dir(name, requirements) - if not pkg_dir: - continue - best_pkg_versions[name] = self.load_manifest(pkg_dir)["version"] - for manifest in self.get_installed(): - if manifest["name"] not in best_pkg_versions: - continue - if manifest["version"] != best_pkg_versions[manifest["name"]]: - self.uninstall(manifest["__pkg_dir"], after_update=True) - self.cache_reset() - return True def get_core_package_dir(name): - if name not in CORE_PACKAGES: + if name not in __core_packages__: raise exception.PlatformioException("Please upgrade PIO Core") - requirements = CORE_PACKAGES[name] - pm = CorePackageManager() - pkg_dir = pm.get_package_dir(name, requirements) - if pkg_dir: - return pkg_dir - return pm.install(name, requirements) + pm = ToolPackageManager() + spec = PackageSpec( + owner="platformio", name=name, requirements=__core_packages__[name] + ) + pkg = pm.get_package(spec) + if pkg: + return pkg.path + pkg = pm.install(spec).path + _remove_unnecessary_packages() + return pkg def update_core_packages(only_check=False, silent=False): - pm = CorePackageManager() - for name, requirements in CORE_PACKAGES.items(): - pkg_dir = pm.get_package_dir(name) - if not pkg_dir: + pm = ToolPackageManager() + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if not pkg: continue - if not silent or pm.outdated(pkg_dir, requirements): - pm.update(name, requirements, only_check=only_check) + if not silent or pm.outdated(pkg, spec).is_outdated(): + pm.update(pkg, spec, only_check=only_check) + if not only_check: + _remove_unnecessary_packages() return True +def _remove_unnecessary_packages(): + pm = ToolPackageManager() + best_pkg_versions = {} + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if not pkg: + continue + best_pkg_versions[pkg.metadata.name] = pkg.metadata.version + for pkg in pm.get_installed(): + if pkg.metadata.name not in best_pkg_versions: + continue + if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]: + pm.uninstall(pkg) + + def inject_contrib_pysite(verify_openssl=False): # pylint: disable=import-outside-toplevel from site import addsitedir diff --git a/platformio/util.py b/platformio/util.py index 362545865f..5b38909d59 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -196,7 +196,7 @@ def get_mdns_services(): import zeroconf except ImportError: from site import addsitedir - from platformio.managers.core import get_core_package_dir + from platformio.package.manager.core import get_core_package_dir contrib_pysite_dir = get_core_package_dir("contrib-pysite") addsitedir(contrib_pysite_dir) diff --git a/tests/test_managers.py b/tests/test_managers.py deleted file mode 100644 index 308523cd1e..0000000000 --- a/tests/test_managers.py +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from os.path import join - -from platformio.managers.package import PackageManager -from platformio.project.helpers import get_project_core_dir - - -def test_pkg_input_parser(): - items = [ - ["PkgName", ("PkgName", None, None)], - [("PkgName", "!=1.2.3,<2.0"), ("PkgName", "!=1.2.3,<2.0", None)], - ["PkgName@1.2.3", ("PkgName", "1.2.3", None)], - [("PkgName@1.2.3", "1.2.5"), ("PkgName@1.2.3", "1.2.5", None)], - ["id=13", ("id=13", None, None)], - ["id=13@~1.2.3", ("id=13", "~1.2.3", None)], - [ - get_project_core_dir(), - (".platformio", None, "file://" + get_project_core_dir()), - ], - [ - "LocalName=" + get_project_core_dir(), - ("LocalName", None, "file://" + get_project_core_dir()), - ], - [ - "LocalName=%s@>2.3.0" % get_project_core_dir(), - ("LocalName", ">2.3.0", "file://" + get_project_core_dir()), - ], - [ - "https://github.com/user/package.git", - ("package", None, "git+https://github.com/user/package.git"), - ], - [ - "MyPackage=https://gitlab.com/user/package.git", - ("MyPackage", None, "git+https://gitlab.com/user/package.git"), - ], - [ - "MyPackage=https://gitlab.com/user/package.git@3.2.1,!=2", - ("MyPackage", "3.2.1,!=2", "git+https://gitlab.com/user/package.git"), - ], - [ - "https://somedomain.com/path/LibraryName-1.2.3.zip", - ( - "LibraryName-1.2.3", - None, - "https://somedomain.com/path/LibraryName-1.2.3.zip", - ), - ], - [ - "https://github.com/user/package/archive/branch.zip", - ("branch", None, "https://github.com/user/package/archive/branch.zip"), - ], - [ - "https://github.com/user/package/archive/branch.zip@~1.2.3", - ("branch", "~1.2.3", "https://github.com/user/package/archive/branch.zip"), - ], - [ - "https://github.com/user/package/archive/branch.tar.gz", - ( - "branch.tar", - None, - "https://github.com/user/package/archive/branch.tar.gz", - ), - ], - [ - "https://github.com/user/package/archive/branch.tar.gz@!=5", - ( - "branch.tar", - "!=5", - "https://github.com/user/package/archive/branch.tar.gz", - ), - ], - [ - "https://developer.mbed.org/users/user/code/package/", - ("package", None, "hg+https://developer.mbed.org/users/user/code/package/"), - ], - [ - "https://os.mbed.com/users/user/code/package/", - ("package", None, "hg+https://os.mbed.com/users/user/code/package/"), - ], - [ - "https://github.com/user/package#v1.2.3", - ("package", None, "git+https://github.com/user/package#v1.2.3"), - ], - [ - "https://github.com/user/package.git#branch", - ("package", None, "git+https://github.com/user/package.git#branch"), - ], - [ - "PkgName=https://github.com/user/package.git#a13d344fg56", - ("PkgName", None, "git+https://github.com/user/package.git#a13d344fg56"), - ], - ["user/package", ("package", None, "git+https://github.com/user/package")], - [ - "PkgName=user/package", - ("PkgName", None, "git+https://github.com/user/package"), - ], - [ - "PkgName=user/package#master", - ("PkgName", None, "git+https://github.com/user/package#master"), - ], - [ - "git+https://github.com/user/package", - ("package", None, "git+https://github.com/user/package"), - ], - [ - "hg+https://example.com/user/package", - ("package", None, "hg+https://example.com/user/package"), - ], - [ - "git@github.com:user/package.git", - ("package", None, "git+git@github.com:user/package.git"), - ], - [ - "git@github.com:user/package.git#v1.2.0", - ("package", None, "git+git@github.com:user/package.git#v1.2.0"), - ], - [ - "LocalName=git@github.com:user/package.git#v1.2.0@~1.2.0", - ("LocalName", "~1.2.0", "git+git@github.com:user/package.git#v1.2.0"), - ], - [ - "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", - ( - "package", - None, - "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", - ), - ], - [ - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ( - "package", - None, - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ), - ], - [ - "LocalName=git+ssh://user@gitlab.private-server.com:1234" - "/package#1.2.0@!=13", - ( - "LocalName", - "!=13", - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ), - ], - ] - for params, result in items: - if isinstance(params, tuple): - assert PackageManager.parse_pkg_uri(*params) == result - else: - assert PackageManager.parse_pkg_uri(params) == result - - -def test_install_packages(isolated_pio_core, tmpdir): - packages = [ - dict(id=1, name="name_1", version="shasum"), - dict(id=1, name="name_1", version="2.0.0"), - dict(id=1, name="name_1", version="2.1.0"), - dict(id=1, name="name_1", version="1.2"), - dict(id=1, name="name_1", version="1.0.0"), - dict(name="name_2", version="1.0.0"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - dict(name="name_2", version="3.0.0", __src_url="git+https://github2.com"), - dict(name="name_2", version="4.0.0", __src_url="git+https://github2.com"), - ] - - pm = PackageManager(join(get_project_core_dir(), "packages")) - for package in packages: - tmp_dir = tmpdir.mkdir("tmp-package") - tmp_dir.join("package.json").write(json.dumps(package)) - pm._install_from_url(package["name"], "file://%s" % str(tmp_dir)) - tmp_dir.remove(rec=1) - - assert len(pm.get_installed()) == len(packages) - 1 - - pkg_dirnames = [ - "name_1_ID1", - "name_1_ID1@1.0.0", - "name_1_ID1@1.2", - "name_1_ID1@2.0.0", - "name_1_ID1@shasum", - "name_2", - "name_2@src-177cbce1f0705580d17790fda1cc2ef5", - "name_2@src-f863b537ab00f4c7b5011fc44b120e1f", - ] - assert set( - [p.basename for p in isolated_pio_core.join("packages").listdir()] - ) == set(pkg_dirnames) - - -def test_get_package(): - tests = [ - [("unknown",), None], - [("1",), None], - [("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")], - [("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("name_1", "<2"), dict(id=1, name="name_1", version="1.2")], - [("name_1", ">2"), None], - [("name_1", "2-0-0"), None], - [("name_2",), dict(name="name_2", version="4.0.0")], - [ - ("url_has_higher_priority", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - ], - [ - ("name_2", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - ], - ] - - pm = PackageManager(join(get_project_core_dir(), "packages")) - for test in tests: - manifest = pm.get_package(*test[0]) - if test[1] is None: - assert manifest is None, test - continue - for key, value in test[1].items(): - assert manifest[key] == value, test From 26fdd0a62c2bcdec5e123b13819ae39f57f16169 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Aug 2020 18:30:33 +0300 Subject: [PATCH 149/223] Bump version to 4.4.0a8 --- platformio/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0a1bc48c77..ee6d438d3c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 4, "0a7") +import sys + +VERSION = (4, 4, "0a8") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From ecc369c2f8facbc1a783e9f873063b0aa86f3ade Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Aug 2020 20:19:27 +0300 Subject: [PATCH 150/223] Minor fixes --- platformio/commands/system/command.py | 8 +++++--- tests/package/test_manifest.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index af4071e072..76d2cb3687 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -26,9 +26,9 @@ install_completion_code, uninstall_completion_code, ) -from platformio.managers.package import PackageManager from platformio.managers.platform import PlatformManager from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig @@ -80,9 +80,11 @@ def system_info(json_output): "value": len(PlatformManager().get_installed()), } data["package_tool_nums"] = { - "title": "Package Tools", + "title": "Tools & Toolchains", "value": len( - PackageManager(project_config.get_optional_dir("packages")).get_installed() + ToolPackageManager( + project_config.get_optional_dir("packages") + ).get_installed() ), } diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 899f1cdf83..35fdf3671e 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -485,7 +485,7 @@ def test_library_properties_schema(): contents = """ name=Mozzi version=1.0.3 -author=Tim Barrass and contributors as documented in source, and at https://github.com/sensorium/Mozzi/graphs/contributors +author=Lorem Ipsum is simply dummy text of the printing and typesetting industry Lorem Ipsum has been the industry's standard dummy text ever since the 1500s when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries but also the leap into electronic typesetting remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. maintainer=Tim Barrass sentence=Sound synthesis library for Arduino paragraph=With Mozzi, you can construct sounds using familiar synthesis units like oscillators, delays, filters and envelopes. @@ -504,6 +504,7 @@ def test_library_properties_schema(): ), ).as_dict() + errors = None try: ManifestSchema().load_manifest(raw_data) except ManifestValidationError as e: From ff8ec43a288d7314ef542e08ba662c06884af7c6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Aug 2020 21:46:46 +0300 Subject: [PATCH 151/223] Ensure tool-type package is compatible with a host system --- platformio/package/manager/base.py | 22 ++++++++++++------- tests/package/test_manager.py | 34 ++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index 58c35d47b3..c0e1e23ff1 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -202,8 +202,17 @@ def get_installed(self): pkg.metadata = self.build_metadata(pkg_dir, spec) except MissingPackageManifestError: pass - if pkg.metadata: - result.append(pkg) + if not pkg.metadata: + continue + if self.pkg_type == PackageType.TOOL: + try: + if not self.is_system_compatible( + self.load_manifest(pkg).get("system") + ): + continue + except MissingPackageManifestError: + pass + result.append(pkg) return result def get_package(self, spec): @@ -212,7 +221,7 @@ def get_package(self, spec): spec = self.ensure_spec(spec) best = None for pkg in self.get_installed(): - if not self._test_pkg_with_spec(pkg, spec): + if not self.test_pkg_spec(pkg, spec): continue assert isinstance(pkg.metadata.version, semantic_version.Version) if spec.requirements and pkg.metadata.version not in spec.requirements: @@ -221,7 +230,8 @@ def get_package(self, spec): best = pkg return best - def _test_pkg_with_spec(self, pkg, spec): + @staticmethod + def test_pkg_spec(pkg, spec): # "id" mismatch if spec.id and spec.id != pkg.metadata.spec.id: return False @@ -244,8 +254,4 @@ def _test_pkg_with_spec(self, pkg, spec): elif not spec.id and not ci_strings_are_equal(spec.name, pkg.metadata.name): return False - if self.pkg_type == PackageType.TOOL: - # TODO: check "system" for pkg - pass - return True diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 131346af77..c91924f414 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -239,7 +239,7 @@ def test_install_force(isolated_pio_core, tmpdir_factory): def test_get_installed(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") - lm = LibraryPackageManager(str(storage_dir)) + pm = ToolPackageManager(str(storage_dir)) # VCS package ( @@ -259,7 +259,7 @@ def test_get_installed(isolated_pio_core, tmpdir_factory): "requirements": null, "url": "git+https://github.com/username/repo.git" }, - "type": "library", + "type": "tool", "version": "0.0.0+sha.1ea4d5e" } """ @@ -270,13 +270,13 @@ def test_get_installed(isolated_pio_core, tmpdir_factory): ( storage_dir.join("foo@3.4.5") .mkdir() - .join("library.json") + .join("package.json") .write('{"name": "foo", "version": "3.4.5"}') ) # package with metadata file foo_dir = storage_dir.join("foo").mkdir() - foo_dir.join("library.json").write('{"name": "foo", "version": "3.6.0"}') + foo_dir.join("package.json").write('{"name": "foo", "version": "3.6.0"}') foo_dir.join(".piopm").write( """ { @@ -286,21 +286,33 @@ def test_get_installed(isolated_pio_core, tmpdir_factory): "owner": null, "requirements": "^3" }, - "type": "library", + "type": "tool", "version": "3.6.0" } """ ) + # test "system" + storage_dir.join("pkg-incompatible-system").mkdir().join("package.json").write( + '{"name": "check-system", "version": "4.0.0", "system": ["unknown"]}' + ) + storage_dir.join("pkg-compatible-system").mkdir().join("package.json").write( + '{"name": "check-system", "version": "3.0.0", "system": "%s"}' + % util.get_systype() + ) + # invalid package - storage_dir.join("invalid-package").mkdir().join("package.json").write( - '{"name": "tool-scons", "version": "4.0.0"}' + storage_dir.join("invalid-package").mkdir().join("library.json").write( + '{"name": "SomeLib", "version": "4.0.0"}' ) - installed = lm.get_installed() - assert len(installed) == 3 - assert set(["pkg-via-vcs", "foo"]) == set(p.metadata.name for p in installed) - assert str(lm.get_package("foo").metadata.version) == "3.6.0" + installed = pm.get_installed() + assert len(installed) == 4 + assert set(["pkg-via-vcs", "foo", "check-system"]) == set( + p.metadata.name for p in installed + ) + assert str(pm.get_package("foo").metadata.version) == "3.6.0" + assert str(pm.get_package("check-system").metadata.version) == "3.0.0" def test_uninstall(isolated_pio_core, tmpdir_factory): From 5f3ad70190cc068f23b6785fe2238ff4a9c7a418 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 14 Aug 2020 16:38:46 +0300 Subject: [PATCH 152/223] Rename meta.PackageSourceItem or PackageItem --- platformio/builder/tools/piolib.py | 4 ++-- platformio/commands/lib/command.py | 8 +++----- platformio/package/manager/_install.py | 18 +++++++++--------- platformio/package/manager/_legacy.py | 6 ++++-- platformio/package/manager/_uninstall.py | 4 ++-- platformio/package/manager/_update.py | 8 ++------ platformio/package/manager/base.py | 15 ++++++++------- platformio/package/meta.py | 6 +++--- platformio/package/pack.py | 4 ++-- 9 files changed, 35 insertions(+), 38 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 3a7b3aad73..8cc1ad58ce 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -40,7 +40,7 @@ ManifestParserError, ManifestParserFactory, ) -from platformio.package.meta import PackageSourceItem +from platformio.package.meta import PackageItem from platformio.project.options import ProjectOptions @@ -1019,7 +1019,7 @@ def _print_deps_tree(root, level=0): margin = "| " * (level) for lb in root.depbuilders: title = "<%s>" % lb.name - pkg = PackageSourceItem(lb.path) + pkg = PackageItem(lb.path) if pkg.metadata: title += " %s" % pkg.metadata.version elif lb.version: diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index ec5fd8e668..03463aaba4 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -30,7 +30,7 @@ from platformio.compat import dump_json_to_unicode from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager -from platformio.package.meta import PackageSourceItem, PackageSpec +from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import get_project_dir, is_platformio_project @@ -262,7 +262,7 @@ def lib_update( # pylint: disable=too-many-arguments for library in _libraries: spec = None pkg = None - if isinstance(library, PackageSourceItem): + if isinstance(library, PackageItem): pkg = library else: spec = PackageSpec(library) @@ -284,9 +284,7 @@ def lib_update( # pylint: disable=too-many-arguments else: for library in _libraries: to_spec = ( - None - if isinstance(library, PackageSourceItem) - else PackageSpec(library) + None if isinstance(library, PackageItem) else PackageSpec(library) ) lm.update( library, to_spec=to_spec, only_check=only_check, silent=silent diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index c63a504db0..04a41f267a 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -21,7 +21,7 @@ from platformio import app, compat, fs, util from platformio.package.exception import PackageException -from platformio.package.meta import PackageSourceItem, PackageSpec +from platformio.package.meta import PackageItem, PackageSpec from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -109,7 +109,7 @@ def _install(self, spec, search_filters=None, silent=False, force=False): return pkg def _install_dependencies(self, pkg, silent=False): - assert isinstance(pkg, PackageSourceItem) + assert isinstance(pkg, PackageItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): return @@ -155,7 +155,7 @@ def install_from_url(self, url, spec, checksum=None, silent=False): assert vcs.export() root_dir = self.find_pkg_root(tmp_dir, spec) - pkg_item = PackageSourceItem( + pkg_item = PackageItem( root_dir, self.build_metadata( root_dir, spec, vcs.get_current_revision() if vcs else None @@ -168,7 +168,7 @@ def install_from_url(self, url, spec, checksum=None, silent=False): fs.rmtree(tmp_dir) def _install_tmp_pkg(self, tmp_pkg): - assert isinstance(tmp_pkg, PackageSourceItem) + assert isinstance(tmp_pkg, PackageItem) # validate package version and declared requirements if ( tmp_pkg.metadata.spec.requirements @@ -182,7 +182,7 @@ def _install_tmp_pkg(self, tmp_pkg): tmp_pkg.metadata, ) ) - dst_pkg = PackageSourceItem( + dst_pkg = PackageItem( os.path.join(self.package_dir, tmp_pkg.get_safe_dirname()) ) @@ -190,7 +190,7 @@ def _install_tmp_pkg(self, tmp_pkg): action = "overwrite" if tmp_pkg.metadata.spec.has_custom_name(): action = "overwrite" - dst_pkg = PackageSourceItem( + dst_pkg = PackageItem( os.path.join(self.package_dir, tmp_pkg.metadata.spec.name) ) elif dst_pkg.metadata and dst_pkg.metadata.spec.external: @@ -231,7 +231,7 @@ def _cleanup_dir(path): # move new source to the destination location _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) - return PackageSourceItem(dst_pkg.path) + return PackageItem(dst_pkg.path) if action == "detach-new": target_dirname = "%s@%s" % ( @@ -248,9 +248,9 @@ def _cleanup_dir(path): pkg_dir = os.path.join(self.package_dir, target_dirname) _cleanup_dir(pkg_dir) shutil.move(tmp_pkg.path, pkg_dir) - return PackageSourceItem(pkg_dir) + return PackageItem(pkg_dir) # otherwise, overwrite existing _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) - return PackageSourceItem(dst_pkg.path) + return PackageItem(dst_pkg.path) diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py index 22478eff19..95f628d010 100644 --- a/platformio/package/manager/_legacy.py +++ b/platformio/package/manager/_legacy.py @@ -15,7 +15,7 @@ import os from platformio import fs -from platformio.package.meta import PackageSourceItem, PackageSpec +from platformio.package.meta import PackageItem, PackageSpec class PackageManagerLegacyMixin(object): @@ -42,7 +42,9 @@ def build_legacy_spec(self, pkg_dir): return PackageSpec(name=manifest.get("name")) def legacy_load_manifest(self, pkg): - assert isinstance(pkg, PackageSourceItem) + if not isinstance(pkg, PackageItem): + assert os.path.isdir(pkg) + pkg = PackageItem(pkg) manifest = self.load_manifest(pkg) manifest["__pkg_dir"] = pkg.path for key in ("name", "version"): diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 603ad38218..e265640172 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -19,7 +19,7 @@ from platformio import fs from platformio.package.exception import UnknownPackageError -from platformio.package.meta import PackageSourceItem, PackageSpec +from platformio.package.meta import PackageItem, PackageSpec class PackageManagerUninstallMixin(object): @@ -73,7 +73,7 @@ def _uninstall(self, spec, silent=False, skip_dependencies=False): return pkg def _uninstall_dependencies(self, pkg, silent=False): - assert isinstance(pkg, PackageSourceItem) + assert isinstance(pkg, PackageItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): return diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index d120e030c4..b0b976de42 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -18,17 +18,13 @@ from platformio import util from platformio.package.exception import UnknownPackageError -from platformio.package.meta import ( - PackageOutdatedResult, - PackageSourceItem, - PackageSpec, -) +from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageSpec from platformio.package.vcsclient import VCSBaseException, VCSClientFactory class PackageManagerUpdateMixin(object): def outdated(self, pkg, spec=None): - assert isinstance(pkg, PackageSourceItem) + assert isinstance(pkg, PackageItem) assert not spec or isinstance(spec, PackageSpec) assert os.path.isdir(pkg.path) and pkg.metadata diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index c0e1e23ff1..dc024edca0 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -31,8 +31,8 @@ from platformio.package.manager._update import PackageManagerUpdateMixin from platformio.package.manifest.parser import ManifestParserFactory from platformio.package.meta import ( + PackageItem, PackageMetaData, - PackageSourceItem, PackageSpec, PackageType, ) @@ -144,7 +144,7 @@ def manifest_exists(self, pkg_dir): return self.get_manifest_path(pkg_dir) def load_manifest(self, src): - path = src.path if isinstance(src, PackageSourceItem) else src + path = src.path if isinstance(src, PackageItem) else src cache_key = "load_manifest-%s" % path result = self.memcache_get(cache_key) if result: @@ -191,11 +191,11 @@ def build_metadata(self, pkg_dir, spec, vcs_revision=None): def get_installed(self): result = [] - for name in os.listdir(self.package_dir): + for name in sorted(os.listdir(self.package_dir)): pkg_dir = os.path.join(self.package_dir, name) if not os.path.isdir(pkg_dir): continue - pkg = PackageSourceItem(pkg_dir) + pkg = PackageItem(pkg_dir) if not pkg.metadata: try: spec = self.build_legacy_spec(pkg_dir) @@ -216,7 +216,7 @@ def get_installed(self): return result def get_package(self, spec): - if isinstance(spec, PackageSourceItem): + if isinstance(spec, PackageItem): return spec spec = self.ensure_spec(spec) best = None @@ -243,8 +243,9 @@ def test_pkg_spec(pkg, spec): # external "URL" mismatch if spec.external: # local folder mismatch - if spec.url == pkg.path or ( - spec.url.startswith("file://") and pkg.path == spec.url[7:] + if os.path.realpath(spec.url) == os.path.realpath(pkg.path) or ( + spec.url.startswith("file://") + and os.path.realpath(pkg.path) == os.path.realpath(spec.url[7:]) ): return True if spec.url != pkg.metadata.spec.url: diff --git a/platformio/package/meta.py b/platformio/package/meta.py index af1e0baaea..f57f049534 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -315,7 +315,7 @@ class PackageMetaData(object): def __init__( # pylint: disable=redefined-builtin self, type, name, version, spec=None ): - assert type in PackageType.items().values() + # assert type in PackageType.items().values() if spec: assert isinstance(spec, PackageSpec) self.type = type @@ -395,7 +395,7 @@ def load(path): return PackageMetaData(**data) -class PackageSourceItem(object): +class PackageItem(object): METAFILE_NAME = ".piopm" @@ -406,7 +406,7 @@ def __init__(self, path, metadata=None): self.metadata = self.load_meta() def __repr__(self): - return "PackageSourceItem Date: Fri, 14 Aug 2020 16:39:15 +0300 Subject: [PATCH 153/223] Refactor dev-platform API --- Makefile | 2 +- platformio/builder/main.py | 2 +- platformio/builder/tools/pioplatform.py | 32 +- platformio/commands/debug/helpers.py | 11 +- platformio/commands/device/command.py | 4 +- platformio/commands/lib/helpers.py | 5 +- platformio/commands/platform.py | 17 +- platformio/commands/project.py | 5 +- platformio/commands/run/processor.py | 10 +- platformio/commands/test/embedded.py | 4 +- platformio/exception.py | 43 -- platformio/maintenance.py | 7 +- platformio/managers/platform.py | 744 +----------------------- platformio/platform/__init__.py | 13 + platformio/platform/_packages.py | 126 ++++ platformio/platform/_run.py | 193 ++++++ platformio/platform/base.py | 274 +++++++++ platformio/platform/board.py | 158 +++++ platformio/platform/exception.py | 49 ++ platformio/platform/factory.py | 60 ++ 20 files changed, 941 insertions(+), 818 deletions(-) create mode 100644 platformio/platform/__init__.py create mode 100644 platformio/platform/_packages.py create mode 100644 platformio/platform/_run.py create mode 100644 platformio/platform/base.py create mode 100644 platformio/platform/board.py create mode 100644 platformio/platform/exception.py create mode 100644 platformio/platform/factory.py diff --git a/Makefile b/Makefile index 36b5d3963d..6b22d261eb 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ clean: clean-docs profile: # Usage $ > make PIOARGS="boards" profile - python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS} + python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS} snakeviz .tox/.tmp/cprofile.prof publish: diff --git a/platformio/builder/main.py b/platformio/builder/main.py index a0a8ab1208..e73f686927 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -30,7 +30,7 @@ from platformio import compat, fs from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformBase +from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_dir diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 8c047365d5..fe7d6e28d4 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -14,15 +14,16 @@ from __future__ import absolute_import +import os import sys -from os.path import isdir, isfile, join from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error -from platformio import exception, fs, util +from platformio import fs, util from platformio.compat import WINDOWS -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownBoard +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectOptions # pylint: disable=too-many-branches, too-many-locals @@ -34,7 +35,7 @@ def PioPlatform(env): if "framework" in variables: # support PIO Core 3.0 dev/platforms variables["pioframework"] = variables["framework"] - p = PlatformFactory.newPlatform(env["PLATFORM_MANIFEST"]) + p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"])) p.configure_default_packages(variables, COMMAND_LINE_TARGETS) return p @@ -46,7 +47,7 @@ def BoardConfig(env, board=None): board = board or env.get("BOARD") assert board, "BoardConfig: Board is not defined" return p.board_config(board) - except (AssertionError, exception.UnknownBoard) as e: + except (AssertionError, UnknownBoard) as e: sys.stderr.write("Error: %s\n" % str(e)) env.Exit(1) @@ -55,8 +56,8 @@ def GetFrameworkScript(env, framework): p = env.PioPlatform() assert p.frameworks and framework in p.frameworks script_path = env.subst(p.frameworks[framework]["script"]) - if not isfile(script_path): - script_path = join(p.get_dir(), script_path) + if not os.path.isfile(script_path): + script_path = os.path.join(p.get_dir(), script_path) return script_path @@ -75,17 +76,24 @@ def LoadPioPlatform(env): continue pkg_dir = p.get_package_dir(name) env.PrependENVPath( - "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir + "PATH", + os.path.join(pkg_dir, "bin") + if os.path.isdir(os.path.join(pkg_dir, "bin")) + else pkg_dir, ) - if not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain": + if ( + not WINDOWS + and os.path.isdir(os.path.join(pkg_dir, "lib")) + and type_ != "toolchain" + ): env.PrependENVPath( "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH", - join(pkg_dir, "lib"), + os.path.join(pkg_dir, "lib"), ) # Platform specific LD Scripts - if isdir(join(p.get_dir(), "ldscripts")): - env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")]) + if os.path.isdir(os.path.join(p.get_dir(), "ldscripts")): + env.Prepend(LIBPATH=[os.path.join(p.get_dir(), "ldscripts")]) if "BOARD" not in env: return diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 4604a86106..657e8c48ea 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -20,13 +20,14 @@ from io import BytesIO from os.path import isfile -from platformio import exception, fs, util +from platformio import fs, util from platformio.commands import PlatformioCLI from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.run.command import cli as cmd_run from platformio.compat import is_bytes -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.options import ProjectOptions @@ -94,14 +95,14 @@ def _cleanup_cmds(items): return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] try: - platform = PlatformFactory.newPlatform(env_options["platform"]) - except exception.UnknownPlatform: + platform = PlatformFactory.new(env_options["platform"]) + except UnknownPlatform: cmd_ctx.invoke( cmd_platform_install, platforms=[env_options["platform"]], skip_default_package=True, ) - platform = PlatformFactory.newPlatform(env_options["platform"]) + platform = PlatformFactory.new(env_options["platform"]) board_config = platform.board_config(env_options["board"]) tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index e93b1214d0..463116f92f 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -22,7 +22,7 @@ from platformio import exception, fs, util from platformio.commands.device import helpers as device_helpers from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformFactory +from platformio.platform.factory import PlatformFactory from platformio.project.exception import NotPlatformIOProjectError @@ -192,7 +192,7 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches platform = None if "platform" in project_options: with fs.cd(kwargs["project_dir"]): - platform = PlatformFactory.newPlatform(project_options["platform"]) + platform = PlatformFactory.new(project_options["platform"]) device_helpers.register_platform_filters( platform, kwargs["project_dir"], kwargs["environment"] ) diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py index a5cc07e35a..23892ac854 100644 --- a/platformio/commands/lib/helpers.py +++ b/platformio/commands/lib/helpers.py @@ -15,8 +15,9 @@ import os from platformio.compat import ci_strings_are_equal -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.platform import PlatformManager from platformio.package.meta import PackageSpec +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import InvalidProjectConfError @@ -29,7 +30,7 @@ def get_builtin_libs(storage_names=None): storage_names = storage_names or [] pm = PlatformManager() for manifest in pm.get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) for storage in p.get_lib_storages(): if storage_names and storage["name"] not in storage_names: continue diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index deaeb4317d..14d5a1f29e 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -16,10 +16,12 @@ import click -from platformio import app, exception, util +from platformio import app, util from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.platform import PlatformManager +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory @click.group(short_help="Platform Manager") @@ -64,12 +66,12 @@ def _get_registry_platforms(): def _get_platform_data(*args, **kwargs): try: return _get_installed_platform_data(*args, **kwargs) - except exception.UnknownPlatform: + except UnknownPlatform: return _get_registry_platform_data(*args, **kwargs) def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): - p = PlatformFactory.newPlatform(platform) + p = PlatformFactory.new(platform) data = dict( name=p.name, title=p.title, @@ -232,7 +234,7 @@ def platform_list(json_output): def platform_show(platform, json_output): # pylint: disable=too-many-branches data = _get_platform_data(platform) if not data: - raise exception.UnknownPlatform(platform) + raise UnknownPlatform(platform) if json_output: return click.echo(dump_json_to_unicode(data)) @@ -384,10 +386,7 @@ def platform_update( # pylint: disable=too-many-locals if not pkg_dir: continue latest = pm.outdated(pkg_dir, requirements) - if ( - not latest - and not PlatformFactory.newPlatform(pkg_dir).are_outdated_packages() - ): + if not latest and not PlatformFactory.new(pkg_dir).are_outdated_packages(): continue data = _get_installed_platform_data( pkg_dir, with_boards=False, expose_packages=False diff --git a/platformio/commands/project.py b/platformio/commands/project.py index c261a9d981..6194a915a8 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -20,10 +20,11 @@ import click from tabulate import tabulate -from platformio import exception, fs +from platformio import fs from platformio.commands.platform import platform_install as cli_platform_install from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager +from platformio.platform.exception import UnknownBoard from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError from platformio.project.helpers import is_platformio_project, load_project_ide_data @@ -112,7 +113,7 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 for id_ in value: try: pm.board_config(id_) - except exception.UnknownBoard: + except UnknownBoard: raise click.BadParameter( "`%s`. Please search for board ID using `platformio boards` " "command" % id_ diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 23ccc33329..d07c581ca4 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio import exception from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.exception import UndefinedEnvPlatformError # pylint: disable=too-many-instance-attributes @@ -67,14 +67,14 @@ def process(self): build_targets.remove("monitor") try: - p = PlatformFactory.newPlatform(self.options["platform"]) - except exception.UnknownPlatform: + p = PlatformFactory.new(self.options["platform"]) + except UnknownPlatform: self.cmd_ctx.invoke( cmd_platform_install, platforms=[self.options["platform"]], skip_default_package=True, ) - p = PlatformFactory.newPlatform(self.options["platform"]) + p = PlatformFactory.new(self.options["platform"]) result = p.run(build_vars, build_targets, self.silent, self.verbose, self.jobs) return result["returncode"] == 0 diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index 6f47eafcab..ca6584961b 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -19,7 +19,7 @@ from platformio import exception, util from platformio.commands.test.processor import TestProcessorBase -from platformio.managers.platform import PlatformFactory +from platformio.platform.factory import PlatformFactory class EmbeddedTestProcessor(TestProcessorBase): @@ -108,7 +108,7 @@ def get_test_port(self): return self.env_options.get("test_port") assert set(["platform", "board"]) & set(self.env_options.keys()) - p = PlatformFactory.newPlatform(self.env_options["platform"]) + p = PlatformFactory.new(self.env_options["platform"]) board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", []) port = None elapsed = 0 diff --git a/platformio/exception.py b/platformio/exception.py index 9ab0e4d810..91fd67cce2 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -47,44 +47,6 @@ class AbortedByUser(UserSideException): MESSAGE = "Aborted by user" -# -# Development Platform -# - - -class UnknownPlatform(PlatformioException): - - MESSAGE = "Unknown development platform '{0}'" - - -class IncompatiblePlatform(PlatformioException): - - MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}" - - -class PlatformNotInstalledYet(PlatformioException): - - MESSAGE = ( - "The platform '{0}' has not been installed yet. " - "Use `platformio platform install {0}` command" - ) - - -class UnknownBoard(PlatformioException): - - MESSAGE = "Unknown board ID '{0}'" - - -class InvalidBoardManifest(PlatformioException): - - MESSAGE = "Invalid board JSON manifest '{0}'" - - -class UnknownFramework(PlatformioException): - - MESSAGE = "Unknown framework '{0}'" - - # Package Manager @@ -195,11 +157,6 @@ class InternetIsOffline(UserSideException): ) -class BuildScriptNotFound(PlatformioException): - - MESSAGE = "Invalid path '{0}' to build script" - - class InvalidSettingName(UserSideException): MESSAGE = "Invalid setting with the name '{0}'" diff --git a/platformio/maintenance.py b/platformio/maintenance.py index cf2e0698e6..4b47d50a39 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -25,11 +25,12 @@ from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.platform import PlatformManager from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec +from platformio.platform.factory import PlatformFactory from platformio.proc import is_container @@ -278,9 +279,7 @@ def check_internal_updates(ctx, what): # pylint: disable=too-many-branches conds = [ pm.outdated(manifest["__pkg_dir"]), what == "platforms" - and PlatformFactory.newPlatform( - manifest["__pkg_dir"] - ).are_outdated_packages(), + and PlatformFactory.new(manifest["__pkg_dir"]).are_outdated_packages(), ] if any(conds): outdated_items.append(manifest["name"]) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8548bba63a..c0e0f98eb9 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -14,31 +14,16 @@ # pylint: disable=too-many-public-methods, too-many-instance-attributes -import base64 -import os -import re -import subprocess -import sys -from os.path import basename, dirname, isdir, isfile, join -import click -import semantic_version +from os.path import isdir, isfile, join -from platformio import __version__, app, exception, fs, proc, telemetry, util -from platformio.commands.debug.exception import ( - DebugInvalidOptionsError, - DebugSupportError, -) -from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module +from platformio import app, exception, util from platformio.managers.package import BasePkgManager, PackageManager -from platformio.package.manager.core import get_core_package_dir +from platformio.platform.base import PlatformBase # pylint: disable=unused-import +from platformio.platform.exception import UnknownBoard, UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig -try: - from urllib.parse import quote -except ImportError: - from urllib import quote - class PlatformManager(BasePkgManager): def __init__(self, package_dir=None, repositories=None): @@ -83,7 +68,7 @@ def install( platform_dir = BasePkgManager.install( self, name, requirements, silent=silent, force=force ) - p = PlatformFactory.newPlatform(platform_dir) + p = PlatformFactory.new(platform_dir) if with_all_packages: with_packages = list(p.packages.keys()) @@ -114,9 +99,9 @@ def uninstall(self, package, requirements=None, after_update=False): pkg_dir = self.get_package_dir(name, requirements, url) if not pkg_dir: - raise exception.UnknownPlatform(package) + raise UnknownPlatform(package) - p = PlatformFactory.newPlatform(pkg_dir) + p = PlatformFactory.new(pkg_dir) BasePkgManager.uninstall(self, pkg_dir, requirements) p.uninstall_python_packages() p.on_uninstalled() @@ -138,15 +123,15 @@ def update( # pylint: disable=arguments-differ pkg_dir = self.get_package_dir(name, requirements, url) if not pkg_dir: - raise exception.UnknownPlatform(package) + raise UnknownPlatform(package) - p = PlatformFactory.newPlatform(pkg_dir) + p = PlatformFactory.new(pkg_dir) pkgs_before = list(p.get_installed_packages()) missed_pkgs = set() if not only_packages: BasePkgManager.update(self, pkg_dir, requirements, only_check) - p = PlatformFactory.newPlatform(pkg_dir) + p = PlatformFactory.new(pkg_dir) missed_pkgs = set(pkgs_before) & set(p.packages) missed_pkgs -= set(p.get_installed_packages()) @@ -164,7 +149,7 @@ def cleanup_packages(self, names): self.cache_reset() deppkgs = {} for manifest in PlatformManager().get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) for pkgname, pkgmanifest in p.get_installed_packages().items(): if pkgname not in deppkgs: deppkgs[pkgname] = set() @@ -190,7 +175,7 @@ def cleanup_packages(self, names): def get_installed_boards(self): boards = [] for manifest in self.get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) for config in p.get_boards().values(): board = config.get_brief_data() if board not in boards: @@ -224,705 +209,4 @@ def board_config(self, id_, platform=None): not platform or manifest["platform"] == platform ): return manifest - raise exception.UnknownBoard(id_) - - -class PlatformFactory(object): - @staticmethod - def get_clsname(name): - name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) - return "%s%sPlatform" % (name.upper()[0], name.lower()[1:]) - - @staticmethod - def load_module(name, path): - try: - return load_python_module("platformio.managers.platform.%s" % name, path) - except ImportError: - raise exception.UnknownPlatform(name) - - @classmethod - def newPlatform(cls, name, requirements=None): - pm = PlatformManager() - platform_dir = None - if isdir(name): - platform_dir = name - name = pm.load_manifest(platform_dir)["name"] - elif name.endswith("platform.json") and isfile(name): - platform_dir = dirname(name) - name = fs.load_json(name)["name"] - else: - name, requirements, url = pm.parse_pkg_uri(name, requirements) - platform_dir = pm.get_package_dir(name, requirements, url) - if platform_dir: - name = pm.load_manifest(platform_dir)["name"] - - if not platform_dir: - raise exception.UnknownPlatform( - name if not requirements else "%s@%s" % (name, requirements) - ) - - platform_cls = None - if isfile(join(platform_dir, "platform.py")): - platform_cls = getattr( - cls.load_module(name, join(platform_dir, "platform.py")), - cls.get_clsname(name), - ) - else: - platform_cls = type(str(cls.get_clsname(name)), (PlatformBase,), {}) - - _instance = platform_cls(join(platform_dir, "platform.json")) - assert isinstance(_instance, PlatformBase) - return _instance - - -class PlatformPackagesMixin(object): - def install_packages( # pylint: disable=too-many-arguments - self, - with_packages=None, - without_packages=None, - skip_default_package=False, - silent=False, - force=False, - ): - with_packages = set(self.find_pkg_names(with_packages or [])) - without_packages = set(self.find_pkg_names(without_packages or [])) - - upkgs = with_packages | without_packages - ppkgs = set(self.packages) - if not upkgs.issubset(ppkgs): - raise exception.UnknownPackage(", ".join(upkgs - ppkgs)) - - for name, opts in self.packages.items(): - version = opts.get("version", "") - if name in without_packages: - continue - if name in with_packages or not ( - skip_default_package or opts.get("optional", False) - ): - if ":" in version: - self.pm.install( - "%s=%s" % (name, version), silent=silent, force=force - ) - else: - self.pm.install(name, version, silent=silent, force=force) - - return True - - def find_pkg_names(self, candidates): - result = [] - for candidate in candidates: - found = False - - # lookup by package types - for _name, _opts in self.packages.items(): - if _opts.get("type") == candidate: - result.append(_name) - found = True - - if ( - self.frameworks - and candidate.startswith("framework-") - and candidate[10:] in self.frameworks - ): - result.append(self.frameworks[candidate[10:]]["package"]) - found = True - - if not found: - result.append(candidate) - - return result - - def update_packages(self, only_check=False): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - self.pm.update(manifest["__pkg_dir"], requirements, only_check) - - def get_installed_packages(self): - items = {} - for name in self.packages: - pkg_dir = self.get_package_dir(name) - if pkg_dir: - items[name] = self.pm.load_manifest(pkg_dir) - return items - - def are_outdated_packages(self): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - if self.pm.outdated(manifest["__pkg_dir"], requirements): - return True - return False - - def get_package_dir(self, name): - version = self.packages[name].get("version", "") - if ":" in version: - return self.pm.get_package_dir( - *self.pm.parse_pkg_uri("%s=%s" % (name, version)) - ) - return self.pm.get_package_dir(name, version) - - def get_package_version(self, name): - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - return None - return self.pm.load_manifest(pkg_dir).get("version") - - def dump_used_packages(self): - result = [] - for name, options in self.packages.items(): - if options.get("optional"): - continue - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - continue - manifest = self.pm.load_manifest(pkg_dir) - item = {"name": manifest["name"], "version": manifest["version"]} - if manifest.get("__src_url"): - item["src_url"] = manifest.get("__src_url") - result.append(item) - return result - - -class PlatformRunMixin(object): - - LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) - - @staticmethod - def encode_scons_arg(value): - data = base64.urlsafe_b64encode(hashlib_encode_data(value)) - return data.decode() if is_bytes(data) else data - - @staticmethod - def decode_scons_arg(data): - value = base64.urlsafe_b64decode(data) - return value.decode() if is_bytes(value) else value - - def run( # pylint: disable=too-many-arguments - self, variables, targets, silent, verbose, jobs - ): - assert isinstance(variables, dict) - assert isinstance(targets, list) - - options = self.config.items(env=variables["pioenv"], as_dict=True) - if "framework" in options: - # support PIO Core 3.0 dev/platforms - options["pioframework"] = options["framework"] - self.configure_default_packages(options, targets) - self.install_packages(silent=True) - - self._report_non_sensitive_data(options, targets) - - self.silent = silent - self.verbose = verbose or app.get_setting("force_verbose") - - if "clean" in targets: - targets = ["-c", "."] - - variables["platform_manifest"] = self.manifest_path - - if "build_script" not in variables: - variables["build_script"] = self.get_build_script() - if not isfile(variables["build_script"]): - raise exception.BuildScriptNotFound(variables["build_script"]) - - result = self._run_scons(variables, targets, jobs) - assert "returncode" in result - - return result - - def _report_non_sensitive_data(self, options, targets): - topts = options.copy() - topts["platform_packages"] = [ - dict(name=item["name"], version=item["version"]) - for item in self.dump_used_packages() - ] - topts["platform"] = {"name": self.name, "version": self.version} - if self.src_version: - topts["platform"]["src_version"] = self.src_version - telemetry.send_run_environment(topts, targets) - - def _run_scons(self, variables, targets, jobs): - args = [ - proc.get_pythonexe_path(), - join(get_core_package_dir("tool-scons"), "script", "scons"), - "-Q", - "--warn=no-no-parallel-support", - "--jobs", - str(jobs), - "--sconstruct", - join(fs.get_source_dir(), "builder", "main.py"), - ] - args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) - # pylint: disable=protected-access - args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) - args += targets - - # encode and append variables - for key, value in variables.items(): - args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - - proc.copy_pythonpath_to_osenv() - - if targets and "menuconfig" in targets: - return proc.exec_command( - args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin - ) - - if click._compat.isatty(sys.stdout): - - def _write_and_flush(stream, data): - try: - stream.write(data) - stream.flush() - except IOError: - pass - - return proc.exec_command( - args, - stdout=proc.BuildAsyncPipe( - line_callback=self._on_stdout_line, - data_callback=lambda data: _write_and_flush(sys.stdout, data), - ), - stderr=proc.BuildAsyncPipe( - line_callback=self._on_stderr_line, - data_callback=lambda data: _write_and_flush(sys.stderr, data), - ), - ) - - return proc.exec_command( - args, - stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), - stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), - ) - - def _on_stdout_line(self, line): - if "`buildprog' is up to date." in line: - return - self._echo_line(line, level=1) - - def _on_stderr_line(self, line): - is_error = self.LINE_ERROR_RE.search(line) is not None - self._echo_line(line, level=3 if is_error else 2) - - a_pos = line.find("fatal error:") - b_pos = line.rfind(": No such file or directory") - if a_pos == -1 or b_pos == -1: - return - self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) - - def _echo_line(self, line, level): - if line.startswith("scons: "): - line = line[7:] - assert 1 <= level <= 3 - if self.silent and (level < 2 or not line): - return - fg = (None, "yellow", "red")[level - 1] - if level == 1 and "is up to date" in line: - fg = "green" - click.secho(line, fg=fg, err=level > 1, nl=False) - - @staticmethod - def _echo_missed_dependency(filename): - if "/" in filename or not filename.endswith((".h", ".hpp")): - return - banner = """ -{dots} -* Looking for {filename_styled} dependency? Check our library registry! -* -* CLI > platformio lib search "header:{filename}" -* Web > {link} -* -{dots} -""".format( - filename=filename, - filename_styled=click.style(filename, fg="cyan"), - link=click.style( - "https://platformio.org/lib/search?query=header:%s" - % quote(filename, safe=""), - fg="blue", - ), - dots="*" * (56 + len(filename)), - ) - click.echo(banner, err=True) - - -class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): - - PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) - _BOARDS_CACHE = {} - - def __init__(self, manifest_path): - self.manifest_path = manifest_path - self.silent = False - self.verbose = False - - self._manifest = fs.load_json(manifest_path) - self._BOARDS_CACHE = {} - self._custom_packages = None - - self.config = ProjectConfig.get_instance() - self.pm = PackageManager( - self.config.get_optional_dir("packages"), self.package_repositories - ) - - self._src_manifest = None - src_manifest_path = self.pm.get_src_manifest_path(self.get_dir()) - if src_manifest_path: - self._src_manifest = fs.load_json(src_manifest_path) - - # if self.engines and "platformio" in self.engines: - # if self.PIO_VERSION not in semantic_version.SimpleSpec( - # self.engines['platformio']): - # raise exception.IncompatiblePlatform(self.name, - # str(self.PIO_VERSION)) - - @property - def name(self): - return self._manifest["name"] - - @property - def title(self): - return self._manifest["title"] - - @property - def description(self): - return self._manifest["description"] - - @property - def version(self): - return self._manifest["version"] - - @property - def src_version(self): - return self._src_manifest.get("version") if self._src_manifest else None - - @property - def src_url(self): - return self._src_manifest.get("url") if self._src_manifest else None - - @property - def homepage(self): - return self._manifest.get("homepage") - - @property - def repository_url(self): - return self._manifest.get("repository", {}).get("url") - - @property - def license(self): - return self._manifest.get("license") - - @property - def frameworks(self): - return self._manifest.get("frameworks") - - @property - def engines(self): - return self._manifest.get("engines") - - @property - def package_repositories(self): - return self._manifest.get("packageRepositories") - - @property - def manifest(self): - return self._manifest - - @property - def packages(self): - packages = self._manifest.get("packages", {}) - for item in self._custom_packages or []: - name = item - version = "*" - if "@" in item: - name, version = item.split("@", 2) - name = name.strip() - if name not in packages: - packages[name] = {} - packages[name].update({"version": version.strip(), "optional": False}) - return packages - - @property - def python_packages(self): - return self._manifest.get("pythonPackages") - - def get_dir(self): - return dirname(self.manifest_path) - - def get_build_script(self): - main_script = join(self.get_dir(), "builder", "main.py") - if isfile(main_script): - return main_script - raise NotImplementedError() - - def is_embedded(self): - for opts in self.packages.values(): - if opts.get("type") == "uploader": - return True - return False - - def get_boards(self, id_=None): - def _append_board(board_id, manifest_path): - config = PlatformBoardConfig(manifest_path) - if "platform" in config and config.get("platform") != self.name: - return - if "platforms" in config and self.name not in config.get("platforms"): - return - config.manifest["platform"] = self.name - self._BOARDS_CACHE[board_id] = config - - bdirs = [ - self.config.get_optional_dir("boards"), - join(self.config.get_optional_dir("core"), "boards"), - join(self.get_dir(), "boards"), - ] - - if id_ is None: - for boards_dir in bdirs: - if not isdir(boards_dir): - continue - for item in sorted(os.listdir(boards_dir)): - _id = item[:-5] - if not item.endswith(".json") or _id in self._BOARDS_CACHE: - continue - _append_board(_id, join(boards_dir, item)) - else: - if id_ not in self._BOARDS_CACHE: - for boards_dir in bdirs: - if not isdir(boards_dir): - continue - manifest_path = join(boards_dir, "%s.json" % id_) - if isfile(manifest_path): - _append_board(id_, manifest_path) - break - if id_ not in self._BOARDS_CACHE: - raise exception.UnknownBoard(id_) - return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE - - def board_config(self, id_): - return self.get_boards(id_) - - def get_package_type(self, name): - return self.packages[name].get("type") - - def configure_default_packages(self, options, targets): - # override user custom packages - self._custom_packages = options.get("platform_packages") - - # enable used frameworks - for framework in options.get("framework", []): - if not self.frameworks: - continue - framework = framework.lower().strip() - if not framework or framework not in self.frameworks: - continue - _pkg_name = self.frameworks[framework].get("package") - if _pkg_name: - self.packages[_pkg_name]["optional"] = False - - # enable upload tools for upload targets - if any(["upload" in t for t in targets] + ["program" in targets]): - for name, opts in self.packages.items(): - if opts.get("type") == "uploader": - self.packages[name]["optional"] = False - # skip all packages in "nobuild" mode - # allow only upload tools and frameworks - elif "nobuild" in targets and opts.get("type") != "framework": - self.packages[name]["optional"] = True - - def get_lib_storages(self): - storages = {} - for opts in (self.frameworks or {}).values(): - if "package" not in opts: - continue - pkg_dir = self.get_package_dir(opts["package"]) - if not pkg_dir or not isdir(join(pkg_dir, "libraries")): - continue - libs_dir = join(pkg_dir, "libraries") - storages[libs_dir] = opts["package"] - libcores_dir = join(libs_dir, "__cores__") - if not isdir(libcores_dir): - continue - for item in os.listdir(libcores_dir): - libcore_dir = join(libcores_dir, item) - if not isdir(libcore_dir): - continue - storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) - - return [dict(name=name, path=path) for path, name in storages.items()] - - def on_installed(self): - pass - - def on_uninstalled(self): - pass - - def install_python_packages(self): - if not self.python_packages: - return None - click.echo( - "Installing Python packages: %s" - % ", ".join(list(self.python_packages.keys())), - ) - args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"] - for name, requirements in self.python_packages.items(): - if any(c in requirements for c in ("<", ">", "=")): - args.append("%s%s" % (name, requirements)) - else: - args.append("%s==%s" % (name, requirements)) - try: - return subprocess.call(args) == 0 - except Exception as e: # pylint: disable=broad-except - click.secho( - "Could not install Python packages -> %s" % e, fg="red", err=True - ) - - def uninstall_python_packages(self): - if not self.python_packages: - return - click.echo("Uninstalling Python packages") - args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"] - args.extend(list(self.python_packages.keys())) - try: - subprocess.call(args) == 0 - except Exception as e: # pylint: disable=broad-except - click.secho( - "Could not install Python packages -> %s" % e, fg="red", err=True - ) - - -class PlatformBoardConfig(object): - def __init__(self, manifest_path): - self._id = basename(manifest_path)[:-5] - assert isfile(manifest_path) - self.manifest_path = manifest_path - try: - self._manifest = fs.load_json(manifest_path) - except ValueError: - raise exception.InvalidBoardManifest(manifest_path) - if not set(["name", "url", "vendor"]) <= set(self._manifest): - raise exception.PlatformioException( - "Please specify name, url and vendor fields for " + manifest_path - ) - - def get(self, path, default=None): - try: - value = self._manifest - for k in path.split("."): - value = value[k] - # pylint: disable=undefined-variable - if PY2 and isinstance(value, unicode): - # cast to plain string from unicode for PY2, resolves issue in - # dev/platform when BoardConfig.get() is used in pair with - # os.path.join(file_encoding, unicode_encoding) - try: - value = value.encode("utf-8") - except UnicodeEncodeError: - pass - return value - except KeyError: - if default is not None: - return default - raise KeyError("Invalid board option '%s'" % path) - - def update(self, path, value): - newdict = None - for key in path.split(".")[::-1]: - if newdict is None: - newdict = {key: value} - else: - newdict = {key: newdict} - util.merge_dicts(self._manifest, newdict) - - def __contains__(self, key): - try: - self.get(key) - return True - except KeyError: - return False - - @property - def id(self): - return self._id - - @property - def id_(self): - return self.id - - @property - def manifest(self): - return self._manifest - - def get_brief_data(self): - result = { - "id": self.id, - "name": self._manifest["name"], - "platform": self._manifest.get("platform"), - "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), - "fcpu": int( - "".join( - [ - c - for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) - if c.isdigit() - ] - ) - ), - "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), - "rom": self._manifest.get("upload", {}).get("maximum_size", 0), - "frameworks": self._manifest.get("frameworks"), - "vendor": self._manifest["vendor"], - "url": self._manifest["url"], - } - if self._manifest.get("connectivity"): - result["connectivity"] = self._manifest.get("connectivity") - debug = self.get_debug_data() - if debug: - result["debug"] = debug - return result - - def get_debug_data(self): - if not self._manifest.get("debug", {}).get("tools"): - return None - tools = {} - for name, options in self._manifest["debug"]["tools"].items(): - tools[name] = {} - for key, value in options.items(): - if key in ("default", "onboard") and value: - tools[name][key] = value - return {"tools": tools} - - def get_debug_tool_name(self, custom=None): - debug_tools = self._manifest.get("debug", {}).get("tools") - tool_name = custom - if tool_name == "custom": - return tool_name - if not debug_tools: - telemetry.send_event("Debug", "Request", self.id) - raise DebugSupportError(self._manifest["name"]) - if tool_name: - if tool_name in debug_tools: - return tool_name - raise DebugInvalidOptionsError( - "Unknown debug tool `%s`. Please use one of `%s` or `custom`" - % (tool_name, ", ".join(sorted(list(debug_tools)))) - ) - - # automatically select best tool - data = {"default": [], "onboard": [], "external": []} - for key, value in debug_tools.items(): - if value.get("default"): - data["default"].append(key) - elif value.get("onboard"): - data["onboard"].append(key) - data["external"].append(key) - - for key, value in data.items(): - if not value: - continue - return sorted(value)[0] - - assert any(item for item in data) + raise UnknownBoard(id_) diff --git a/platformio/platform/__init__.py b/platformio/platform/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/platform/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py new file mode 100644 index 0000000000..e626eb4b83 --- /dev/null +++ b/platformio/platform/_packages.py @@ -0,0 +1,126 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.exception import UnknownPackageError + + +class PlatformPackagesMixin(object): + def install_packages( # pylint: disable=too-many-arguments + self, + with_packages=None, + without_packages=None, + skip_default_package=False, + silent=False, + force=False, + ): + with_packages = set(self.find_pkg_names(with_packages or [])) + without_packages = set(self.find_pkg_names(without_packages or [])) + + upkgs = with_packages | without_packages + ppkgs = set(self.packages) + if not upkgs.issubset(ppkgs): + raise UnknownPackageError(", ".join(upkgs - ppkgs)) + + for name, opts in self.packages.items(): + version = opts.get("version", "") + if name in without_packages: + continue + if name in with_packages or not ( + skip_default_package or opts.get("optional", False) + ): + if ":" in version: + self.pm.install( + "%s=%s" % (name, version), silent=silent, force=force + ) + else: + self.pm.install(name, version, silent=silent, force=force) + + return True + + def find_pkg_names(self, candidates): + result = [] + for candidate in candidates: + found = False + + # lookup by package types + for _name, _opts in self.packages.items(): + if _opts.get("type") == candidate: + result.append(_name) + found = True + + if ( + self.frameworks + and candidate.startswith("framework-") + and candidate[10:] in self.frameworks + ): + result.append(self.frameworks[candidate[10:]]["package"]) + found = True + + if not found: + result.append(candidate) + + return result + + def update_packages(self, only_check=False): + for name, manifest in self.get_installed_packages().items(): + requirements = self.packages[name].get("version", "") + if ":" in requirements: + _, requirements, __ = self.pm.parse_pkg_uri(requirements) + self.pm.update(manifest["__pkg_dir"], requirements, only_check) + + def get_installed_packages(self): + items = {} + for name in self.packages: + pkg_dir = self.get_package_dir(name) + if pkg_dir: + items[name] = self.pm.load_manifest(pkg_dir) + return items + + def are_outdated_packages(self): + for name, manifest in self.get_installed_packages().items(): + requirements = self.packages[name].get("version", "") + if ":" in requirements: + _, requirements, __ = self.pm.parse_pkg_uri(requirements) + if self.pm.outdated(manifest["__pkg_dir"], requirements): + return True + return False + + def get_package_dir(self, name): + version = self.packages[name].get("version", "") + if ":" in version: + return self.pm.get_package_dir( + *self.pm.parse_pkg_uri("%s=%s" % (name, version)) + ) + return self.pm.get_package_dir(name, version) + + def get_package_version(self, name): + pkg_dir = self.get_package_dir(name) + if not pkg_dir: + return None + return self.pm.load_manifest(pkg_dir).get("version") + + def dump_used_packages(self): + result = [] + for name, options in self.packages.items(): + if options.get("optional"): + continue + pkg_dir = self.get_package_dir(name) + if not pkg_dir: + continue + manifest = self.pm.load_manifest(pkg_dir) + item = {"name": manifest["name"], "version": manifest["version"]} + if manifest.get("__src_url"): + item["src_url"] = manifest.get("__src_url") + result.append(item) + return result diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py new file mode 100644 index 0000000000..39e30fce1c --- /dev/null +++ b/platformio/platform/_run.py @@ -0,0 +1,193 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import re +import sys + +import click + +from platformio import app, fs, proc, telemetry +from platformio.compat import hashlib_encode_data, is_bytes +from platformio.package.manager.core import get_core_package_dir +from platformio.platform.exception import BuildScriptNotFound + +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + + +class PlatformRunMixin(object): + + LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) + + @staticmethod + def encode_scons_arg(value): + data = base64.urlsafe_b64encode(hashlib_encode_data(value)) + return data.decode() if is_bytes(data) else data + + @staticmethod + def decode_scons_arg(data): + value = base64.urlsafe_b64decode(data) + return value.decode() if is_bytes(value) else value + + def run( # pylint: disable=too-many-arguments + self, variables, targets, silent, verbose, jobs + ): + assert isinstance(variables, dict) + assert isinstance(targets, list) + + options = self.config.items(env=variables["pioenv"], as_dict=True) + if "framework" in options: + # support PIO Core 3.0 dev/platforms + options["pioframework"] = options["framework"] + self.configure_default_packages(options, targets) + self.install_packages(silent=True) + + self._report_non_sensitive_data(options, targets) + + self.silent = silent + self.verbose = verbose or app.get_setting("force_verbose") + + if "clean" in targets: + targets = ["-c", "."] + + variables["platform_manifest"] = self.manifest_path + + if "build_script" not in variables: + variables["build_script"] = self.get_build_script() + if not os.path.isfile(variables["build_script"]): + raise BuildScriptNotFound(variables["build_script"]) + + result = self._run_scons(variables, targets, jobs) + assert "returncode" in result + + return result + + def _report_non_sensitive_data(self, options, targets): + topts = options.copy() + topts["platform_packages"] = [ + dict(name=item["name"], version=item["version"]) + for item in self.dump_used_packages() + ] + topts["platform"] = {"name": self.name, "version": self.version} + if self.src_version: + topts["platform"]["src_version"] = self.src_version + telemetry.send_run_environment(topts, targets) + + def _run_scons(self, variables, targets, jobs): + args = [ + proc.get_pythonexe_path(), + os.path.join(get_core_package_dir("tool-scons"), "script", "scons"), + "-Q", + "--warn=no-no-parallel-support", + "--jobs", + str(jobs), + "--sconstruct", + os.path.join(fs.get_source_dir(), "builder", "main.py"), + ] + args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + # pylint: disable=protected-access + args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) + args += targets + + # encode and append variables + for key, value in variables.items(): + args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) + + proc.copy_pythonpath_to_osenv() + + if targets and "menuconfig" in targets: + return proc.exec_command( + args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin + ) + + if click._compat.isatty(sys.stdout): + + def _write_and_flush(stream, data): + try: + stream.write(data) + stream.flush() + except IOError: + pass + + return proc.exec_command( + args, + stdout=proc.BuildAsyncPipe( + line_callback=self._on_stdout_line, + data_callback=lambda data: _write_and_flush(sys.stdout, data), + ), + stderr=proc.BuildAsyncPipe( + line_callback=self._on_stderr_line, + data_callback=lambda data: _write_and_flush(sys.stderr, data), + ), + ) + + return proc.exec_command( + args, + stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), + stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), + ) + + def _on_stdout_line(self, line): + if "`buildprog' is up to date." in line: + return + self._echo_line(line, level=1) + + def _on_stderr_line(self, line): + is_error = self.LINE_ERROR_RE.search(line) is not None + self._echo_line(line, level=3 if is_error else 2) + + a_pos = line.find("fatal error:") + b_pos = line.rfind(": No such file or directory") + if a_pos == -1 or b_pos == -1: + return + self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) + + def _echo_line(self, line, level): + if line.startswith("scons: "): + line = line[7:] + assert 1 <= level <= 3 + if self.silent and (level < 2 or not line): + return + fg = (None, "yellow", "red")[level - 1] + if level == 1 and "is up to date" in line: + fg = "green" + click.secho(line, fg=fg, err=level > 1, nl=False) + + @staticmethod + def _echo_missed_dependency(filename): + if "/" in filename or not filename.endswith((".h", ".hpp")): + return + banner = """ +{dots} +* Looking for {filename_styled} dependency? Check our library registry! +* +* CLI > platformio lib search "header:{filename}" +* Web > {link} +* +{dots} +""".format( + filename=filename, + filename_styled=click.style(filename, fg="cyan"), + link=click.style( + "https://platformio.org/lib/search?query=header:%s" + % quote(filename, safe=""), + fg="blue", + ), + dots="*" * (56 + len(filename)), + ) + click.echo(banner, err=True) diff --git a/platformio/platform/base.py b/platformio/platform/base.py new file mode 100644 index 0000000000..a2fcd1582c --- /dev/null +++ b/platformio/platform/base.py @@ -0,0 +1,274 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +import click +import semantic_version + +from platformio import __version__, fs, proc, util +from platformio.managers.package import PackageManager +from platformio.platform._packages import PlatformPackagesMixin +from platformio.platform._run import PlatformRunMixin +from platformio.platform.board import PlatformBoardConfig +from platformio.platform.exception import UnknownBoard +from platformio.project.config import ProjectConfig + + +class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods + PlatformPackagesMixin, PlatformRunMixin +): + + PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) + _BOARDS_CACHE = {} + + def __init__(self, manifest_path): + self.manifest_path = manifest_path + self.silent = False + self.verbose = False + + self._manifest = fs.load_json(manifest_path) + self._BOARDS_CACHE = {} + self._custom_packages = None + + self.config = ProjectConfig.get_instance() + self.pm = PackageManager( + self.config.get_optional_dir("packages"), self.package_repositories + ) + + self._src_manifest = None + src_manifest_path = self.pm.get_src_manifest_path(self.get_dir()) + if src_manifest_path: + self._src_manifest = fs.load_json(src_manifest_path) + + # if self.engines and "platformio" in self.engines: + # if self.PIO_VERSION not in semantic_version.SimpleSpec( + # self.engines['platformio']): + # raise exception.IncompatiblePlatform(self.name, + # str(self.PIO_VERSION)) + + @property + def name(self): + return self._manifest["name"] + + @property + def title(self): + return self._manifest["title"] + + @property + def description(self): + return self._manifest["description"] + + @property + def version(self): + return self._manifest["version"] + + @property + def src_version(self): + return self._src_manifest.get("version") if self._src_manifest else None + + @property + def src_url(self): + return self._src_manifest.get("url") if self._src_manifest else None + + @property + def homepage(self): + return self._manifest.get("homepage") + + @property + def repository_url(self): + return self._manifest.get("repository", {}).get("url") + + @property + def license(self): + return self._manifest.get("license") + + @property + def frameworks(self): + return self._manifest.get("frameworks") + + @property + def engines(self): + return self._manifest.get("engines") + + @property + def package_repositories(self): + return self._manifest.get("packageRepositories") + + @property + def manifest(self): + return self._manifest + + @property + def packages(self): + packages = self._manifest.get("packages", {}) + for item in self._custom_packages or []: + name = item + version = "*" + if "@" in item: + name, version = item.split("@", 2) + name = name.strip() + if name not in packages: + packages[name] = {} + packages[name].update({"version": version.strip(), "optional": False}) + return packages + + @property + def python_packages(self): + return self._manifest.get("pythonPackages") + + def get_dir(self): + return os.path.dirname(self.manifest_path) + + def get_build_script(self): + main_script = os.path.join(self.get_dir(), "builder", "main.py") + if os.path.isfile(main_script): + return main_script + raise NotImplementedError() + + def is_embedded(self): + for opts in self.packages.values(): + if opts.get("type") == "uploader": + return True + return False + + def get_boards(self, id_=None): + def _append_board(board_id, manifest_path): + config = PlatformBoardConfig(manifest_path) + if "platform" in config and config.get("platform") != self.name: + return + if "platforms" in config and self.name not in config.get("platforms"): + return + config.manifest["platform"] = self.name + self._BOARDS_CACHE[board_id] = config + + bdirs = [ + self.config.get_optional_dir("boards"), + os.path.join(self.config.get_optional_dir("core"), "boards"), + os.path.join(self.get_dir(), "boards"), + ] + + if id_ is None: + for boards_dir in bdirs: + if not os.path.isdir(boards_dir): + continue + for item in sorted(os.listdir(boards_dir)): + _id = item[:-5] + if not item.endswith(".json") or _id in self._BOARDS_CACHE: + continue + _append_board(_id, os.path.join(boards_dir, item)) + else: + if id_ not in self._BOARDS_CACHE: + for boards_dir in bdirs: + if not os.path.isdir(boards_dir): + continue + manifest_path = os.path.join(boards_dir, "%s.json" % id_) + if os.path.isfile(manifest_path): + _append_board(id_, manifest_path) + break + if id_ not in self._BOARDS_CACHE: + raise UnknownBoard(id_) + return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE + + def board_config(self, id_): + return self.get_boards(id_) + + def get_package_type(self, name): + return self.packages[name].get("type") + + def configure_default_packages(self, options, targets): + # override user custom packages + self._custom_packages = options.get("platform_packages") + + # enable used frameworks + for framework in options.get("framework", []): + if not self.frameworks: + continue + framework = framework.lower().strip() + if not framework or framework not in self.frameworks: + continue + _pkg_name = self.frameworks[framework].get("package") + if _pkg_name: + self.packages[_pkg_name]["optional"] = False + + # enable upload tools for upload targets + if any(["upload" in t for t in targets] + ["program" in targets]): + for name, opts in self.packages.items(): + if opts.get("type") == "uploader": + self.packages[name]["optional"] = False + # skip all packages in "nobuild" mode + # allow only upload tools and frameworks + elif "nobuild" in targets and opts.get("type") != "framework": + self.packages[name]["optional"] = True + + def get_lib_storages(self): + storages = {} + for opts in (self.frameworks or {}).values(): + if "package" not in opts: + continue + pkg_dir = self.get_package_dir(opts["package"]) + if not pkg_dir or not os.path.isdir(os.path.join(pkg_dir, "libraries")): + continue + libs_dir = os.path.join(pkg_dir, "libraries") + storages[libs_dir] = opts["package"] + libcores_dir = os.path.join(libs_dir, "__cores__") + if not os.path.isdir(libcores_dir): + continue + for item in os.listdir(libcores_dir): + libcore_dir = os.path.join(libcores_dir, item) + if not os.path.isdir(libcore_dir): + continue + storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) + + return [dict(name=name, path=path) for path, name in storages.items()] + + def on_installed(self): + pass + + def on_uninstalled(self): + pass + + def install_python_packages(self): + if not self.python_packages: + return None + click.echo( + "Installing Python packages: %s" + % ", ".join(list(self.python_packages.keys())), + ) + args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"] + for name, requirements in self.python_packages.items(): + if any(c in requirements for c in ("<", ">", "=")): + args.append("%s%s" % (name, requirements)) + else: + args.append("%s==%s" % (name, requirements)) + try: + return subprocess.call(args) == 0 + except Exception as e: # pylint: disable=broad-except + click.secho( + "Could not install Python packages -> %s" % e, fg="red", err=True + ) + + def uninstall_python_packages(self): + if not self.python_packages: + return + click.echo("Uninstalling Python packages") + args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"] + args.extend(list(self.python_packages.keys())) + try: + subprocess.call(args) == 0 + except Exception as e: # pylint: disable=broad-except + click.secho( + "Could not install Python packages -> %s" % e, fg="red", err=True + ) diff --git a/platformio/platform/board.py b/platformio/platform/board.py new file mode 100644 index 0000000000..900892cdf3 --- /dev/null +++ b/platformio/platform/board.py @@ -0,0 +1,158 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import fs, telemetry, util +from platformio.commands.debug.exception import ( + DebugInvalidOptionsError, + DebugSupportError, +) +from platformio.compat import PY2 +from platformio.exception import UserSideException +from platformio.platform.exception import InvalidBoardManifest + + +class PlatformBoardConfig(object): + def __init__(self, manifest_path): + self._id = os.path.basename(manifest_path)[:-5] + assert os.path.isfile(manifest_path) + self.manifest_path = manifest_path + try: + self._manifest = fs.load_json(manifest_path) + except ValueError: + raise InvalidBoardManifest(manifest_path) + if not set(["name", "url", "vendor"]) <= set(self._manifest): + raise UserSideException( + "Please specify name, url and vendor fields for " + manifest_path + ) + + def get(self, path, default=None): + try: + value = self._manifest + for k in path.split("."): + value = value[k] + # pylint: disable=undefined-variable + if PY2 and isinstance(value, unicode): + # cast to plain string from unicode for PY2, resolves issue in + # dev/platform when BoardConfig.get() is used in pair with + # os.path.join(file_encoding, unicode_encoding) + try: + value = value.encode("utf-8") + except UnicodeEncodeError: + pass + return value + except KeyError: + if default is not None: + return default + raise KeyError("Invalid board option '%s'" % path) + + def update(self, path, value): + newdict = None + for key in path.split(".")[::-1]: + if newdict is None: + newdict = {key: value} + else: + newdict = {key: newdict} + util.merge_dicts(self._manifest, newdict) + + def __contains__(self, key): + try: + self.get(key) + return True + except KeyError: + return False + + @property + def id(self): + return self._id + + @property + def id_(self): + return self.id + + @property + def manifest(self): + return self._manifest + + def get_brief_data(self): + result = { + "id": self.id, + "name": self._manifest["name"], + "platform": self._manifest.get("platform"), + "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), + "fcpu": int( + "".join( + [ + c + for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) + if c.isdigit() + ] + ) + ), + "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), + "rom": self._manifest.get("upload", {}).get("maximum_size", 0), + "frameworks": self._manifest.get("frameworks"), + "vendor": self._manifest["vendor"], + "url": self._manifest["url"], + } + if self._manifest.get("connectivity"): + result["connectivity"] = self._manifest.get("connectivity") + debug = self.get_debug_data() + if debug: + result["debug"] = debug + return result + + def get_debug_data(self): + if not self._manifest.get("debug", {}).get("tools"): + return None + tools = {} + for name, options in self._manifest["debug"]["tools"].items(): + tools[name] = {} + for key, value in options.items(): + if key in ("default", "onboard") and value: + tools[name][key] = value + return {"tools": tools} + + def get_debug_tool_name(self, custom=None): + debug_tools = self._manifest.get("debug", {}).get("tools") + tool_name = custom + if tool_name == "custom": + return tool_name + if not debug_tools: + telemetry.send_event("Debug", "Request", self.id) + raise DebugSupportError(self._manifest["name"]) + if tool_name: + if tool_name in debug_tools: + return tool_name + raise DebugInvalidOptionsError( + "Unknown debug tool `%s`. Please use one of `%s` or `custom`" + % (tool_name, ", ".join(sorted(list(debug_tools)))) + ) + + # automatically select best tool + data = {"default": [], "onboard": [], "external": []} + for key, value in debug_tools.items(): + if value.get("default"): + data["default"].append(key) + elif value.get("onboard"): + data["onboard"].append(key) + data["external"].append(key) + + for key, value in data.items(): + if not value: + continue + return sorted(value)[0] + + assert any(item for item in data) diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py new file mode 100644 index 0000000000..40431d7f41 --- /dev/null +++ b/platformio/platform/exception.py @@ -0,0 +1,49 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException + + +class PlatformException(PlatformioException): + pass + + +class UnknownPlatform(PlatformException): + + MESSAGE = "Unknown development platform '{0}'" + + +class IncompatiblePlatform(PlatformException): + + MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}" + + +class UnknownBoard(PlatformException): + + MESSAGE = "Unknown board ID '{0}'" + + +class InvalidBoardManifest(PlatformException): + + MESSAGE = "Invalid board JSON manifest '{0}'" + + +class UnknownFramework(PlatformException): + + MESSAGE = "Unknown framework '{0}'" + + +class BuildScriptNotFound(PlatformException): + + MESSAGE = "Invalid path '{0}' to build script" diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py new file mode 100644 index 0000000000..99e5f7c435 --- /dev/null +++ b/platformio/platform/factory.py @@ -0,0 +1,60 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +from platformio.compat import load_python_module +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.base import PlatformBase +from platformio.platform.exception import UnknownPlatform + + +class PlatformFactory(object): + @staticmethod + def get_clsname(name): + name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) + return "%s%sPlatform" % (name.upper()[0], name.lower()[1:]) + + @staticmethod + def load_module(name, path): + try: + return load_python_module("platformio.platform.%s" % name, path) + except ImportError: + raise UnknownPlatform(name) + + @classmethod + def new(cls, pkg_or_spec): + pkg = PlatformPackageManager().get_package( + "file://%s" % pkg_or_spec if os.path.isdir(pkg_or_spec) else pkg_or_spec + ) + if not pkg: + raise UnknownPlatform(pkg_or_spec) + + platform_cls = None + if os.path.isfile(os.path.join(pkg.path, "platform.py")): + platform_cls = getattr( + cls.load_module( + pkg.metadata.name, os.path.join(pkg.path, "platform.py") + ), + cls.get_clsname(pkg.metadata.name), + ) + else: + platform_cls = type( + str(cls.get_clsname(pkg.metadata.name)), (PlatformBase,), {} + ) + + _instance = platform_cls(os.path.join(pkg.path, "platform.json")) + assert isinstance(_instance, PlatformBase) + return _instance From 332874cd4b392e718590255fc5cc72fac5ae4e85 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 14 Aug 2020 16:48:12 +0300 Subject: [PATCH 154/223] Fix relative import of platform module on Py27 --- platformio/app.py | 2 ++ platformio/util.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/platformio/app.py b/platformio/app.py index a1a4ee7351..00a8e89f1c 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import absolute_import + import codecs import getpass import hashlib diff --git a/platformio/util.py b/platformio/util.py index 5b38909d59..a1974f1750 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import absolute_import + import json import math import os From 4ec64f89800d81437f1d931c22fa7f3ea6f10b0f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 14 Aug 2020 17:00:18 +0300 Subject: [PATCH 155/223] Fix a test for examples --- tests/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index c9f64c1880..d0a580d48c 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -35,7 +35,7 @@ def pytest_generate_tests(metafunc): # dev/platforms for manifest in PlatformManager().get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) examples_dir = join(p.get_dir(), "examples") assert isdir(examples_dir) examples_dirs.append(examples_dir) From bb6fb3fdf8a72ccfb01c85a31c19718e0f148b84 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 15 Aug 2020 15:24:35 +0300 Subject: [PATCH 156/223] Fix bug with parsing detached packages --- platformio/package/meta.py | 7 ++++++- tests/package/test_meta.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/platformio/package/meta.py b/platformio/package/meta.py index f57f049534..fa93780e3f 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -229,6 +229,8 @@ def _parse(self, raw): def _parse_requirements(self, raw): if "@" not in raw: return raw + if raw.startswith("file://") and os.path.exists(raw[7:]): + return raw tokens = raw.rsplit("@", 1) if any(s in tokens[1] for s in (":", "/")): return raw @@ -296,7 +298,10 @@ def _parse_url(self, raw): def _parse_name_from_url(url): if url.endswith("/"): url = url[:-1] - for c in ("#", "?"): + stop_chars = ["#", "?"] + if url.startswith("file://"): + stop_chars.append("@") # detached path + for c in stop_chars: if c in url: url = url[: url.index(c)] diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index d7d4b820e2..0629f274fc 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -80,7 +80,7 @@ def test_spec_requirements(): assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0") -def test_spec_local_urls(): +def test_spec_local_urls(tmpdir_factory): assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( url="file:///tmp/foo.tar.gz", name="foo" ) @@ -93,6 +93,11 @@ def test_spec_local_urls(): assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec( url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1" ) + # detached folder with "@" symbol + pkg_dir = tmpdir_factory.mktemp("storage").join("detached@1.2.3").mkdir() + assert PackageSpec("file://%s" % str(pkg_dir)) == PackageSpec( + name="detached", url="file://%s" % pkg_dir + ) def test_spec_external_urls(): From 04694b4126ba793bd24dbcd537f35f554f6973fc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 15 Aug 2020 23:11:01 +0300 Subject: [PATCH 157/223] Switch legacy platform manager to the new --- platformio/builder/tools/pioide.py | 6 +- platformio/builder/tools/pioplatform.py | 29 +- platformio/commands/boards.py | 4 +- .../commands/home/rpc/handlers/project.py | 10 +- platformio/commands/lib/helpers.py | 8 +- platformio/commands/platform.py | 115 +-- platformio/commands/project.py | 10 +- platformio/commands/system/command.py | 4 +- platformio/maintenance.py | 30 +- platformio/managers/package.py | 816 ------------------ platformio/managers/platform.py | 198 +---- platformio/package/manager/_install.py | 20 +- platformio/package/manager/_registry.py | 7 +- platformio/package/manager/_uninstall.py | 4 +- platformio/package/manager/_update.py | 45 +- platformio/package/manager/base.py | 6 + platformio/package/manager/platform.py | 164 ++++ platformio/package/manager/tool.py | 7 +- platformio/platform/_packages.py | 129 +-- platformio/platform/_run.py | 6 +- platformio/platform/base.py | 66 +- platformio/platform/exception.py | 5 +- platformio/platform/factory.py | 15 +- tests/commands/test_platform.py | 29 +- tests/test_maintenance.py | 10 +- 25 files changed, 471 insertions(+), 1272 deletions(-) delete mode 100644 platformio/managers/package.py diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 6a3d343d5c..c21b150022 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -45,10 +45,10 @@ def _dump_includes(env): # includes from toolchains p = env.PioPlatform() includes["toolchain"] = [] - for name in p.get_installed_packages(): - if p.get_package_type(name) != "toolchain": + for pkg in p.get_installed_packages(): + if p.get_package_type(pkg.metadata.name) != "toolchain": continue - toolchain_dir = glob_escape(p.get_package_dir(name)) + toolchain_dir = glob_escape(pkg.path) toolchain_incglobs = [ os.path.join(toolchain_dir, "*", "include", "c++", "*"), os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"), diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index fe7d6e28d4..740f61486e 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -22,6 +22,7 @@ from platformio import fs, util from platformio.compat import WINDOWS +from platformio.package.meta import PackageItem from platformio.platform.exception import UnknownBoard from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectOptions @@ -63,32 +64,30 @@ def GetFrameworkScript(env, framework): def LoadPioPlatform(env): p = env.PioPlatform() - installed_packages = p.get_installed_packages() # Ensure real platform name env["PIOPLATFORM"] = p.name # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH systype = util.get_systype() - for name in installed_packages: - type_ = p.get_package_type(name) + for pkg in p.get_installed_packages(): + type_ = p.get_package_type(pkg.metadata.name) if type_ not in ("toolchain", "uploader", "debugger"): continue - pkg_dir = p.get_package_dir(name) env.PrependENVPath( "PATH", - os.path.join(pkg_dir, "bin") - if os.path.isdir(os.path.join(pkg_dir, "bin")) - else pkg_dir, + os.path.join(pkg.path, "bin") + if os.path.isdir(os.path.join(pkg.path, "bin")) + else pkg.path, ) if ( not WINDOWS - and os.path.isdir(os.path.join(pkg_dir, "lib")) + and os.path.isdir(os.path.join(pkg.path, "lib")) and type_ != "toolchain" ): env.PrependENVPath( "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH", - os.path.join(pkg_dir, "lib"), + os.path.join(pkg.path, "lib"), ) # Platform specific LD Scripts @@ -133,6 +132,7 @@ def LoadPioPlatform(env): def PrintConfiguration(env): # pylint: disable=too-many-statements platform = env.PioPlatform() + pkg_metadata = PackageItem(platform.get_dir()).metadata board_config = env.BoardConfig() if "BOARD" in env else None def _get_configuration_data(): @@ -147,11 +147,12 @@ def _get_configuration_data(): ) def _get_plaform_data(): - data = ["PLATFORM: %s (%s)" % (platform.title, platform.version)] - if platform.src_version: - data.append("#" + platform.src_version) - if int(ARGUMENTS.get("PIOVERBOSE", 0)) and platform.src_url: - data.append("(%s)" % platform.src_url) + data = [ + "PLATFORM: %s (%s)" + % (platform.title, pkg_metadata.version or platform.version) + ] + if int(ARGUMENTS.get("PIOVERBOSE", 0)) and pkg_metadata.spec.external: + data.append("(%s)" % pkg_metadata.spec.url) if board_config: data.extend([">", board_config.get("name")]) return data diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index e7e2ecd6bb..21614b1356 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -19,7 +19,7 @@ from platformio import fs from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager @click.command("boards", short_help="Embedded Board Explorer") @@ -71,7 +71,7 @@ def print_boards(boards): def _get_boards(installed=False): - pm = PlatformManager() + pm = PlatformPackageManager() return pm.get_installed_boards() if installed else pm.get_all_boards() diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 3f4cdc88f5..2db966b699 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -25,7 +25,7 @@ from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.compat import PY2, get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError from platformio.project.helpers import get_project_dir, is_platformio_project @@ -105,7 +105,7 @@ def _path_to_name(path): return (os.path.sep).join(path.split(os.path.sep)[-2:]) result = [] - pm = PlatformManager() + pm = PlatformPackageManager() for project_dir in AppRPC.load_state()["storage"]["recentProjects"]: if not os.path.isdir(project_dir): continue @@ -148,8 +148,9 @@ def _path_to_name(path): @staticmethod def get_project_examples(): result = [] - for manifest in PlatformManager().get_installed(): - examples_dir = os.path.join(manifest["__pkg_dir"], "examples") + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + examples_dir = os.path.join(pkg.path, "examples") if not os.path.isdir(examples_dir): continue items = [] @@ -172,6 +173,7 @@ def get_project_examples(): "description": project_description, } ) + manifest = pm.load_manifest(pkg) result.append( { "platform": { diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py index 23892ac854..a5b0e260d8 100644 --- a/platformio/commands/lib/helpers.py +++ b/platformio/commands/lib/helpers.py @@ -15,7 +15,7 @@ import os from platformio.compat import ci_strings_are_equal -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.package.meta import PackageSpec from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -28,9 +28,9 @@ def get_builtin_libs(storage_names=None): items = [] storage_names = storage_names or [] - pm = PlatformManager() - for manifest in pm.get_installed(): - p = PlatformFactory.new(manifest["__pkg_dir"]) + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + p = PlatformFactory.new(pkg) for storage in p.get_lib_storages(): if storage_names and storage["name"] not in storage_names: continue diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 14d5a1f29e..b996a16afa 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import dirname, isdir +import os import click from platformio import app, util from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.meta import PackageItem, PackageSpec from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory @@ -48,7 +49,7 @@ def _print_platforms(platforms): if "version" in platform: if "__src_url" in platform: click.echo( - "Version: #%s (%s)" % (platform["version"], platform["__src_url"]) + "Version: %s (%s)" % (platform["version"], platform["__src_url"]) ) else: click.echo("Version: " + platform["version"]) @@ -56,11 +57,7 @@ def _print_platforms(platforms): def _get_registry_platforms(): - platforms = util.get_api_result("/platforms", cache_valid="7d") - pm = PlatformManager() - for platform in platforms or []: - platform["versions"] = pm.get_all_repo_versions(platform["name"]) - return platforms + return util.get_api_result("/platforms", cache_valid="7d") def _get_platform_data(*args, **kwargs): @@ -91,7 +88,9 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru # return data # overwrite VCS version and add extra fields - manifest = PlatformManager().load_manifest(dirname(p.manifest_path)) + manifest = PlatformPackageManager().legacy_load_manifest( + os.path.dirname(p.manifest_path) + ) assert manifest for key in manifest: if key == "version" or key.startswith("__"): @@ -104,13 +103,15 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru return data data["packages"] = [] - installed_pkgs = p.get_installed_packages() - for name, opts in p.packages.items(): + installed_pkgs = { + pkg.metadata.name: p.pm.load_manifest(pkg) for pkg in p.get_installed_packages() + } + for name, options in p.packages.items(): item = dict( name=name, type=p.get_package_type(name), - requirements=opts.get("version"), - optional=opts.get("optional") is True, + requirements=options.get("version"), + optional=options.get("optional") is True, ) if name in installed_pkgs: for key, value in installed_pkgs[name].items(): @@ -147,13 +148,13 @@ def _get_registry_platform_data( # pylint: disable=unused-argument forDesktop=_data["forDesktop"], frameworks=_data["frameworks"], packages=_data["packages"], - versions=_data["versions"], + versions=_data.get("versions"), ) if with_boards: data["boards"] = [ board - for board in PlatformManager().get_registered_boards() + for board in PlatformPackageManager().get_registered_boards() if board["platform"] == _data["name"] ] @@ -213,12 +214,10 @@ def platform_frameworks(query, json_output): @click.option("--json-output", is_flag=True) def platform_list(json_output): platforms = [] - pm = PlatformManager() - for manifest in pm.get_installed(): + pm = PlatformPackageManager() + for pkg in pm.get_installed(): platforms.append( - _get_installed_platform_data( - manifest["__pkg_dir"], with_boards=False, expose_packages=False - ) + _get_installed_platform_data(pkg, with_boards=False, expose_packages=False) ) platforms = sorted(platforms, key=lambda manifest: manifest["name"]) @@ -300,6 +299,7 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches @click.option("--without-package", multiple=True) @click.option("--skip-default-package", is_flag=True) @click.option("--with-all-packages", is_flag=True) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option( "-f", "--force", @@ -312,21 +312,24 @@ def platform_install( # pylint: disable=too-many-arguments without_package, skip_default_package, with_all_packages, + silent, force, ): - pm = PlatformManager() + pm = PlatformPackageManager() for platform in platforms: - if pm.install( - name=platform, + pkg = pm.install( + spec=platform, with_packages=with_package, without_packages=without_package, skip_default_package=skip_default_package, with_all_packages=with_all_packages, + silent=silent, force=force, - ): + ) + if pkg and not silent: click.secho( "The platform '%s' has been successfully installed!\n" - "The rest of packages will be installed automatically " + "The rest of the packages will be installed later " "depending on your build environment." % platform, fg="green", ) @@ -335,11 +338,11 @@ def platform_install( # pylint: disable=too-many-arguments @cli.command("uninstall", short_help="Uninstall development platform") @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]") def platform_uninstall(platforms): - pm = PlatformManager() + pm = PlatformPackageManager() for platform in platforms: if pm.uninstall(platform): click.secho( - "The platform '%s' has been successfully uninstalled!" % platform, + "The platform '%s' has been successfully removed!" % platform, fg="green", ) @@ -358,41 +361,40 @@ def platform_uninstall(platforms): @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option("--json-output", is_flag=True) -def platform_update( # pylint: disable=too-many-locals - platforms, only_packages, only_check, dry_run, json_output +def platform_update( # pylint: disable=too-many-locals, too-many-arguments + platforms, only_packages, only_check, dry_run, silent, json_output ): - pm = PlatformManager() - pkg_dir_to_name = {} - if not platforms: - platforms = [] - for manifest in pm.get_installed(): - platforms.append(manifest["__pkg_dir"]) - pkg_dir_to_name[manifest["__pkg_dir"]] = manifest.get( - "title", manifest["name"] - ) - + pm = PlatformPackageManager() + platforms = platforms or pm.get_installed() only_check = dry_run or only_check if only_check and json_output: result = [] for platform in platforms: - pkg_dir = platform if isdir(platform) else None - requirements = None - url = None - if not pkg_dir: - name, requirements, url = pm.parse_pkg_uri(platform) - pkg_dir = pm.get_package_dir(name, requirements, url) - if not pkg_dir: + spec = None + pkg = None + if isinstance(platform, PackageItem): + pkg = platform + else: + spec = PackageSpec(platform) + pkg = pm.get_package(spec) + if not pkg: continue - latest = pm.outdated(pkg_dir, requirements) - if not latest and not PlatformFactory.new(pkg_dir).are_outdated_packages(): + outdated = pm.outdated(pkg, spec) + if ( + not outdated.is_outdated(allow_incompatible=True) + and not PlatformFactory.new(pkg).are_outdated_packages() + ): continue data = _get_installed_platform_data( - pkg_dir, with_boards=False, expose_packages=False + pkg, with_boards=False, expose_packages=False ) - if latest: - data["versionLatest"] = latest + if outdated.is_outdated(allow_incompatible=True): + data["versionLatest"] = ( + str(outdated.latest) if outdated.latest else None + ) result.append(data) return click.echo(dump_json_to_unicode(result)) @@ -401,10 +403,17 @@ def platform_update( # pylint: disable=too-many-locals for platform in platforms: click.echo( "Platform %s" - % click.style(pkg_dir_to_name.get(platform, platform), fg="cyan") + % click.style( + platform.metadata.name + if isinstance(platform, PackageItem) + else platform, + fg="cyan", + ) ) click.echo("--------") - pm.update(platform, only_packages=only_packages, only_check=only_check) + pm.update( + platform, only_packages=only_packages, only_check=only_check, silent=silent + ) click.echo() return True diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 6194a915a8..6900ce74ac 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -23,7 +23,7 @@ from platformio import fs from platformio.commands.platform import platform_install as cli_platform_install from platformio.ide.projectgenerator import ProjectGenerator -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.exception import UnknownBoard from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError @@ -109,7 +109,7 @@ def project_idedata(project_dir, environment, json_output): def validate_boards(ctx, param, value): # pylint: disable=W0613 - pm = PlatformManager() + pm = PlatformPackageManager() for id_ in value: try: pm.board_config(id_) @@ -367,7 +367,7 @@ def fill_project_envs( if all(cond): used_boards.append(config.get(section, "board")) - pm = PlatformManager() + pm = PlatformPackageManager() used_platforms = [] modified = False for id_ in board_ids: @@ -404,7 +404,9 @@ def fill_project_envs( def _install_dependent_platforms(ctx, platforms): - installed_platforms = [p["name"] for p in PlatformManager().get_installed()] + installed_platforms = [ + pkg.metadata.name for pkg in PlatformPackageManager().get_installed() + ] if set(platforms) <= set(installed_platforms): return ctx.invoke( diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index 76d2cb3687..2fee54710c 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -26,8 +26,8 @@ install_completion_code, uninstall_completion_code, ) -from platformio.managers.platform import PlatformManager from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig @@ -77,7 +77,7 @@ def system_info(json_output): } data["dev_platform_nums"] = { "title": "Development Platforms", - "value": len(PlatformManager().get_installed()), + "value": len(PlatformPackageManager().get_installed()), } data["package_tool_nums"] = { "title": "Tools & Toolchains", diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 4b47d50a39..54b0ad6d21 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -25,9 +25,9 @@ from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.managers.platform import PlatformManager from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec from platformio.platform.factory import PlatformFactory @@ -271,24 +271,16 @@ def check_internal_updates(ctx, what): # pylint: disable=too-many-branches util.internet_on(raise_exception=True) outdated_items = [] - pm = PlatformManager() if what == "platforms" else LibraryPackageManager() - if isinstance(pm, PlatformManager): - for manifest in pm.get_installed(): - if manifest["name"] in outdated_items: - continue - conds = [ - pm.outdated(manifest["__pkg_dir"]), - what == "platforms" - and PlatformFactory.new(manifest["__pkg_dir"]).are_outdated_packages(), - ] - if any(conds): - outdated_items.append(manifest["name"]) - else: - for pkg in pm.get_installed(): - if pkg.metadata.name in outdated_items: - continue - if pm.outdated(pkg).is_outdated(): - outdated_items.append(pkg.metadata.name) + pm = PlatformPackageManager() if what == "platforms" else LibraryPackageManager() + for pkg in pm.get_installed(): + if pkg.metadata.name in outdated_items: + continue + conds = [ + pm.outdated(pkg).is_outdated(), + what == "platforms" and PlatformFactory.new(pkg).are_outdated_packages(), + ] + if any(conds): + outdated_items.append(pkg.metadata.name) if not outdated_items: return diff --git a/platformio/managers/package.py b/platformio/managers/package.py deleted file mode 100644 index 071d67886d..0000000000 --- a/platformio/managers/package.py +++ /dev/null @@ -1,816 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import hashlib -import json -import os -import re -import shutil -from os.path import basename, getsize, isdir, isfile, islink, join, realpath -from tempfile import mkdtemp - -import click -import requests -import semantic_version - -from platformio import __version__, app, exception, fs, util -from platformio.compat import hashlib_encode_data -from platformio.package.download import FileDownloader -from platformio.package.exception import ManifestException -from platformio.package.lockfile import LockFile -from platformio.package.manifest.parser import ManifestParserFactory -from platformio.package.unpack import FileUnpacker -from platformio.package.vcsclient import VCSClientFactory - -# pylint: disable=too-many-arguments, too-many-return-statements - - -class PackageRepoIterator(object): - def __init__(self, package, repositories): - assert isinstance(repositories, list) - self.package = package - self.repositories = iter(repositories) - - def __iter__(self): - return self - - def __next__(self): - return self.next() # pylint: disable=not-callable - - @staticmethod - @util.memoized(expire="60s") - def load_manifest(url): - r = None - try: - r = requests.get(url, headers={"User-Agent": app.get_user_agent()}) - r.raise_for_status() - return r.json() - except: # pylint: disable=bare-except - pass - finally: - if r: - r.close() - return None - - def next(self): - repo = next(self.repositories) - manifest = repo if isinstance(repo, dict) else self.load_manifest(repo) - if manifest and self.package in manifest: - return manifest[self.package] - return next(self) - - -class PkgRepoMixin(object): - - PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) - - @staticmethod - def is_system_compatible(valid_systems): - if not valid_systems or "*" in valid_systems: - return True - if not isinstance(valid_systems, list): - valid_systems = list([valid_systems]) - return util.get_systype() in valid_systems - - def max_satisfying_repo_version(self, versions, requirements=None): - item = None - reqspec = None - try: - reqspec = ( - semantic_version.SimpleSpec(requirements) if requirements else None - ) - except ValueError: - pass - - for v in versions: - if not self.is_system_compatible(v.get("system")): - continue - # if "platformio" in v.get("engines", {}): - # if PkgRepoMixin.PIO_VERSION not in requirements.SimpleSpec( - # v['engines']['platformio']): - # continue - specver = semantic_version.Version(v["version"]) - if reqspec and specver not in reqspec: - continue - if not item or semantic_version.Version(item["version"]) < specver: - item = v - return item - - def get_latest_repo_version( # pylint: disable=unused-argument - self, name, requirements, silent=False - ): - version = None - for versions in PackageRepoIterator(name, self.repositories): - pkgdata = self.max_satisfying_repo_version(versions, requirements) - if not pkgdata: - continue - if ( - not version - or semantic_version.compare(pkgdata["version"], version) == 1 - ): - version = pkgdata["version"] - return version - - def get_all_repo_versions(self, name): - result = [] - for versions in PackageRepoIterator(name, self.repositories): - result.extend([semantic_version.Version(v["version"]) for v in versions]) - return [str(v) for v in sorted(set(result))] - - -class PkgInstallerMixin(object): - - SRC_MANIFEST_NAME = ".piopkgmanager.json" - TMP_FOLDER_PREFIX = "_tmp_installing-" - - FILE_CACHE_VALID = None # for example, 1 week = "7d" - FILE_CACHE_MAX_SIZE = 1024 * 1024 * 50 # 50 Mb - - MEMORY_CACHE = {} # cache for package manifests and read dirs - - def cache_get(self, key, default=None): - return self.MEMORY_CACHE.get(key, default) - - def cache_set(self, key, value): - self.MEMORY_CACHE[key] = value - - def cache_reset(self): - self.MEMORY_CACHE.clear() - - def read_dirs(self, src_dir): - cache_key = "read_dirs-%s" % src_dir - result = self.cache_get(cache_key) - if result: - return result - result = [ - join(src_dir, name) - for name in sorted(os.listdir(src_dir)) - if isdir(join(src_dir, name)) - ] - self.cache_set(cache_key, result) - return result - - def download(self, url, dest_dir, sha1=None): - cache_key_fname = app.ContentCache.key_from_args(url, "fname") - cache_key_data = app.ContentCache.key_from_args(url, "data") - if self.FILE_CACHE_VALID: - with app.ContentCache() as cc: - fname = str(cc.get(cache_key_fname)) - cache_path = cc.get_cache_path(cache_key_data) - if fname and isfile(cache_path): - dst_path = join(dest_dir, fname) - shutil.copy(cache_path, dst_path) - click.echo("Using cache: %s" % cache_path) - return dst_path - - with_progress = not app.is_disabled_progressbar() - try: - fd = FileDownloader(url, dest_dir) - fd.start(with_progress=with_progress) - except IOError as e: - raise_error = not with_progress - if with_progress: - try: - fd = FileDownloader(url, dest_dir) - fd.start(with_progress=False) - except IOError: - raise_error = True - if raise_error: - click.secho( - "Error: Please read http://bit.ly/package-manager-ioerror", - fg="red", - err=True, - ) - raise e - - if sha1: - fd.verify(sha1) - dst_path = fd.get_filepath() - if ( - not self.FILE_CACHE_VALID - or getsize(dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE - ): - return dst_path - - with app.ContentCache() as cc: - cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID) - cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID) - shutil.copy(dst_path, cc.get_cache_path(cache_key_data)) - return dst_path - - @staticmethod - def unpack(source_path, dest_dir): - with_progress = not app.is_disabled_progressbar() - try: - with FileUnpacker(source_path) as fu: - return fu.unpack(dest_dir, with_progress=with_progress) - except IOError as e: - if not with_progress: - raise e - with FileUnpacker(source_path) as fu: - return fu.unpack(dest_dir, with_progress=False) - - @staticmethod - def parse_semver_version(value, raise_exception=False): - try: - try: - return semantic_version.Version(value) - except ValueError: - if "." not in str(value) and not str(value).isdigit(): - raise ValueError("Invalid SemVer version %s" % value) - return semantic_version.Version.coerce(value) - except ValueError as e: - if raise_exception: - raise e - return None - - @staticmethod - def parse_pkg_uri(text, requirements=None): # pylint: disable=too-many-branches - text = str(text) - name, url = None, None - - # Parse requirements - req_conditions = [ - "@" in text, - not requirements, - ":" not in text or text.rfind("/") < text.rfind("@"), - ] - if all(req_conditions): - text, requirements = text.rsplit("@", 1) - - # Handle PIO Library Registry ID - if text.isdigit(): - text = "id=" + text - # Parse custom name - elif "=" in text and not text.startswith("id="): - name, text = text.split("=", 1) - - # Parse URL - # if valid URL with scheme vcs+protocol:// - if "+" in text and text.find("+") < text.find("://"): - url = text - elif "/" in text or "\\" in text: - git_conditions = [ - # Handle GitHub URL (https://github.com/user/package) - text.startswith("https://github.com/") - and not text.endswith((".zip", ".tar.gz")), - (text.split("#", 1)[0] if "#" in text else text).endswith(".git"), - ] - hg_conditions = [ - # Handle Developer Mbed URL - # (https://developer.mbed.org/users/user/code/package/) - # (https://os.mbed.com/users/user/code/package/) - text.startswith("https://developer.mbed.org"), - text.startswith("https://os.mbed.com"), - ] - if any(git_conditions): - url = "git+" + text - elif any(hg_conditions): - url = "hg+" + text - elif "://" not in text and (isfile(text) or isdir(text)): - url = "file://" + text - elif "://" in text: - url = text - # Handle short version of GitHub URL - elif text.count("/") == 1: - url = "git+https://github.com/" + text - - # Parse name from URL - if url and not name: - _url = url.split("#", 1)[0] if "#" in url else url - if _url.endswith(("\\", "/")): - _url = _url[:-1] - name = basename(_url) - if "." in name and not name.startswith("."): - name = name.rsplit(".", 1)[0] - - return (name or text, requirements, url) - - @staticmethod - def get_install_dirname(manifest): - name = re.sub(r"[^\da-z\_\-\. ]", "_", manifest["name"], flags=re.I) - if "id" in manifest: - name += "_ID%d" % manifest["id"] - return str(name) - - @classmethod - def get_src_manifest_path(cls, pkg_dir): - if not isdir(pkg_dir): - return None - for item in os.listdir(pkg_dir): - if not isdir(join(pkg_dir, item)): - continue - if isfile(join(pkg_dir, item, cls.SRC_MANIFEST_NAME)): - return join(pkg_dir, item, cls.SRC_MANIFEST_NAME) - return None - - def get_manifest_path(self, pkg_dir): - if not isdir(pkg_dir): - return None - for name in self.manifest_names: - manifest_path = join(pkg_dir, name) - if isfile(manifest_path): - return manifest_path - return None - - def manifest_exists(self, pkg_dir): - return self.get_manifest_path(pkg_dir) or self.get_src_manifest_path(pkg_dir) - - def load_manifest(self, pkg_dir): # pylint: disable=too-many-branches - cache_key = "load_manifest-%s" % pkg_dir - result = self.cache_get(cache_key) - if result: - return result - - manifest = {} - src_manifest = None - manifest_path = self.get_manifest_path(pkg_dir) - src_manifest_path = self.get_src_manifest_path(pkg_dir) - if src_manifest_path: - src_manifest = fs.load_json(src_manifest_path) - - if not manifest_path and not src_manifest_path: - return None - - try: - manifest = ManifestParserFactory.new_from_file(manifest_path).as_dict() - except ManifestException: - pass - - if src_manifest: - if "version" in src_manifest: - manifest["version"] = src_manifest["version"] - manifest["__src_url"] = src_manifest["url"] - # handle a custom package name - autogen_name = self.parse_pkg_uri(manifest["__src_url"])[0] - if "name" not in manifest or autogen_name != src_manifest["name"]: - manifest["name"] = src_manifest["name"] - - if "name" not in manifest: - manifest["name"] = basename(pkg_dir) - if "version" not in manifest: - manifest["version"] = "0.0.0" - - manifest["__pkg_dir"] = realpath(pkg_dir) - self.cache_set(cache_key, manifest) - return manifest - - def get_installed(self): - items = [] - for pkg_dir in self.read_dirs(self.package_dir): - if self.TMP_FOLDER_PREFIX in pkg_dir: - continue - manifest = self.load_manifest(pkg_dir) - if not manifest: - continue - assert "name" in manifest - items.append(manifest) - return items - - def get_package(self, name, requirements=None, url=None): - pkg_id = int(name[3:]) if name.startswith("id=") else 0 - best = None - for manifest in self.get_installed(): - if url: - if manifest.get("__src_url") != url: - continue - elif pkg_id and manifest.get("id") != pkg_id: - continue - elif not pkg_id and manifest["name"] != name: - continue - elif not PkgRepoMixin.is_system_compatible(manifest.get("system")): - continue - - # strict version or VCS HASH - if requirements and requirements == manifest["version"]: - return manifest - - try: - if requirements and not semantic_version.SimpleSpec(requirements).match( - self.parse_semver_version(manifest["version"], raise_exception=True) - ): - continue - if not best or ( - self.parse_semver_version(manifest["version"], raise_exception=True) - > self.parse_semver_version(best["version"], raise_exception=True) - ): - best = manifest - except ValueError: - pass - - return best - - def get_package_dir(self, name, requirements=None, url=None): - manifest = self.get_package(name, requirements, url) - return ( - manifest.get("__pkg_dir") - if manifest and isdir(manifest.get("__pkg_dir")) - else None - ) - - def get_package_by_dir(self, pkg_dir): - for manifest in self.get_installed(): - if manifest["__pkg_dir"] == realpath(pkg_dir): - return manifest - return None - - def find_pkg_root(self, src_dir): - if self.manifest_exists(src_dir): - return src_dir - for root, _, _ in os.walk(src_dir): - if self.manifest_exists(root): - return root - raise exception.MissingPackageManifest(", ".join(self.manifest_names)) - - def _install_from_piorepo(self, name, requirements): - pkg_dir = None - pkgdata = None - versions = None - last_exc = None - for versions in PackageRepoIterator(name, self.repositories): - pkgdata = self.max_satisfying_repo_version(versions, requirements) - if not pkgdata: - continue - try: - pkg_dir = self._install_from_url( - name, pkgdata["url"], requirements, pkgdata.get("sha1") - ) - break - except Exception as e: # pylint: disable=broad-except - last_exc = e - click.secho("Warning! Package Mirror: %s" % e, fg="yellow") - click.secho("Looking for another mirror...", fg="yellow") - - if versions is None: - util.internet_on(raise_exception=True) - raise exception.UnknownPackage( - name + (". Error -> %s" % last_exc if last_exc else "") - ) - if not pkgdata: - raise exception.UndefinedPackageVersion( - requirements or "latest", util.get_systype() - ) - return pkg_dir - - def _install_from_url(self, name, url, requirements=None, sha1=None, track=False): - tmp_dir = mkdtemp("-package", self.TMP_FOLDER_PREFIX, self.package_dir) - src_manifest_dir = None - src_manifest = {"name": name, "url": url, "requirements": requirements} - - try: - if url.startswith("file://"): - _url = url[7:] - if isfile(_url): - self.unpack(_url, tmp_dir) - else: - fs.rmtree(tmp_dir) - shutil.copytree(_url, tmp_dir, symlinks=True) - elif url.startswith(("http://", "https://")): - dlpath = self.download(url, tmp_dir, sha1) - assert isfile(dlpath) - self.unpack(dlpath, tmp_dir) - os.remove(dlpath) - else: - vcs = VCSClientFactory.new(tmp_dir, url) - assert vcs.export() - src_manifest_dir = vcs.storage_dir - src_manifest["version"] = vcs.get_current_revision() - - _tmp_dir = tmp_dir - if not src_manifest_dir: - _tmp_dir = self.find_pkg_root(tmp_dir) - src_manifest_dir = join(_tmp_dir, ".pio") - - # write source data to a special manifest - if track: - self._update_src_manifest(src_manifest, src_manifest_dir) - - return self._install_from_tmp_dir(_tmp_dir, requirements) - finally: - if isdir(tmp_dir): - fs.rmtree(tmp_dir) - return None - - def _update_src_manifest(self, data, src_dir): - if not isdir(src_dir): - os.makedirs(src_dir) - src_manifest_path = join(src_dir, self.SRC_MANIFEST_NAME) - _data = {} - if isfile(src_manifest_path): - _data = fs.load_json(src_manifest_path) - _data.update(data) - with open(src_manifest_path, "w") as fp: - json.dump(_data, fp) - - def _install_from_tmp_dir( # pylint: disable=too-many-branches - self, tmp_dir, requirements=None - ): - tmp_manifest = self.load_manifest(tmp_dir) - assert set(["name", "version"]) <= set(tmp_manifest) - - pkg_dirname = self.get_install_dirname(tmp_manifest) - pkg_dir = join(self.package_dir, pkg_dirname) - cur_manifest = self.load_manifest(pkg_dir) - - tmp_semver = self.parse_semver_version(tmp_manifest["version"]) - cur_semver = None - if cur_manifest: - cur_semver = self.parse_semver_version(cur_manifest["version"]) - - # package should satisfy requirements - if requirements: - mismatch_error = "Package version %s doesn't satisfy requirements %s" % ( - tmp_manifest["version"], - requirements, - ) - try: - assert tmp_semver and tmp_semver in semantic_version.SimpleSpec( - requirements - ), mismatch_error - except (AssertionError, ValueError): - assert tmp_manifest["version"] == requirements, mismatch_error - - # check if package already exists - if cur_manifest: - # 0-overwrite, 1-rename, 2-fix to a version - action = 0 - if "__src_url" in cur_manifest: - if cur_manifest["__src_url"] != tmp_manifest.get("__src_url"): - action = 1 - elif "__src_url" in tmp_manifest: - action = 2 - else: - if tmp_semver and (not cur_semver or tmp_semver > cur_semver): - action = 1 - elif tmp_semver and cur_semver and tmp_semver != cur_semver: - action = 2 - - # rename - if action == 1: - target_dirname = "%s@%s" % (pkg_dirname, cur_manifest["version"]) - if "__src_url" in cur_manifest: - target_dirname = "%s@src-%s" % ( - pkg_dirname, - hashlib.md5( - hashlib_encode_data(cur_manifest["__src_url"]) - ).hexdigest(), - ) - shutil.move(pkg_dir, join(self.package_dir, target_dirname)) - # fix to a version - elif action == 2: - target_dirname = "%s@%s" % (pkg_dirname, tmp_manifest["version"]) - if "__src_url" in tmp_manifest: - target_dirname = "%s@src-%s" % ( - pkg_dirname, - hashlib.md5( - hashlib_encode_data(tmp_manifest["__src_url"]) - ).hexdigest(), - ) - pkg_dir = join(self.package_dir, target_dirname) - - # remove previous/not-satisfied package - if isdir(pkg_dir): - fs.rmtree(pkg_dir) - shutil.copytree(tmp_dir, pkg_dir, symlinks=True) - try: - shutil.rmtree(tmp_dir) - except: # pylint: disable=bare-except - pass - assert isdir(pkg_dir) - self.cache_reset() - return pkg_dir - - -class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): - - # Handle circle dependencies - INSTALL_HISTORY = None - - def __init__(self, package_dir, repositories=None): - self.repositories = repositories - self.package_dir = package_dir - if not isdir(self.package_dir): - os.makedirs(self.package_dir) - assert isdir(self.package_dir) - - @property - def manifest_names(self): - raise NotImplementedError() - - def print_message(self, message, nl=True): - click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl) - - def outdated(self, pkg_dir, requirements=None): - """ - Has 3 different results: - `None` - unknown package, VCS is detached to commit - `False` - package is up-to-date - `String` - a found latest version - """ - if not isdir(pkg_dir): - return None - latest = None - manifest = self.load_manifest(pkg_dir) - # skip detached package to a specific version - if "@" in pkg_dir and "__src_url" not in manifest and not requirements: - return None - - if "__src_url" in manifest: - try: - vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"], silent=True) - except (AttributeError, exception.PlatformioException): - return None - if not vcs.can_be_updated: - return None - latest = vcs.get_latest_revision() - else: - try: - latest = self.get_latest_repo_version( - "id=%d" % manifest["id"] if "id" in manifest else manifest["name"], - requirements, - silent=True, - ) - except (exception.PlatformioException, ValueError): - return None - - if not latest: - return None - - up_to_date = False - try: - assert "__src_url" not in manifest - up_to_date = self.parse_semver_version( - manifest["version"], raise_exception=True - ) >= self.parse_semver_version(latest, raise_exception=True) - except (AssertionError, ValueError): - up_to_date = latest == manifest["version"] - - return False if up_to_date else latest - - def install( - self, name, requirements=None, silent=False, after_update=False, force=False - ): # pylint: disable=unused-argument - pkg_dir = None - # interprocess lock - with LockFile(self.package_dir): - self.cache_reset() - - name, requirements, url = self.parse_pkg_uri(name, requirements) - package_dir = self.get_package_dir(name, requirements, url) - - # avoid circle dependencies - if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = [] - history_key = "%s-%s-%s" % (name, requirements or "", url or "") - if history_key in self.INSTALL_HISTORY: - return package_dir - self.INSTALL_HISTORY.append(history_key) - - if package_dir and force: - self.uninstall(package_dir) - package_dir = None - - if not package_dir or not silent: - msg = "Installing " + click.style(name, fg="cyan") - if requirements: - msg += " @ " + requirements - self.print_message(msg) - if package_dir: - if not silent: - click.secho( - "{name} @ {version} is already installed".format( - **self.load_manifest(package_dir) - ), - fg="yellow", - ) - return package_dir - - if url: - pkg_dir = self._install_from_url(name, url, requirements, track=True) - else: - pkg_dir = self._install_from_piorepo(name, requirements) - - if not pkg_dir or not self.manifest_exists(pkg_dir): - raise exception.PackageInstallError( - name, requirements or "*", util.get_systype() - ) - - manifest = self.load_manifest(pkg_dir) - assert manifest - - click.secho( - "{name} @ {version} has been successfully installed!".format( - **manifest - ), - fg="green", - ) - - return pkg_dir - - def uninstall( - self, package, requirements=None, after_update=False - ): # pylint: disable=unused-argument - # interprocess lock - with LockFile(self.package_dir): - self.cache_reset() - - if isdir(package) and self.get_package_by_dir(package): - pkg_dir = package - else: - name, requirements, url = self.parse_pkg_uri(package, requirements) - pkg_dir = self.get_package_dir(name, requirements, url) - - if not pkg_dir: - raise exception.UnknownPackage( - "%s @ %s" % (package, requirements or "*") - ) - - manifest = self.load_manifest(pkg_dir) - click.echo( - "Uninstalling %s @ %s: \t" - % (click.style(manifest["name"], fg="cyan"), manifest["version"]), - nl=False, - ) - - if islink(pkg_dir): - os.unlink(pkg_dir) - else: - fs.rmtree(pkg_dir) - self.cache_reset() - - # unfix package with the same name - pkg_dir = self.get_package_dir(manifest["name"]) - if pkg_dir and "@" in pkg_dir: - shutil.move( - pkg_dir, join(self.package_dir, self.get_install_dirname(manifest)) - ) - self.cache_reset() - - click.echo("[%s]" % click.style("OK", fg="green")) - - return True - - def update(self, package, requirements=None, only_check=False): - self.cache_reset() - if isdir(package) and self.get_package_by_dir(package): - pkg_dir = package - else: - pkg_dir = self.get_package_dir(*self.parse_pkg_uri(package)) - - if not pkg_dir: - raise exception.UnknownPackage("%s @ %s" % (package, requirements or "*")) - - manifest = self.load_manifest(pkg_dir) - name = manifest["name"] - - click.echo( - "{} {:<40} @ {:<15}".format( - "Checking" if only_check else "Updating", - click.style(manifest["name"], fg="cyan"), - manifest["version"], - ), - nl=False, - ) - if not util.internet_on(): - click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) - return None - - latest = self.outdated(pkg_dir, requirements) - if latest: - click.echo("[%s]" % (click.style(latest, fg="red"))) - elif latest is False: - click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) - else: - click.echo("[%s]" % (click.style("Detached", fg="yellow"))) - - if only_check or not latest: - return True - - if "__src_url" in manifest: - vcs = VCSClientFactory.new(pkg_dir, manifest["__src_url"]) - assert vcs.update() - self._update_src_manifest( - dict(version=vcs.get_current_revision()), vcs.storage_dir - ) - else: - self.uninstall(pkg_dir, after_update=True) - self.install(name, latest, after_update=True) - - return True - - -class PackageManager(BasePkgManager): - @property - def manifest_names(self): - return ["package.json"] diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index c0e0f98eb9..9ef5e646c6 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -12,201 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-public-methods, too-many-instance-attributes - - -from os.path import isdir, isfile, join - -from platformio import app, exception, util -from platformio.managers.package import BasePkgManager, PackageManager +# Backward compatibility with legacy dev-platforms from platformio.platform.base import PlatformBase # pylint: disable=unused-import -from platformio.platform.exception import UnknownBoard, UnknownPlatform -from platformio.platform.factory import PlatformFactory -from platformio.project.config import ProjectConfig - - -class PlatformManager(BasePkgManager): - def __init__(self, package_dir=None, repositories=None): - if not repositories: - repositories = [ - "https://dl.bintray.com/platformio/dl-platforms/manifest.json", - "{0}://dl.platformio.org/platforms/manifest.json".format( - "https" if app.get_setting("strict_ssl") else "http" - ), - ] - self.config = ProjectConfig.get_instance() - BasePkgManager.__init__( - self, package_dir or self.config.get_optional_dir("platforms"), repositories - ) - - @property - def manifest_names(self): - return ["platform.json"] - - def get_manifest_path(self, pkg_dir): - if not isdir(pkg_dir): - return None - for name in self.manifest_names: - manifest_path = join(pkg_dir, name) - if isfile(manifest_path): - return manifest_path - return None - - def install( - self, - name, - requirements=None, - with_packages=None, - without_packages=None, - skip_default_package=False, - with_all_packages=False, - after_update=False, - silent=False, - force=False, - **_ - ): # pylint: disable=too-many-arguments, arguments-differ - platform_dir = BasePkgManager.install( - self, name, requirements, silent=silent, force=force - ) - p = PlatformFactory.new(platform_dir) - - if with_all_packages: - with_packages = list(p.packages.keys()) - - # don't cleanup packages or install them after update - # we check packages for updates in def update() - if after_update: - p.install_python_packages() - p.on_installed() - return True - - p.install_packages( - with_packages, - without_packages, - skip_default_package, - silent=silent, - force=force, - ) - p.install_python_packages() - p.on_installed() - return self.cleanup_packages(list(p.packages)) - - def uninstall(self, package, requirements=None, after_update=False): - if isdir(package): - pkg_dir = package - else: - name, requirements, url = self.parse_pkg_uri(package, requirements) - pkg_dir = self.get_package_dir(name, requirements, url) - - if not pkg_dir: - raise UnknownPlatform(package) - - p = PlatformFactory.new(pkg_dir) - BasePkgManager.uninstall(self, pkg_dir, requirements) - p.uninstall_python_packages() - p.on_uninstalled() - - # don't cleanup packages or install them after update - # we check packages for updates in def update() - if after_update: - return True - - return self.cleanup_packages(list(p.packages)) - - def update( # pylint: disable=arguments-differ - self, package, requirements=None, only_check=False, only_packages=False - ): - if isdir(package): - pkg_dir = package - else: - name, requirements, url = self.parse_pkg_uri(package, requirements) - pkg_dir = self.get_package_dir(name, requirements, url) - - if not pkg_dir: - raise UnknownPlatform(package) - - p = PlatformFactory.new(pkg_dir) - pkgs_before = list(p.get_installed_packages()) - - missed_pkgs = set() - if not only_packages: - BasePkgManager.update(self, pkg_dir, requirements, only_check) - p = PlatformFactory.new(pkg_dir) - missed_pkgs = set(pkgs_before) & set(p.packages) - missed_pkgs -= set(p.get_installed_packages()) - - p.update_packages(only_check) - self.cleanup_packages(list(p.packages)) - - if missed_pkgs: - p.install_packages( - with_packages=list(missed_pkgs), skip_default_package=True - ) - - return True - - def cleanup_packages(self, names): - self.cache_reset() - deppkgs = {} - for manifest in PlatformManager().get_installed(): - p = PlatformFactory.new(manifest["__pkg_dir"]) - for pkgname, pkgmanifest in p.get_installed_packages().items(): - if pkgname not in deppkgs: - deppkgs[pkgname] = set() - deppkgs[pkgname].add(pkgmanifest["version"]) - - pm = PackageManager(self.config.get_optional_dir("packages")) - for manifest in pm.get_installed(): - if manifest["name"] not in names: - continue - if ( - manifest["name"] not in deppkgs - or manifest["version"] not in deppkgs[manifest["name"]] - ): - try: - pm.uninstall(manifest["__pkg_dir"], after_update=True) - except exception.UnknownPackage: - pass - - self.cache_reset() - return True - - @util.memoized(expire="5s") - def get_installed_boards(self): - boards = [] - for manifest in self.get_installed(): - p = PlatformFactory.new(manifest["__pkg_dir"]) - for config in p.get_boards().values(): - board = config.get_brief_data() - if board not in boards: - boards.append(board) - return boards - - @staticmethod - def get_registered_boards(): - return util.get_api_result("/boards", cache_valid="7d") - - def get_all_boards(self): - boards = self.get_installed_boards() - know_boards = ["%s:%s" % (b["platform"], b["id"]) for b in boards] - try: - for board in self.get_registered_boards(): - key = "%s:%s" % (board["platform"], board["id"]) - if key not in know_boards: - boards.append(board) - except (exception.APIRequestError, exception.InternetIsOffline): - pass - return sorted(boards, key=lambda b: b["name"]) - - def board_config(self, id_, platform=None): - for manifest in self.get_installed_boards(): - if manifest["id"] == id_ and ( - not platform or manifest["platform"] == platform - ): - return manifest - for manifest in self.get_registered_boards(): - if manifest["id"] == id_ and ( - not platform or manifest["platform"] == platform - ): - return manifest - raise UnknownBoard(id_) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 04a41f267a..003c238644 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -42,17 +42,26 @@ def unpack(src, dst): with FileUnpacker(src) as fu: return fu.unpack(dst, with_progress=False) - def install(self, spec, silent=False, force=False): + def install(self, spec, silent=False, skip_dependencies=False, force=False): try: self.lock() - pkg = self._install(spec, silent=silent, force=force) + pkg = self._install( + spec, silent=silent, skip_dependencies=skip_dependencies, force=force + ) self.memcache_reset() self.cleanup_expired_downloads() return pkg finally: self.unlock() - def _install(self, spec, search_filters=None, silent=False, force=False): + def _install( # pylint: disable=too-many-arguments + self, + spec, + search_filters=None, + silent=False, + skip_dependencies=False, + force=False, + ): spec = self.ensure_spec(spec) # avoid circle dependencies @@ -104,11 +113,12 @@ def _install(self, spec, search_filters=None, silent=False, force=False): ) self.memcache_reset() - self._install_dependencies(pkg, silent) + if not skip_dependencies: + self.install_dependencies(pkg, silent) self._INSTALL_HISTORY[spec] = pkg return pkg - def _install_dependencies(self, pkg, silent=False): + def install_dependencies(self, pkg, silent=False): assert isinstance(pkg, PackageItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 7dc09964eb..72f189fb27 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -91,6 +91,9 @@ def install_from_registry(self, spec, search_filters=None, silent=False): self.print_multi_package_issue(packages, spec) package, version = self.find_best_registry_version(packages, spec) + if not package or not version: + raise UnknownPackageError(spec.humanize()) + pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None if not pkgfile: raise UnknownPackageError(spec.humanize()) @@ -189,7 +192,7 @@ def find_best_registry_version(self, packages, spec): return (package, version) if not spec.requirements: - return None + return (None, None) # if the custom version requirements, check ALL package versions for package in packages: @@ -206,7 +209,7 @@ def find_best_registry_version(self, packages, spec): if version: return (package, version) time.sleep(1) - return None + return (None, None) def pick_best_registry_version(self, versions, spec=None): assert not spec or isinstance(spec, PackageSpec) diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index e265640172..2cca850535 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -44,7 +44,7 @@ def _uninstall(self, spec, silent=False, skip_dependencies=False): # firstly, remove dependencies if not skip_dependencies: - self._uninstall_dependencies(pkg, silent) + self.uninstall_dependencies(pkg, silent) if os.path.islink(pkg.path): os.unlink(pkg.path) @@ -72,7 +72,7 @@ def _uninstall(self, spec, silent=False, skip_dependencies=False): return pkg - def _uninstall_dependencies(self, pkg, silent=False): + def uninstall_dependencies(self, pkg, silent=False): assert isinstance(pkg, PackageItem) manifest = self.load_manifest(pkg) if not manifest.get("dependencies"): diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index b0b976de42..d3e8dbb1d7 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -74,17 +74,24 @@ def _fetch_vcs_latest_version(self, pkg): ).version ) - def update(self, from_spec, to_spec=None, only_check=False, silent=False): + def update( # pylint: disable=too-many-arguments + self, + from_spec, + to_spec=None, + only_check=False, + silent=False, + show_incompatible=True, + ): pkg = self.get_package(from_spec) if not pkg or not pkg.metadata: raise UnknownPackageError(from_spec) if not silent: click.echo( - "{} {:<45} {:<30}".format( + "{} {:<45} {:<35}".format( "Checking" if only_check else "Updating", click.style(pkg.metadata.spec.humanize(), fg="cyan"), - "%s (%s)" % (pkg.metadata.version, to_spec.requirements) + "%s @ %s" % (pkg.metadata.version, to_spec.requirements) if to_spec and to_spec.requirements else str(pkg.metadata.version), ), @@ -97,17 +104,9 @@ def update(self, from_spec, to_spec=None, only_check=False, silent=False): outdated = self.outdated(pkg, to_spec) if not silent: - self.print_outdated_state(outdated) - - up_to_date = any( - [ - outdated.detached, - not outdated.latest, - outdated.latest and outdated.current == outdated.latest, - outdated.wanted and outdated.current == outdated.wanted, - ] - ) - if only_check or up_to_date: + self.print_outdated_state(outdated, show_incompatible) + + if only_check or not outdated.is_outdated(allow_incompatible=False): return pkg try: @@ -117,18 +116,26 @@ def update(self, from_spec, to_spec=None, only_check=False, silent=False): self.unlock() @staticmethod - def print_outdated_state(outdated): + def print_outdated_state(outdated, show_incompatible=True): if outdated.detached: return click.echo("[%s]" % (click.style("Detached", fg="yellow"))) - if not outdated.latest or outdated.current == outdated.latest: + if ( + not outdated.latest + or outdated.current == outdated.latest + or (not show_incompatible and outdated.current == outdated.wanted) + ): return click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) if outdated.wanted and outdated.current == outdated.wanted: return click.echo( - "[%s]" - % (click.style("Incompatible (%s)" % outdated.latest, fg="yellow")) + "[%s]" % (click.style("Incompatible %s" % outdated.latest, fg="yellow")) ) return click.echo( - "[%s]" % (click.style(str(outdated.wanted or outdated.latest), fg="red")) + "[%s]" + % ( + click.style( + "Outdated %s" % str(outdated.wanted or outdated.latest), fg="red" + ) + ) ) def _update(self, pkg, outdated, silent=False): diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index dc024edca0..b307753c4e 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -190,6 +190,10 @@ def build_metadata(self, pkg_dir, spec, vcs_revision=None): return metadata def get_installed(self): + cache_key = "get_installed" + if self.memcache_get(cache_key): + return self.memcache_get(cache_key) + result = [] for name in sorted(os.listdir(self.package_dir)): pkg_dir = os.path.join(self.package_dir, name) @@ -213,6 +217,8 @@ def get_installed(self): except MissingPackageManifestError: pass result.append(pkg) + + self.memcache_set(cache_key, result) return result def get_package(self, spec): diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index c79e7d1084..91eabf6af1 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -12,8 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from platformio import util +from platformio.exception import APIRequestError, InternetIsOffline +from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager +from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageType +from platformio.platform.exception import IncompatiblePlatform, UnknownBoard +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -28,3 +34,161 @@ def __init__(self, package_dir=None): @property def manifest_names(self): return PackageType.get_manifest_map()[PackageType.PLATFORM] + + def install( # pylint: disable=arguments-differ, too-many-arguments + self, + spec, + with_packages=None, + without_packages=None, + skip_default_package=False, + with_all_packages=False, + silent=False, + force=False, + ): + pkg = super(PlatformPackageManager, self).install( + spec, silent=silent, force=force, skip_dependencies=True + ) + try: + p = PlatformFactory.new(pkg) + p.ensure_engine_compatible() + except IncompatiblePlatform as e: + super(PlatformPackageManager, self).uninstall( + pkg, silent=silent, skip_dependencies=True + ) + raise e + + if with_all_packages: + with_packages = list(p.packages) + + p.install_packages( + with_packages, + without_packages, + skip_default_package, + silent=silent, + force=force, + ) + p.install_python_packages() + p.on_installed() + self.cleanup_packages(list(p.packages)) + return pkg + + def uninstall(self, spec, silent=False, skip_dependencies=False): + pkg = self.get_package(spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(spec) + p = PlatformFactory.new(pkg) + assert super(PlatformPackageManager, self).uninstall( + pkg, silent=silent, skip_dependencies=True + ) + if not skip_dependencies: + p.uninstall_python_packages() + p.on_uninstalled() + self.cleanup_packages(list(p.packages)) + return pkg + + def update( # pylint: disable=arguments-differ, too-many-arguments + self, + from_spec, + to_spec=None, + only_check=False, + silent=False, + show_incompatible=True, + only_packages=False, + ): + pkg = self.get_package(from_spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(from_spec) + p = PlatformFactory.new(pkg) + pkgs_before = [item.metadata.name for item in p.get_installed_packages()] + + new_pkg = None + missed_pkgs = set() + if not only_packages: + new_pkg = super(PlatformPackageManager, self).update( + from_spec, + to_spec, + only_check=only_check, + silent=silent, + show_incompatible=show_incompatible, + ) + p = PlatformFactory.new(new_pkg) + missed_pkgs = set(pkgs_before) & set(p.packages) + missed_pkgs -= set( + item.metadata.name for item in p.get_installed_packages() + ) + + p.update_packages(only_check) + self.cleanup_packages(list(p.packages)) + + if missed_pkgs: + p.install_packages( + with_packages=list(missed_pkgs), skip_default_package=True + ) + + return new_pkg or pkg + + def cleanup_packages(self, names): + self.memcache_reset() + deppkgs = {} + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for pkg in p.get_installed_packages(): + if pkg.metadata.name not in deppkgs: + deppkgs[pkg.metadata.name] = set() + deppkgs[pkg.metadata.name].add(pkg.metadata.version) + + pm = ToolPackageManager() + for pkg in pm.get_installed(): + if pkg.metadata.name not in names: + continue + if ( + pkg.metadata.name not in deppkgs + or pkg.metadata.version not in deppkgs[pkg.metadata.name] + ): + try: + pm.uninstall(pkg.metadata.spec) + except UnknownPackageError: + pass + + self.memcache_reset() + return True + + @util.memoized(expire="5s") + def get_installed_boards(self): + boards = [] + for pkg in self.get_installed(): + p = PlatformFactory.new(pkg) + for config in p.get_boards().values(): + board = config.get_brief_data() + if board not in boards: + boards.append(board) + return boards + + @staticmethod + def get_registered_boards(): + return util.get_api_result("/boards", cache_valid="7d") + + def get_all_boards(self): + boards = self.get_installed_boards() + know_boards = ["%s:%s" % (b["platform"], b["id"]) for b in boards] + try: + for board in self.get_registered_boards(): + key = "%s:%s" % (board["platform"], board["id"]) + if key not in know_boards: + boards.append(board) + except (APIRequestError, InternetIsOffline): + pass + return sorted(boards, key=lambda b: b["name"]) + + def board_config(self, id_, platform=None): + for manifest in self.get_installed_boards(): + if manifest["id"] == id_ and ( + not platform or manifest["platform"] == platform + ): + return manifest + for manifest in self.get_registered_boards(): + if manifest["id"] == id_ and ( + not platform or manifest["platform"] == platform + ): + return manifest + raise UnknownBoard(id_) diff --git a/platformio/package/manager/tool.py b/platformio/package/manager/tool.py index ae11179888..60aededd45 100644 --- a/platformio/package/manager/tool.py +++ b/platformio/package/manager/tool.py @@ -19,10 +19,9 @@ class ToolPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): - self.config = ProjectConfig.get_instance() - super(ToolPackageManager, self).__init__( - PackageType.TOOL, package_dir or self.config.get_optional_dir("packages"), - ) + if not package_dir: + package_dir = ProjectConfig.get_instance().get_optional_dir("packages") + super(ToolPackageManager, self).__init__(PackageType.TOOL, package_dir) @property def manifest_names(self): diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py index e626eb4b83..ac495b4827 100644 --- a/platformio/platform/_packages.py +++ b/platformio/platform/_packages.py @@ -13,9 +13,62 @@ # limitations under the License. from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageSpec class PlatformPackagesMixin(object): + def get_package_spec(self, name): + version = self.packages[name].get("version", "") + if any(c in version for c in (":", "/", "@")): + return PackageSpec("%s=%s" % (name, version)) + return PackageSpec( + owner=self.packages[name].get("owner"), name=name, requirements=version + ) + + def get_package(self, name): + if not name: + return None + return self.pm.get_package(self.get_package_spec(name)) + + def get_package_dir(self, name): + pkg = self.get_package(name) + return pkg.path if pkg else None + + def get_package_version(self, name): + pkg = self.get_package(name) + return str(pkg.metadata.version) if pkg else None + + def get_installed_packages(self): + result = [] + for name in self.packages: + pkg = self.get_package(name) + if pkg: + result.append(pkg) + return result + + def dump_used_packages(self): + result = [] + for name, options in self.packages.items(): + if options.get("optional"): + continue + pkg = self.get_package(name) + if not pkg or not pkg.metadata: + continue + item = {"name": pkg.metadata.name, "version": str(pkg.metadata.version)} + if pkg.metadata.spec.external: + item["src_url"] = pkg.metadata.spec.url + result.append(item) + return result + + def autoinstall_runtime_packages(self): + for name, options in self.packages.items(): + if options.get("optional", False): + continue + if self.get_package(name): + continue + self.pm.install(self.get_package_spec(name)) + return True + def install_packages( # pylint: disable=too-many-arguments self, with_packages=None, @@ -24,31 +77,25 @@ def install_packages( # pylint: disable=too-many-arguments silent=False, force=False, ): - with_packages = set(self.find_pkg_names(with_packages or [])) - without_packages = set(self.find_pkg_names(without_packages or [])) + with_packages = set(self._find_pkg_names(with_packages or [])) + without_packages = set(self._find_pkg_names(without_packages or [])) upkgs = with_packages | without_packages ppkgs = set(self.packages) if not upkgs.issubset(ppkgs): raise UnknownPackageError(", ".join(upkgs - ppkgs)) - for name, opts in self.packages.items(): - version = opts.get("version", "") + for name, options in self.packages.items(): if name in without_packages: continue if name in with_packages or not ( - skip_default_package or opts.get("optional", False) + skip_default_package or options.get("optional", False) ): - if ":" in version: - self.pm.install( - "%s=%s" % (name, version), silent=silent, force=force - ) - else: - self.pm.install(name, version, silent=silent, force=force) + self.pm.install(self.get_package_spec(name), silent=silent, force=force) return True - def find_pkg_names(self, candidates): + def _find_pkg_names(self, candidates): result = [] for candidate in candidates: found = False @@ -73,54 +120,18 @@ def find_pkg_names(self, candidates): return result def update_packages(self, only_check=False): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - self.pm.update(manifest["__pkg_dir"], requirements, only_check) - - def get_installed_packages(self): - items = {} - for name in self.packages: - pkg_dir = self.get_package_dir(name) - if pkg_dir: - items[name] = self.pm.load_manifest(pkg_dir) - return items + for pkg in self.get_installed_packages(): + self.pm.update( + pkg, + to_spec=self.get_package_spec(pkg.metadata.name), + only_check=only_check, + show_incompatible=False, + ) def are_outdated_packages(self): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - if self.pm.outdated(manifest["__pkg_dir"], requirements): + for pkg in self.get_installed_packages(): + if self.pm.outdated( + pkg, self.get_package_spec(pkg.metadata.name) + ).is_outdated(allow_incompatible=False): return True return False - - def get_package_dir(self, name): - version = self.packages[name].get("version", "") - if ":" in version: - return self.pm.get_package_dir( - *self.pm.parse_pkg_uri("%s=%s" % (name, version)) - ) - return self.pm.get_package_dir(name, version) - - def get_package_version(self, name): - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - return None - return self.pm.load_manifest(pkg_dir).get("version") - - def dump_used_packages(self): - result = [] - for name, options in self.packages.items(): - if options.get("optional"): - continue - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - continue - manifest = self.pm.load_manifest(pkg_dir) - item = {"name": manifest["name"], "version": manifest["version"]} - if manifest.get("__src_url"): - item["src_url"] = manifest.get("__src_url") - result.append(item) - return result diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 39e30fce1c..38cf232c5b 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -50,12 +50,14 @@ def run( # pylint: disable=too-many-arguments assert isinstance(variables, dict) assert isinstance(targets, list) + self.ensure_engine_compatible() + options = self.config.items(env=variables["pioenv"], as_dict=True) if "framework" in options: # support PIO Core 3.0 dev/platforms options["pioframework"] = options["framework"] self.configure_default_packages(options, targets) - self.install_packages(silent=True) + self.autoinstall_runtime_packages() self._report_non_sensitive_data(options, targets) @@ -84,8 +86,6 @@ def _report_non_sensitive_data(self, options, targets): for item in self.dump_used_packages() ] topts["platform"] = {"name": self.name, "version": self.version} - if self.src_version: - topts["platform"]["src_version"] = self.src_version telemetry.send_run_environment(topts, targets) def _run_scons(self, variables, targets, jobs): diff --git a/platformio/platform/base.py b/platformio/platform/base.py index a2fcd1582c..3860295076 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -19,11 +19,11 @@ import semantic_version from platformio import __version__, fs, proc, util -from platformio.managers.package import PackageManager +from platformio.package.manager.tool import ToolPackageManager from platformio.platform._packages import PlatformPackagesMixin from platformio.platform._run import PlatformRunMixin from platformio.platform.board import PlatformBoardConfig -from platformio.platform.exception import UnknownBoard +from platformio.platform.exception import IncompatiblePlatform, UnknownBoard from platformio.project.config import ProjectConfig @@ -44,20 +44,7 @@ def __init__(self, manifest_path): self._custom_packages = None self.config = ProjectConfig.get_instance() - self.pm = PackageManager( - self.config.get_optional_dir("packages"), self.package_repositories - ) - - self._src_manifest = None - src_manifest_path = self.pm.get_src_manifest_path(self.get_dir()) - if src_manifest_path: - self._src_manifest = fs.load_json(src_manifest_path) - - # if self.engines and "platformio" in self.engines: - # if self.PIO_VERSION not in semantic_version.SimpleSpec( - # self.engines['platformio']): - # raise exception.IncompatiblePlatform(self.name, - # str(self.PIO_VERSION)) + self.pm = ToolPackageManager(self.config.get_optional_dir("packages")) @property def name(self): @@ -75,14 +62,6 @@ def description(self): def version(self): return self._manifest["version"] - @property - def src_version(self): - return self._src_manifest.get("version") if self._src_manifest else None - - @property - def src_url(self): - return self._src_manifest.get("url") if self._src_manifest else None - @property def homepage(self): return self._manifest.get("homepage") @@ -103,10 +82,6 @@ def frameworks(self): def engines(self): return self._manifest.get("engines") - @property - def package_repositories(self): - return self._manifest.get("packageRepositories") - @property def manifest(self): return self._manifest @@ -114,21 +89,32 @@ def manifest(self): @property def packages(self): packages = self._manifest.get("packages", {}) - for item in self._custom_packages or []: - name = item - version = "*" - if "@" in item: - name, version = item.split("@", 2) - name = name.strip() - if name not in packages: - packages[name] = {} - packages[name].update({"version": version.strip(), "optional": False}) + for spec in self._custom_packages or []: + spec = self.pm.ensure_spec(spec) + if spec.external: + version = spec.url + else: + version = str(spec.requirements) or "*" + if spec.name not in packages: + packages[spec.name] = {} + packages[spec.name].update( + {"owner": spec.owner, "version": version, "optional": False} + ) return packages @property def python_packages(self): return self._manifest.get("pythonPackages") + def ensure_engine_compatible(self): + if not self.engines or "platformio" not in self.engines: + return True + if self.PIO_VERSION in semantic_version.SimpleSpec(self.engines["platformio"]): + return True + raise IncompatiblePlatform( + self.name, str(self.PIO_VERSION), self.engines["platformio"] + ) + def get_dir(self): return os.path.dirname(self.manifest_path) @@ -218,10 +204,10 @@ def get_lib_storages(self): for opts in (self.frameworks or {}).values(): if "package" not in opts: continue - pkg_dir = self.get_package_dir(opts["package"]) - if not pkg_dir or not os.path.isdir(os.path.join(pkg_dir, "libraries")): + pkg = self.get_package(opts["package"]) + if not pkg or not os.path.isdir(os.path.join(pkg.path, "libraries")): continue - libs_dir = os.path.join(pkg_dir, "libraries") + libs_dir = os.path.join(pkg.path, "libraries") storages[libs_dir] = opts["package"] libcores_dir = os.path.join(libs_dir, "__cores__") if not os.path.isdir(libcores_dir): diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py index 40431d7f41..604c322876 100644 --- a/platformio/platform/exception.py +++ b/platformio/platform/exception.py @@ -26,7 +26,10 @@ class UnknownPlatform(PlatformException): class IncompatiblePlatform(PlatformException): - MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}" + MESSAGE = ( + "Development platform '{0}' is not compatible with PlatformIO Core v{1} and " + "depends on PlatformIO Core {2}.\n" + ) class UnknownBoard(PlatformException): diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py index 99e5f7c435..e48142ce1a 100644 --- a/platformio/platform/factory.py +++ b/platformio/platform/factory.py @@ -16,7 +16,7 @@ import re from platformio.compat import load_python_module -from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.meta import PackageItem from platformio.platform.base import PlatformBase from platformio.platform.exception import UnknownPlatform @@ -36,9 +36,16 @@ def load_module(name, path): @classmethod def new(cls, pkg_or_spec): - pkg = PlatformPackageManager().get_package( - "file://%s" % pkg_or_spec if os.path.isdir(pkg_or_spec) else pkg_or_spec - ) + if isinstance(pkg_or_spec, PackageItem): + pkg = pkg_or_spec + else: + from platformio.package.manager.platform import ( # pylint: disable=import-outside-toplevel + PlatformPackageManager, + ) + + pkg = PlatformPackageManager().get_package( + "file://%s" % pkg_or_spec if os.path.isdir(pkg_or_spec) else pkg_or_spec + ) if not pkg: raise UnknownPlatform(pkg_or_spec) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 377c1e28df..74bb6f4582 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -14,8 +14,9 @@ import json -from platformio import exception from platformio.commands import platform as cli_platform +from platformio.package.exception import UnknownPackageError +from platformio.platform.exception import IncompatiblePlatform def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): @@ -39,22 +40,30 @@ def test_search_raw_output(clirunner, validate_cliresult): def test_install_unknown_version(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["atmelavr@99.99.99"]) assert result.exit_code != 0 - assert isinstance(result.exception, exception.UndefinedPackageVersion) + assert isinstance(result.exception, UnknownPackageError) def test_install_unknown_from_registry(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["unknown-platform"]) assert result.exit_code != 0 - assert isinstance(result.exception, exception.UnknownPackage) + assert isinstance(result.exception, UnknownPackageError) + + +def test_install_incompatbile(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], + ) + assert result.exit_code != 0 + assert isinstance(result.exception, IncompatiblePlatform) def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, - ["atmelavr@1.2.0", "--skip-default-package", "--with-package", "tool-avrdude"], + ["atmelavr@2.0.0", "--skip-default-package", "--with-package", "tool-avrdude"], ) validate_cliresult(result) - assert "atmelavr @ 1.2.0" in result.output + assert "atmelavr @ 2.0.0" in result.output assert "Installing tool-avrdude @" in result.output assert len(isolated_pio_core.join("packages").listdir()) == 1 @@ -63,7 +72,7 @@ def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, [ - "https://github.com/platformio/" "platform-espressif8266.git", + "https://github.com/platformio/platform-espressif8266.git", "--skip-default-package", ], ) @@ -90,7 +99,7 @@ def test_list_raw_output(clirunner, validate_cliresult): def test_update_check(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( - cli_platform.platform_update, ["--only-check", "--json-output"] + cli_platform.platform_update, ["--dry-run", "--json-output"] ) validate_cliresult(result) output = json.loads(result.output) @@ -102,9 +111,9 @@ def test_update_check(clirunner, validate_cliresult, isolated_pio_core): def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cli_platform.platform_update) validate_cliresult(result) - assert "Uninstalling atmelavr @ 1.2.0:" in result.output - assert "PlatformManager: Installing atmelavr @" in result.output - assert len(isolated_pio_core.join("packages").listdir()) == 1 + assert "Removing atmelavr @ 2.0.0:" in result.output + assert "Platform Manager: Installing platformio/atmelavr @" in result.output + assert len(isolated_pio_core.join("packages").listdir()) == 2 def test_uninstall(clirunner, validate_cliresult, isolated_pio_core): diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 07fbabf80f..4150bcddde 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -13,13 +13,13 @@ # limitations under the License. import json +import os import re from time import time from platformio import app, maintenance from platformio.__main__ import cli as cli_pio from platformio.commands import upgrade as cmd_upgrade -from platformio.managers.platform import PlatformManager def test_check_pio_upgrade(clirunner, isolated_pio_core, validate_cliresult): @@ -89,7 +89,8 @@ def test_check_and_update_libraries(clirunner, isolated_pio_core, validate_clire assert "There are the new updates for libraries (ArduinoJson)" in result.output assert "Please wait while updating libraries" in result.output assert re.search( - r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[[\d\.]+\]", result.output + r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[Outdated [\d\.]+\]", + result.output, ) # check updated version @@ -102,12 +103,11 @@ def test_check_platform_updates(clirunner, isolated_pio_core, validate_cliresult # install obsolete platform result = clirunner.invoke(cli_pio, ["platform", "install", "native"]) validate_cliresult(result) + os.remove(str(isolated_pio_core.join("platforms", "native", ".piopm"))) manifest_path = isolated_pio_core.join("platforms", "native", "platform.json") manifest = json.loads(manifest_path.read()) manifest["version"] = "0.0.0" manifest_path.write(json.dumps(manifest)) - # reset cached manifests - PlatformManager().cache_reset() # reset check time interval = int(app.get_setting("check_platforms_interval")) * 3600 * 24 @@ -141,7 +141,7 @@ def test_check_and_update_platforms(clirunner, isolated_pio_core, validate_clire validate_cliresult(result) assert "There are the new updates for platforms (native)" in result.output assert "Please wait while updating platforms" in result.output - assert re.search(r"Updating native\s+@ 0.0.0\s+\[[\d\.]+\]", result.output) + assert re.search(r"Updating native\s+0.0.0\s+\[Outdated [\d\.]+\]", result.output) # check updated version result = clirunner.invoke(cli_pio, ["platform", "list", "--json-output"]) From 67e6d177b443c189660f8d21efed05a3ceaacf80 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 16 Aug 2020 18:48:05 +0300 Subject: [PATCH 158/223] Minor fixes for dev-platform factory --- platformio/builder/tools/pioplatform.py | 11 ++++++-- platformio/platform/factory.py | 36 ++++++++++++++++++------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 740f61486e..5ca7794f3e 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -149,9 +149,16 @@ def _get_configuration_data(): def _get_plaform_data(): data = [ "PLATFORM: %s (%s)" - % (platform.title, pkg_metadata.version or platform.version) + % ( + platform.title, + pkg_metadata.version if pkg_metadata else platform.version, + ) ] - if int(ARGUMENTS.get("PIOVERBOSE", 0)) and pkg_metadata.spec.external: + if ( + int(ARGUMENTS.get("PIOVERBOSE", 0)) + and pkg_metadata + and pkg_metadata.spec.external + ): data.append("(%s)" % pkg_metadata.spec.url) if board_config: data.extend([">", board_config.get("name")]) diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py index e48142ce1a..0f2bd15f34 100644 --- a/platformio/platform/factory.py +++ b/platformio/platform/factory.py @@ -15,6 +15,7 @@ import os import re +from platformio import fs from platformio.compat import load_python_module from platformio.package.meta import PackageItem from platformio.platform.base import PlatformBase @@ -36,32 +37,47 @@ def load_module(name, path): @classmethod def new(cls, pkg_or_spec): + platform_dir = None + platform_name = None if isinstance(pkg_or_spec, PackageItem): - pkg = pkg_or_spec + platform_dir = pkg_or_spec.path + platform_name = pkg_or_spec.metadata.name + elif os.path.isdir(pkg_or_spec): + platform_dir = pkg_or_spec else: from platformio.package.manager.platform import ( # pylint: disable=import-outside-toplevel PlatformPackageManager, ) - pkg = PlatformPackageManager().get_package( - "file://%s" % pkg_or_spec if os.path.isdir(pkg_or_spec) else pkg_or_spec - ) - if not pkg: + pkg = PlatformPackageManager().get_package(pkg_or_spec) + if not pkg: + raise UnknownPlatform(pkg_or_spec) + platform_dir = pkg.path + platform_name = pkg.metadata.name + + if not platform_dir or not os.path.isfile( + os.path.join(platform_dir, "platform.json") + ): raise UnknownPlatform(pkg_or_spec) + if not platform_name: + platform_name = fs.load_json(os.path.join(platform_dir, "platform.json"))[ + "name" + ] + platform_cls = None - if os.path.isfile(os.path.join(pkg.path, "platform.py")): + if os.path.isfile(os.path.join(platform_dir, "platform.py")): platform_cls = getattr( cls.load_module( - pkg.metadata.name, os.path.join(pkg.path, "platform.py") + platform_name, os.path.join(platform_dir, "platform.py") ), - cls.get_clsname(pkg.metadata.name), + cls.get_clsname(platform_name), ) else: platform_cls = type( - str(cls.get_clsname(pkg.metadata.name)), (PlatformBase,), {} + str(cls.get_clsname(platform_name)), (PlatformBase,), {} ) - _instance = platform_cls(os.path.join(pkg.path, "platform.json")) + _instance = platform_cls(os.path.join(platform_dir, "platform.json")) assert isinstance(_instance, PlatformBase) return _instance From 808852f4cc37ac97f031f850c995c7690015ca71 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 16 Aug 2020 20:21:30 +0300 Subject: [PATCH 159/223] Set default timeout for http requests // Resolve #3623 --- platformio/__init__.py | 2 ++ platformio/app.py | 13 ++++------- platformio/clients/http.py | 7 +++++- platformio/commands/home/rpc/handlers/os.py | 10 +++++--- platformio/commands/upgrade.py | 20 ++++++---------- platformio/package/download.py | 5 ++-- platformio/package/manifest/parser.py | 7 ++---- platformio/package/manifest/schema.py | 11 +++++---- platformio/util.py | 26 +++++++++++++++++++-- 9 files changed, 62 insertions(+), 39 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index ee6d438d3c..6ef8af69ed 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,6 +14,8 @@ import sys +DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) + VERSION = (4, 4, "0a8") __version__ = ".".join([str(s) for s in VERSION]) diff --git a/platformio/app.py b/platformio/app.py index 00a8e89f1c..21adba1cd3 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -17,6 +17,7 @@ import codecs import getpass import hashlib +import json import os import platform import socket @@ -25,9 +26,7 @@ from os.path import dirname, isdir, isfile, join, realpath from time import time -import requests - -from platformio import __version__, exception, fs, proc +from platformio import __version__, exception, fs, proc, util from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.helpers import ( @@ -403,16 +402,14 @@ def get_cid(): uid = getenv("C9_UID") elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): try: - uid = ( - requests.get( + uid = json.loads( + util.fetch_remote_content( "{api}/user?token={token}".format( api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), token=getenv("USER_TOKEN"), ) ) - .json() - .get("id") - ) + ).get("id") except: # pylint: disable=bare-except pass if not uid: diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 974017b733..e18d2eedda 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -15,7 +15,7 @@ import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error -from platformio import app, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util from platformio.exception import PlatformioException @@ -58,6 +58,11 @@ def send_request(self, method, path, **kwargs): # check Internet before and resolve issue with 60 seconds timeout # print(self, method, path, kwargs) util.internet_on(raise_exception=True) + + # set default timeout + if "timeout" not in kwargs: + kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + try: return getattr(self._session, method)(self.base_url + path, **kwargs) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 3b4bd4e1ae..2b1662f298 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -22,7 +22,7 @@ import click from twisted.internet import defer # pylint: disable=import-error -from platformio import app, fs, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding, glob_recursive @@ -51,9 +51,13 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None): session = helpers.requests_session() if data: - r = yield session.post(uri, data=data, headers=headers) + r = yield session.post( + uri, data=data, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT + ) else: - r = yield session.get(uri, headers=headers) + r = yield session.get( + uri, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT + ) r.raise_for_status() result = r.text diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 6303ea69a5..c8c8b9fee0 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import re from zipfile import ZipFile import click -import requests -from platformio import VERSION, __version__, app, exception +from platformio import VERSION, __version__, app, exception, util from platformio.compat import WINDOWS from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir @@ -130,13 +130,11 @@ def get_latest_version(): def get_develop_latest_version(): version = None - r = requests.get( + content = util.fetch_remote_content( "https://raw.githubusercontent.com/platformio/platformio" - "/develop/platformio/__init__.py", - headers={"User-Agent": app.get_user_agent()}, + "/develop/platformio/__init__.py" ) - r.raise_for_status() - for line in r.text.split("\n"): + for line in content.split("\n"): line = line.strip() if not line.startswith("VERSION"): continue @@ -152,9 +150,5 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - r = requests.get( - "https://pypi.org/pypi/platformio/json", - headers={"User-Agent": app.get_user_agent()}, - ) - r.raise_for_status() - return r.json()["info"]["version"] + content = util.fetch_remote_content("https://pypi.org/pypi/platformio/json") + return json.loads(content)["info"]["version"] diff --git a/platformio/package/download.py b/platformio/package/download.py index 3c723c4baf..7f29e7acf4 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -14,7 +14,6 @@ import io import math -import sys from email.utils import parsedate_tz from os.path import getsize, join from time import mktime @@ -22,7 +21,7 @@ import click import requests -from platformio import app, fs, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util from platformio.package.exception import PackageException @@ -34,7 +33,7 @@ def __init__(self, url, dest_dir=None): url, stream=True, headers={"User-Agent": app.get_user_agent()}, - verify=sys.version_info >= (2, 7, 9), + timeout=DEFAULT_REQUESTS_TIMEOUT, ) if self._request.status_code != 200: raise PackageException( diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index b4a93d982e..d453c83efc 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -19,8 +19,6 @@ import re import tarfile -import requests - from platformio import util from platformio.compat import get_object_members, string_types from platformio.package.exception import ManifestParserError, UnknownManifestError @@ -108,10 +106,9 @@ def new_from_dir(cls, path, remote_url=None): @staticmethod def new_from_url(remote_url): - r = requests.get(remote_url) - r.raise_for_status() + content = util.fetch_remote_content(remote_url) return ManifestParserFactory.new( - r.text, + content, ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON, remote_url, ) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 8befab521a..7dafaa2368 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -14,11 +14,14 @@ # pylint: disable=too-many-ancestors +import json + import marshmallow import requests import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates +from platformio import util from platformio.package.exception import ManifestValidationError from platformio.util import memoized @@ -248,9 +251,9 @@ def validate_license(self, value): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - r = requests.get( + version = "3.10" + spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data" - "/v3.10/json/licenses.json" + "/v%s/json/licenses.json" % version ) - r.raise_for_status() - return r.json() + return json.loads(util.fetch_remote_content(spdx_data_url)) diff --git a/platformio/util.py b/platformio/util.py index a1974f1750..982b0bab10 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -29,7 +29,7 @@ import click import requests -from platformio import __apiurl__, __version__, exception +from platformio import DEFAULT_REQUESTS_TIMEOUT, __apiurl__, __version__, exception from platformio.commands import PlatformioCLI from platformio.compat import PY2, WINDOWS from platformio.fs import cd # pylint: disable=unused-import @@ -303,10 +303,16 @@ def _get_api_result( headers=headers, auth=auth, verify=verify_ssl, + timeout=DEFAULT_REQUESTS_TIMEOUT, ) else: r = _api_request_session().get( - url, params=params, headers=headers, auth=auth, verify=verify_ssl + url, + params=params, + headers=headers, + auth=auth, + verify=verify_ssl, + timeout=DEFAULT_REQUESTS_TIMEOUT, ) result = r.json() r.raise_for_status() @@ -398,6 +404,22 @@ def internet_on(raise_exception=False): return result +def fetch_remote_content(*args, **kwargs): + # pylint: disable=import-outside-toplevel + from platformio.app import get_user_agent + + kwargs["headers"] = kwargs.get("headers", {}) + if "User-Agent" not in kwargs["headers"]: + kwargs["headers"]["User-Agent"] = get_user_agent() + + if "timeout" not in kwargs: + kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + + r = requests.get(*args, **kwargs) + r.raise_for_status() + return r.text + + def pepver_to_semver(pepver): return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) From 74e27a2edc94e9531eaf81bb7047199f6a7a9296 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 16 Aug 2020 20:26:59 +0300 Subject: [PATCH 160/223] Enable "cyclic reference" for GCC linker only for the embedded dev-platforms // Resolve #3570 --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 147c4b5f4f..7f86e40f39 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -54,6 +54,7 @@ PlatformIO Core 4 * Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command * Do not generate ".travis.yml" for a new project, let the user have a choice * Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) +* Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) * Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 * Do not escape compiler arguments in VSCode template on Windows * Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 560cbe3754..5d8f8e8bca 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -66,7 +66,11 @@ def BuildProgram(env): env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")]) # enable "cyclic reference" for linker - if env.get("LIBS") and env.GetCompilerType() == "gcc": + if ( + env.get("LIBS") + and env.GetCompilerType() == "gcc" + and env.PioPlatform().is_embedded() + ): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") From 2459e85c1dc9a1ecfe632c611594c6c302c8af6b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Aug 2020 12:13:25 +0300 Subject: [PATCH 161/223] Fix a bug with the custom platform packages // Resolve #3628 --- platformio/platform/base.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 3860295076..a5f3dd0175 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -89,17 +89,20 @@ def manifest(self): @property def packages(self): packages = self._manifest.get("packages", {}) - for spec in self._custom_packages or []: - spec = self.pm.ensure_spec(spec) - if spec.external: - version = spec.url - else: - version = str(spec.requirements) or "*" + for item in self._custom_packages or []: + name = item + version = "*" + if "@" in item: + name, version = item.split("@", 2) + spec = self.pm.ensure_spec(name) + options = {"version": version.strip(), "optional": False} + if spec.owner: + options["owner"] = spec.owner if spec.name not in packages: packages[spec.name] = {} - packages[spec.name].update( - {"owner": spec.owner, "version": version, "optional": False} - ) + packages[spec.name].update(**options) + + print(13, packages) return packages @property From 6f7fc638c7dad4315a318ea00b357916558ca6a3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Aug 2020 12:56:57 +0300 Subject: [PATCH 162/223] Fix PyLint errors in tests --- .pylintrc | 3 +++ Makefile | 3 ++- platformio/platform/base.py | 2 -- tests/__init__.py | 13 +++++++++++++ tests/commands/__init__.py | 13 +++++++++++++ tests/commands/test_account_org_team.py | 5 ++++- tests/commands/test_boards.py | 2 +- tests/commands/test_check.py | 2 ++ tests/commands/test_lib.py | 2 ++ tests/commands/test_lib_complex.py | 8 ++++++-- tests/commands/test_platform.py | 2 ++ tests/commands/test_test.py | 1 + tests/commands/test_update.py | 2 ++ tests/ino2cpp/__init__.py | 13 +++++++++++++ tests/package/__init__.py | 13 +++++++++++++ tests/package/test_manager.py | 2 ++ tests/package/test_manifest.py | 2 +- tests/test_examples.py | 7 ++++--- tests/test_maintenance.py | 2 ++ tests/test_misc.py | 3 +++ tests/test_projectconf.py | 4 +++- tox.ini | 2 +- 22 files changed, 93 insertions(+), 13 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/commands/__init__.py create mode 100644 tests/ino2cpp/__init__.py create mode 100644 tests/package/__init__.py diff --git a/.pylintrc b/.pylintrc index 67ce4ef721..6ce74864ca 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,3 +1,6 @@ +[REPORTS] +output-format=colorized + [MESSAGES CONTROL] disable= bad-continuation, diff --git a/Makefile b/Makefile index 6b22d261eb..fa301e5994 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ lint: - pylint --rcfile=./.pylintrc ./platformio + pylint -j 6 --rcfile=./.pylintrc ./platformio ./tests + pylint -j 6 --rcfile=./.pylintrc ./tests isort: isort -rc ./platformio diff --git a/platformio/platform/base.py b/platformio/platform/base.py index a5f3dd0175..8e49288d10 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -101,8 +101,6 @@ def packages(self): if spec.name not in packages: packages[spec.name] = {} packages[spec.name].update(**options) - - print(13, packages) return packages @property diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/tests/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index b6d5b6d4f1..fc64db41ec 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=global-statement,unused-argument + import json import os import random @@ -30,6 +32,7 @@ username = None email = None +splited_email = None firstname = None lastname = None password = None @@ -43,7 +46,7 @@ def test_prepare(): - global username, splited_email, email, firstname, lastname + global username, email, splited_email, firstname, lastname global password, orgname, display_name, second_username, teamname, team_description username = "test-piocore-%s" % str(random.randint(0, 100000)) diff --git a/tests/commands/test_boards.py b/tests/commands/test_boards.py index bcb1f2804c..21142dd49a 100644 --- a/tests/commands/test_boards.py +++ b/tests/commands/test_boards.py @@ -40,7 +40,7 @@ def test_board_options(clirunner, validate_cliresult): validate_cliresult(result) search_result = json.loads(result.output) assert isinstance(search_result, list) - assert len(search_result) + assert search_result platforms = [item["name"] for item in search_result] result = clirunner.invoke(cmd_boards, ["mbed", "--json-output"]) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 998d44cf3a..655449b002 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=redefined-outer-name + import json import sys from os.path import isfile, join diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 1880d67163..332161e1f0 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import json import os diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index a71330db06..dfb853f472 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=line-too-long + import json import re @@ -129,7 +131,9 @@ def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_c assert set(items1) >= set(items2) -def test_install_duplicates(clirunner, validate_cliresult, without_internet): +def test_install_duplicates( # pylint: disable=unused-argument + clirunner, validate_cliresult, without_internet +): # registry result = clirunner.invoke( cmd_lib, @@ -231,7 +235,7 @@ def test_global_lib_update_check(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "update", "--dry-run", "--json-output"]) validate_cliresult(result) output = json.loads(result.output) - assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([lib["name"] for lib in output]) + assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set(lib["name"] for lib in output) def test_global_lib_update(clirunner, validate_cliresult): diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 74bb6f4582..2c197ec3e6 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import json from platformio.commands import platform as cli_platform diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 16e0556cf0..9f07286813 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -33,6 +33,7 @@ def test_local_env(): ) if result["returncode"] != 1: pytest.fail(str(result)) + # pylint: disable=unsupported-membership-test assert all([s in result["err"] for s in ("PASSED", "IGNORED", "FAILED")]), result[ "out" ] diff --git a/tests/commands/test_update.py b/tests/commands/test_update.py index 1817be339f..b9ecb5c186 100644 --- a/tests/commands/test_update.py +++ b/tests/commands/test_update.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + from platformio.commands.update import cli as cmd_update diff --git a/tests/ino2cpp/__init__.py b/tests/ino2cpp/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/tests/ino2cpp/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/package/__init__.py b/tests/package/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/tests/package/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index c91924f414..f014a66ba0 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import os import time diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 35fdf3671e..9dd5b8789a 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -741,7 +741,7 @@ def _to_unix_path(path): return re.sub(r"[\\/]+", "/", path) def _sort_examples(items): - for i, item in enumerate(items): + for i, _ in enumerate(items): items[i]["base"] = _to_unix_path(items[i]["base"]) items[i]["files"] = [_to_unix_path(f) for f in sorted(items[i]["files"])] return sorted(items, key=lambda item: item["name"]) diff --git a/tests/test_examples.py b/tests/test_examples.py index d0a580d48c..ada20d35c1 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -21,7 +21,8 @@ from platformio import util from platformio.compat import PY2 -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -34,8 +35,8 @@ def pytest_generate_tests(metafunc): examples_dirs.append(normpath(join(dirname(__file__), "..", "examples"))) # dev/platforms - for manifest in PlatformManager().get_installed(): - p = PlatformFactory.new(manifest["__pkg_dir"]) + for pkg in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(pkg) examples_dir = join(p.get_dir(), "examples") assert isdir(examples_dir) examples_dirs.append(examples_dir) diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 4150bcddde..46bc82c803 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import json import os import re diff --git a/tests/test_misc.py b/tests/test_misc.py index d01cc46d41..ae019f6b97 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import pytest import requests @@ -21,6 +23,7 @@ def test_platformio_cli(): result = util.exec_command(["pio", "--help"]) assert result["returncode"] == 0 + # pylint: disable=unsupported-membership-test assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result["out"] diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 4d832c9754..832ceb716d 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=redefined-outer-name + import os import pytest @@ -345,7 +347,7 @@ def test_update_and_save(tmpdir_factory): ["check_types", [("float_option", 13.99), ("bool_option", True)]], ] ) - config.get("platformio", "extra_configs") == "extra.ini" + assert config.get("platformio", "extra_configs") == ["extra.ini"] config.remove_section("platformio") assert config.as_tuple() == [ ("env:myenv", [("board", "myboard"), ("framework", ["espidf", "arduino"])]), diff --git a/tox.ini b/tox.ini index fbe285e24e..0faae46b5e 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ commands = [testenv:lint] commands = {envpython} --version - pylint --rcfile=./.pylintrc ./platformio + pylint --rcfile=./.pylintrc ./platformio ./tests [testenv:testcore] commands = From fb6e1fd33cf4bb23a99c4526a34f4cd01736a41a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Aug 2020 15:33:08 +0300 Subject: [PATCH 163/223] PyLint fixes --- Makefile | 2 +- tests/package/test_manager.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index fa301e5994..57aa76c40b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ lint: - pylint -j 6 --rcfile=./.pylintrc ./platformio ./tests + pylint -j 6 --rcfile=./.pylintrc ./platformio pylint -j 6 --rcfile=./.pylintrc ./tests isort: diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index f014a66ba0..2c331dbea1 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -349,7 +349,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): assert lm.uninstall(foo_1_0_0_pkg.path, silent=True) assert lm.uninstall(bar_pkg, silent=True) - assert len(lm.get_installed()) == 0 + assert not lm.get_installed() # test uninstall dependencies assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) @@ -360,7 +360,7 @@ def test_uninstall(isolated_pio_core, tmpdir_factory): lm = LibraryPackageManager(str(storage_dir)) assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) assert lm.uninstall("AsyncMqttClient-esphome", silent=True) - assert len(lm.get_installed()) == 0 + assert not lm.get_installed() def test_registry(isolated_pio_core): From d9801946008f539d2a485fe359b3ba079e4e32d9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 17 Aug 2020 15:34:02 +0300 Subject: [PATCH 164/223] Bump version to 4.4.0b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6ef8af69ed..3ecfb5965c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -16,7 +16,7 @@ DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) -VERSION = (4, 4, "0a8") +VERSION = (4, 4, "0b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f79fb4190ec79bc31d3eaa3c46ce6a0672a8c308 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 21 Aug 2020 14:25:59 +0300 Subject: [PATCH 165/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index e8ee370a33..2f5498050a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e8ee370a338270b453d4d97bd286f537d5f06456 +Subproject commit 2f5498050ae5602b92ab56e2ade0af33e088a944 From 49b70f44caa717e086706a42dbdaa85da165cfc9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 13:56:57 +0300 Subject: [PATCH 166/223] Ignore legacy tmp pkg folders --- platformio/package/manager/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index b307753c4e..cf359d1302 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -196,6 +196,8 @@ def get_installed(self): result = [] for name in sorted(os.listdir(self.package_dir)): + if name.startswith("_tmp_installing"): # legacy tmp folder + continue pkg_dir = os.path.join(self.package_dir, name) if not os.path.isdir(pkg_dir): continue From 70366d34b931603116b9760e3e202d28aeb08a3a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 13:57:18 +0300 Subject: [PATCH 167/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 2f5498050a..d01bbede6c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2f5498050ae5602b92ab56e2ade0af33e088a944 +Subproject commit d01bbede6ca90420ed24736be4963a48228eff42 From aa186382a84db1af5e62c1faa0e487aaf712d09b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 14:22:37 +0300 Subject: [PATCH 168/223] Upgraded to SCons 4.0 --- HISTORY.rst | 32 +++++++++++++++++++------------- platformio/__init__.py | 2 +- platformio/platform/_run.py | 10 ++++++++-- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7f86e40f39..17a8786b0c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -46,20 +46,26 @@ PlatformIO Core 4 - Command launcher with own arguments - Launch command with custom options declared in `"platformio.ini" `__ - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) + - List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) -* Display system-wide information using a new `platformio system info `__ command (`issue #3521 `_) -* List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) -* Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. -* Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment -* Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command -* Do not generate ".travis.yml" for a new project, let the user have a choice -* Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) -* Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) -* Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 -* Do not escape compiler arguments in VSCode template on Windows -* Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) -* Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) -* Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) +* **PlatformIO Build System** + + - Upgraded to `SCons 4.0 - a next-generation software construction tool `__ + - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) + - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) + - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) + - Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) + +* **Miscellaneous** + + - Display system-wide information using a new `platformio system info `__ command (`issue #3521 `_) + - Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command + - Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment + - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. + - Do not generate ".travis.yml" for a new project, let the user have a choice + - Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 + - Do not escape compiler arguments in VSCode template on Windows + - Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__init__.py b/platformio/__init__.py index 3ecfb5965c..b51a0be079 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -49,7 +49,7 @@ "contrib-piohome": "~3.2.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~3.30102.0", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", "tool-cppcheck": "~1.190.0", "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.7.0", diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 38cf232c5b..457983c49b 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -20,7 +20,7 @@ import click from platformio import app, fs, proc, telemetry -from platformio.compat import hashlib_encode_data, is_bytes +from platformio.compat import PY2, hashlib_encode_data, is_bytes from platformio.package.manager.core import get_core_package_dir from platformio.platform.exception import BuildScriptNotFound @@ -89,9 +89,15 @@ def _report_non_sensitive_data(self, options, targets): telemetry.send_run_environment(topts, targets) def _run_scons(self, variables, targets, jobs): + scons_dir = get_core_package_dir("tool-scons") + script_path = ( + os.path.join(scons_dir, "script", "scons") + if PY2 + else os.path.join(scons_dir, "scons.py") + ) args = [ proc.get_pythonexe_path(), - os.path.join(get_core_package_dir("tool-scons"), "script", "scons"), + script_path, "-Q", "--warn=no-no-parallel-support", "--jobs", From d92c1d3442e7c93ba2d0a8510854349dcada6e3b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 17:48:49 +0300 Subject: [PATCH 169/223] Refactor HTTP related operations --- .pylintrc | 6 +- platformio/app.py | 7 +- platformio/clients/account.py | 14 +-- platformio/clients/http.py | 86 +++++++++++++- platformio/commands/debug/command.py | 4 +- platformio/commands/home/rpc/handlers/os.py | 3 +- platformio/commands/lib/command.py | 4 +- .../commands/remote/client/run_or_test.py | 4 +- platformio/commands/upgrade.py | 7 +- platformio/exception.py | 73 ------------ platformio/fs.py | 4 + platformio/maintenance.py | 9 +- platformio/package/download.py | 4 +- platformio/package/exception.py | 11 ++ platformio/package/lockfile.py | 10 +- platformio/package/manager/_update.py | 4 +- platformio/package/manager/core.py | 4 +- platformio/package/manager/platform.py | 4 +- platformio/package/manifest/parser.py | 3 +- platformio/package/manifest/schema.py | 4 +- platformio/package/unpack.py | 4 +- platformio/proc.py | 12 ++ platformio/telemetry.py | 2 +- platformio/util.py | 112 +++--------------- tests/commands/test_check.py | 6 +- tests/commands/test_test.py | 4 +- tests/conftest.py | 4 +- tests/test_examples.py | 8 +- tests/test_misc.py | 22 ++-- 29 files changed, 206 insertions(+), 233 deletions(-) diff --git a/.pylintrc b/.pylintrc index 6ce74864ca..e21dfef9f5 100644 --- a/.pylintrc +++ b/.pylintrc @@ -15,4 +15,8 @@ disable= useless-object-inheritance, useless-import-alias, fixme, - bad-option-value + bad-option-value, + + ; PY2 Compat + super-with-arguments, + raise-missing-from diff --git a/platformio/app.py b/platformio/app.py index 21adba1cd3..0933892b28 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -26,7 +26,7 @@ from os.path import dirname, isdir, isfile, join, realpath from time import time -from platformio import __version__, exception, fs, proc, util +from platformio import __version__, exception, fs, proc from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.helpers import ( @@ -394,6 +394,9 @@ def is_disabled_progressbar(): def get_cid(): + # pylint: disable=import-outside-toplevel + from platformio.clients.http import fetch_remote_content + cid = get_state_item("cid") if cid: return cid @@ -403,7 +406,7 @@ def get_cid(): elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): try: uid = json.loads( - util.fetch_remote_content( + fetch_remote_content( "{api}/user?token={token}".format( api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), token=getenv("USER_TOKEN"), diff --git a/platformio/clients/account.py b/platformio/clients/account.py index c29ef9f9bc..d492b65df4 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -67,7 +67,7 @@ def send_auth_request(self, *args, **kwargs): token = self.fetch_authentication_token() headers["Authorization"] = "Bearer %s" % token kwargs["headers"] = headers - return self.request_json_data(*args, **kwargs) + return self.fetch_json_data(*args, **kwargs) def login(self, username, password): try: @@ -79,7 +79,7 @@ def login(self, username, password): app.get_state_item("account", {}).get("email", "") ) - data = self.request_json_data( + data = self.fetch_json_data( "post", "/v1/login", data={"username": username, "password": password}, ) app.set_state_item("account", data) @@ -95,7 +95,7 @@ def login_with_code(self, client_id, code, redirect_uri): app.get_state_item("account", {}).get("email", "") ) - result = self.request_json_data( + result = self.fetch_json_data( "post", "/v1/login/code", data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, @@ -107,7 +107,7 @@ def logout(self): refresh_token = self.get_refresh_token() self.delete_local_session() try: - self.request_json_data( + self.fetch_json_data( "post", "/v1/logout", data={"refresh_token": refresh_token}, ) except AccountError: @@ -133,7 +133,7 @@ def registration( app.get_state_item("account", {}).get("email", "") ) - return self.request_json_data( + return self.fetch_json_data( "post", "/v1/registration", data={ @@ -153,7 +153,7 @@ def auth_token(self, password, regenerate): ).get("auth_token") def forgot_password(self, username): - return self.request_json_data( + return self.fetch_json_data( "post", "/v1/forgot", data={"username": username}, ) @@ -278,7 +278,7 @@ def fetch_authentication_token(self): return auth.get("access_token") if auth.get("refresh_token"): try: - data = self.request_json_data( + data = self.fetch_json_data( "post", "/v1/login", headers={ diff --git a/platformio/clients/http.py b/platformio/clients/http.py index e18d2eedda..0b4ca3737b 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -12,11 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import os +import socket + import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util -from platformio.exception import PlatformioException +from platformio.exception import PlatformioException, UserSideException + + +PING_REMOTE_HOSTS = [ + "140.82.118.3", # Github.com + "35.231.145.151", # Gitlab.com + "88.198.170.159", # platformio.org + "github.com", + "platformio.org", +] class HTTPClientError(PlatformioException): @@ -29,6 +42,15 @@ def __str__(self): # pragma: no cover return self.message +class InternetIsOffline(UserSideException): + + MESSAGE = ( + "You are not connected to the Internet.\n" + "PlatformIO needs the Internet connection to" + " download dependent packages or to work with PIO Account." + ) + + class HTTPClient(object): def __init__( self, base_url, @@ -57,7 +79,7 @@ def __del__(self): def send_request(self, method, path, **kwargs): # check Internet before and resolve issue with 60 seconds timeout # print(self, method, path, kwargs) - util.internet_on(raise_exception=True) + ensure_internet_on(raise_exception=True) # set default timeout if "timeout" not in kwargs: @@ -68,9 +90,18 @@ def send_request(self, method, path, **kwargs): except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: raise HTTPClientError(str(e)) - def request_json_data(self, *args, **kwargs): - response = self.send_request(*args, **kwargs) - return self.raise_error_from_response(response) + def fetch_json_data(self, *args, **kwargs): + cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None + if not cache_valid: + return self.raise_error_from_response(self.send_request(*args, **kwargs)) + cache_key = app.ContentCache.key_from_args(*args, kwargs) + with app.ContentCache() as cc: + result = cc.get(cache_key) + if result is not None: + return json.loads(result) + response = self.send_request(*args, **kwargs) + cc.set(cache_key, response.text, cache_valid) + return self.raise_error_from_response(response) @staticmethod def raise_error_from_response(response, expected_codes=(200, 201, 202)): @@ -84,3 +115,48 @@ def raise_error_from_response(response, expected_codes=(200, 201, 202)): except (KeyError, ValueError): message = response.text raise HTTPClientError(message, response) + + +# +# Helpers +# + + +@util.memoized(expire="10s") +def _internet_on(): + timeout = 2 + socket.setdefaulttimeout(timeout) + for host in PING_REMOTE_HOSTS: + try: + for var in ("HTTP_PROXY", "HTTPS_PROXY"): + if not os.getenv(var) and not os.getenv(var.lower()): + continue + requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) + return True + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((host, 80)) + s.close() + return True + except: # pylint: disable=bare-except + pass + return False + + +def ensure_internet_on(raise_exception=False): + result = _internet_on() + if raise_exception and not result: + raise InternetIsOffline() + return result + + +def fetch_remote_content(*args, **kwargs): + kwargs["headers"] = kwargs.get("headers", {}) + if "User-Agent" not in kwargs["headers"]: + kwargs["headers"]["User-Agent"] = app.get_user_agent() + + if "timeout" not in kwargs: + kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + + r = requests.get(*args, **kwargs) + r.raise_for_status() + return r.text diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 78a43eef09..98115cbfbf 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -21,7 +21,7 @@ import click -from platformio import app, exception, fs, proc, util +from platformio import app, exception, fs, proc from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.package.manager.core import inject_contrib_pysite @@ -130,7 +130,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro nl=False, ) stream = helpers.GDBMIConsoleStream() - with util.capture_std_streams(stream): + with proc.capture_std_streams(stream): helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) stream.close() else: diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 2b1662f298..7c8331805f 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -23,6 +23,7 @@ from twisted.internet import defer # pylint: disable=import-error from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util +from platformio.clients.http import ensure_internet_on from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding, glob_recursive @@ -47,7 +48,7 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None): defer.returnValue(result) # check internet before and resolve issue with 60 seconds timeout - util.internet_on(raise_exception=True) + ensure_internet_on(raise_exception=True) session = helpers.requests_session() if data: diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 03463aaba4..6ca0ee773f 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -28,7 +28,7 @@ save_project_libdeps, ) from platformio.compat import dump_json_to_unicode -from platformio.package.exception import UnknownPackageError +from platformio.package.exception import NotGlobalLibDir, UnknownPackageError from platformio.package.manager.library import LibraryPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import is_ci @@ -97,7 +97,7 @@ def cli(ctx, **options): ) if not storage_dirs: - raise exception.NotGlobalLibDir( + raise NotGlobalLibDir( get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand ) diff --git a/platformio/commands/remote/client/run_or_test.py b/platformio/commands/remote/client/run_or_test.py index c986ad0a79..10a9b008da 100644 --- a/platformio/commands/remote/client/run_or_test.py +++ b/platformio/commands/remote/client/run_or_test.py @@ -20,7 +20,7 @@ from twisted.spread import pb # pylint: disable=import-error -from platformio import util +from platformio import fs from platformio.commands.remote.client.async_base import AsyncClientBase from platformio.commands.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync from platformio.compat import hashlib_encode_data @@ -64,7 +64,7 @@ def generate_project_id(self, path): return "%s-%s" % (os.path.basename(path), h.hexdigest()) def add_project_items(self, psync): - with util.cd(self.options["project_dir"]): + with fs.cd(self.options["project_dir"]): cfg = ProjectConfig.get_instance( os.path.join(self.options["project_dir"], "platformio.ini") ) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index c8c8b9fee0..2411f49cf5 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -19,7 +19,8 @@ import click -from platformio import VERSION, __version__, app, exception, util +from platformio import VERSION, __version__, app, exception +from platformio.clients.http import fetch_remote_content from platformio.compat import WINDOWS from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir @@ -130,7 +131,7 @@ def get_latest_version(): def get_develop_latest_version(): version = None - content = util.fetch_remote_content( + content = fetch_remote_content( "https://raw.githubusercontent.com/platformio/platformio" "/develop/platformio/__init__.py" ) @@ -150,5 +151,5 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - content = util.fetch_remote_content("https://pypi.org/pypi/platformio/json") + content = fetch_remote_content("https://pypi.org/pypi/platformio/json") return json.loads(content)["info"]["version"] diff --git a/platformio/exception.py b/platformio/exception.py index 91fd67cce2..8ae549bc3d 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -30,10 +30,6 @@ class ReturnErrorCode(PlatformioException): MESSAGE = "{0}" -class LockFileTimeoutError(PlatformioException): - pass - - class MinitermException(PlatformioException): pass @@ -47,61 +43,6 @@ class AbortedByUser(UserSideException): MESSAGE = "Aborted by user" -# Package Manager - - -class PlatformIOPackageException(PlatformioException): - pass - - -class UnknownPackage(UserSideException): - - MESSAGE = "Detected unknown package '{0}'" - - -class MissingPackageManifest(PlatformIOPackageException): - - MESSAGE = "Could not find one of '{0}' manifest files in the package" - - -class UndefinedPackageVersion(PlatformIOPackageException): - - MESSAGE = ( - "Could not find a version that satisfies the requirement '{0}'" - " for your system '{1}'" - ) - - -class PackageInstallError(PlatformIOPackageException): - - MESSAGE = ( - "Could not install '{0}' with version requirements '{1}' " - "for your system '{2}'.\n\n" - "Please try this solution -> http://bit.ly/faq-package-manager" - ) - - -# -# Library -# - - -class NotGlobalLibDir(UserSideException): - - MESSAGE = ( - "The `{0}` is not a PlatformIO project.\n\n" - "To manage libraries in global storage `{1}`,\n" - "please use `platformio lib --global {2}` or specify custom storage " - "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" - "Check `platformio lib --help` for details." - ) - - -class InvalidLibConfURL(UserSideException): - - MESSAGE = "Invalid library config URL '{0}'" - - # # UDEV Rules # @@ -143,20 +84,6 @@ class GetLatestVersionError(PlatformioException): MESSAGE = "Can not retrieve the latest PlatformIO version" -class APIRequestError(PlatformioException): - - MESSAGE = "[API] {0}" - - -class InternetIsOffline(UserSideException): - - MESSAGE = ( - "You are not connected to the Internet.\n" - "PlatformIO needs the Internet connection to" - " download dependent packages or to work with PIO Account." - ) - - class InvalidSettingName(UserSideException): MESSAGE = "Invalid setting with the name '{0}'" diff --git a/platformio/fs.py b/platformio/fs.py index 7a592746ec..a4dc6ee4d5 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -176,6 +176,10 @@ def expanduser(path): return os.environ["USERPROFILE"] + path[1:] +def change_filemtime(path, mtime): + os.utime(path, (mtime, mtime)) + + def rmtree(path): def _onerror(func, path, __): try: diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 54b0ad6d21..d70c5ca8f4 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -20,6 +20,7 @@ import semantic_version from platformio import __version__, app, exception, fs, telemetry, util +from platformio.clients import http from platformio.commands import PlatformioCLI from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update @@ -53,9 +54,9 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument check_internal_updates(ctx, "platforms") check_internal_updates(ctx, "libraries") except ( - exception.InternetIsOffline, + http.HTTPClientError, + http.InternetIsOffline, exception.GetLatestVersionError, - exception.APIRequestError, ): click.secho( "Failed to check for PlatformIO upgrades. " @@ -221,7 +222,7 @@ def check_platformio_upgrade(): last_check["platformio_upgrade"] = int(time()) app.set_state_item("last_check", last_check) - util.internet_on(raise_exception=True) + http.ensure_internet_on(raise_exception=True) # Update PlatformIO's Core packages update_core_packages(silent=True) @@ -268,7 +269,7 @@ def check_internal_updates(ctx, what): # pylint: disable=too-many-branches last_check[what + "_update"] = int(time()) app.set_state_item("last_check", last_check) - util.internet_on(raise_exception=True) + http.ensure_internet_on(raise_exception=True) outdated_items = [] pm = PlatformPackageManager() if what == "platforms" else LibraryPackageManager() diff --git a/platformio/package/download.py b/platformio/package/download.py index 7f29e7acf4..0f40fd0dc8 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -21,7 +21,7 @@ import click import requests -from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs from platformio.package.exception import PackageException @@ -134,7 +134,7 @@ def verify(self, checksum=None): def _preserve_filemtime(self, lmdate): timedata = parsedate_tz(lmdate) lmtime = mktime(timedata[:9]) - util.change_filemtime(self._destination, lmtime) + fs.change_filemtime(self._destination, lmtime) def __del__(self): if self._request: diff --git a/platformio/package/exception.py b/platformio/package/exception.py index f32c89ce8c..0f34592f7e 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -58,3 +58,14 @@ class UnknownPackageError(UserSideException): "Could not find a package with '{0}' requirements for your system '%s'" % util.get_systype() ) + + +class NotGlobalLibDir(UserSideException): + + MESSAGE = ( + "The `{0}` is not a PlatformIO project.\n\n" + "To manage libraries in global storage `{1}`,\n" + "please use `platformio lib --global {2}` or specify custom storage " + "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" + "Check `platformio lib --help` for details." + ) diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index 44d2e4cf2f..db4b1d3ff9 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -15,7 +15,7 @@ import os from time import sleep, time -from platformio import exception +from platformio.exception import PlatformioException LOCKFILE_TIMEOUT = 3600 # in seconds, 1 hour LOCKFILE_DELAY = 0.2 @@ -36,7 +36,11 @@ LOCKFILE_CURRENT_INTERFACE = None -class LockFileExists(Exception): +class LockFileExists(PlatformioException): + pass + + +class LockFileTimeoutError(PlatformioException): pass @@ -88,7 +92,7 @@ def acquire(self): sleep(self.delay) elapsed += self.delay - raise exception.LockFileTimeoutError() + raise LockFileTimeoutError() def release(self): self._unlock() diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index d3e8dbb1d7..10fdd978d3 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -16,10 +16,10 @@ import click -from platformio import util from platformio.package.exception import UnknownPackageError from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageSpec from platformio.package.vcsclient import VCSBaseException, VCSClientFactory +from platformio.clients.http import ensure_internet_on class PackageManagerUpdateMixin(object): @@ -97,7 +97,7 @@ def update( # pylint: disable=too-many-arguments ), nl=False, ) - if not util.internet_on(): + if not ensure_internet_on(): if not silent: click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) return pkg diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 2b872ab6d8..9f02b84604 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -17,7 +17,7 @@ import subprocess import sys -from platformio import __core_packages__, exception, util +from platformio import __core_packages__, fs, exception, util from platformio.compat import PY2 from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec @@ -93,7 +93,7 @@ def inject_contrib_pysite(verify_openssl=False): def build_contrib_pysite_deps(target_dir): if os.path.isdir(target_dir): - util.rmtree_(target_dir) + fs.rmtree(target_dir) os.makedirs(target_dir) with open(os.path.join(target_dir, "package.json"), "w") as fp: json.dump( diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 91eabf6af1..2172e67233 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -13,7 +13,7 @@ # limitations under the License. from platformio import util -from platformio.exception import APIRequestError, InternetIsOffline +from platformio.clients.http import HTTPClientError, InternetIsOffline from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager from platformio.package.manager.tool import ToolPackageManager @@ -176,7 +176,7 @@ def get_all_boards(self): key = "%s:%s" % (board["platform"], board["id"]) if key not in know_boards: boards.append(board) - except (APIRequestError, InternetIsOffline): + except (HTTPClientError, InternetIsOffline): pass return sorted(boards, key=lambda b: b["name"]) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index d453c83efc..689de80b7e 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -20,6 +20,7 @@ import tarfile from platformio import util +from platformio.clients.http import fetch_remote_content from platformio.compat import get_object_members, string_types from platformio.package.exception import ManifestParserError, UnknownManifestError from platformio.project.helpers import is_platformio_project @@ -106,7 +107,7 @@ def new_from_dir(cls, path, remote_url=None): @staticmethod def new_from_url(remote_url): - content = util.fetch_remote_content(remote_url) + content = fetch_remote_content(remote_url) return ManifestParserFactory.new( content, ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON, diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 7dafaa2368..39327f4a55 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -21,7 +21,7 @@ import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates -from platformio import util +from platformio.clients.http import fetch_remote_content from platformio.package.exception import ManifestValidationError from platformio.util import memoized @@ -256,4 +256,4 @@ def load_spdx_licenses(): "https://raw.githubusercontent.com/spdx/license-list-data" "/v%s/json/licenses.json" % version ) - return json.loads(util.fetch_remote_content(spdx_data_url)) + return json.loads(fetch_remote_content(spdx_data_url)) diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index a00873cd6a..9956b46a07 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -19,7 +19,7 @@ import click -from platformio import util +from platformio import fs from platformio.package.exception import PackageException @@ -109,7 +109,7 @@ def preserve_permissions(item, dest_dir): @staticmethod def preserve_mtime(item, dest_dir): - util.change_filemtime( + fs.change_filemtime( os.path.join(dest_dir, item.filename), mktime(tuple(item.date_time) + tuple([0, 0, 0])), ) diff --git a/platformio/proc.py b/platformio/proc.py index 04f15a579c..82f5a9cf2e 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -15,6 +15,7 @@ import os import subprocess import sys +from contextlib import contextmanager from threading import Thread from platformio import exception @@ -137,6 +138,17 @@ def exec_command(*args, **kwargs): return result +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr + + def is_ci(): return os.getenv("CI", "").lower() == "true" diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 7435bdabbd..5e5878c8fd 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -124,7 +124,7 @@ def _filter_args(items): caller_id = str(app.get_session_var("caller_id")) self["cd1"] = util.get_systype() self["cd4"] = ( - 1 if (not util.is_ci() and (caller_id or not is_container())) else 0 + 1 if (not is_ci() and (caller_id or not is_container())) else 0 ) if caller_id: self["cd5"] = caller_id.lower() diff --git a/platformio/util.py b/platformio/util.py index 982b0bab10..a0686377b3 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -19,26 +19,17 @@ import os import platform import re -import socket import sys import time -from contextlib import contextmanager from functools import wraps from glob import glob import click -import requests -from platformio import DEFAULT_REQUESTS_TIMEOUT, __apiurl__, __version__, exception -from platformio.commands import PlatformioCLI +from platformio import __version__, exception, proc from platformio.compat import PY2, WINDOWS -from platformio.fs import cd # pylint: disable=unused-import from platformio.fs import load_json # pylint: disable=unused-import -from platformio.fs import rmtree as rmtree_ # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import -from platformio.proc import is_ci # pylint: disable=unused-import - -# KEEP unused imports for backward compatibility with PIO Core 3.0 API class memoized(object): @@ -97,17 +88,6 @@ def get_instance(*args, **kwargs): return get_instance -@contextmanager -def capture_std_streams(stdout, stderr=None): - _stdout = sys.stdout - _stderr = sys.stderr - sys.stdout = stdout - sys.stderr = stderr or stdout - yield - sys.stdout = _stdout - sys.stderr = _stderr - - def get_systype(): type_ = platform.system().lower() arch = platform.machine().lower() @@ -116,16 +96,6 @@ def get_systype(): return "%s_%s" % (type_, arch) if arch else type_ -def pioversion_to_intstr(): - vermatch = re.match(r"^([\d\.]+)", __version__) - assert vermatch - return [int(i) for i in vermatch.group(1).split(".")[:3]] - - -def change_filemtime(path, mtime): - os.utime(path, (mtime, mtime)) - - def get_serial_ports(filter_hwid=False): try: # pylint: disable=import-outside-toplevel @@ -164,7 +134,7 @@ def get_logical_devices(): items = [] if WINDOWS: try: - result = exec_command( + result = proc.exec_command( ["wmic", "logicaldisk", "get", "name,VolumeName"] ).get("out", "") devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?") @@ -177,12 +147,12 @@ def get_logical_devices(): except WindowsError: # pylint: disable=undefined-variable pass # try "fsutil" - result = exec_command(["fsutil", "fsinfo", "drives"]).get("out", "") + result = proc.exec_command(["fsutil", "fsinfo", "drives"]).get("out", "") for device in re.findall(r"[A-Z]:\\", result): items.append({"path": device, "name": None}) return items - result = exec_command(["df"]).get("out") + result = proc.exec_command(["df"]).get("out") devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) for line in result.split("\n"): match = devicenamere.match(line.strip()) @@ -370,60 +340,27 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): ) -PING_REMOTE_HOSTS = [ - "140.82.118.3", # Github.com - "35.231.145.151", # Gitlab.com - "88.198.170.159", # platformio.org - "github.com", - "platformio.org", -] - - -@memoized(expire="10s") -def _internet_on(): - timeout = 2 - socket.setdefaulttimeout(timeout) - for host in PING_REMOTE_HOSTS: - try: - for var in ("HTTP_PROXY", "HTTPS_PROXY"): - if not os.getenv(var, var.lower()): - continue - requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) - return True - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80)) - return True - except: # pylint: disable=bare-except - pass - return False - - -def internet_on(raise_exception=False): - result = _internet_on() - if raise_exception and not result: - raise exception.InternetIsOffline() - return result - - -def fetch_remote_content(*args, **kwargs): - # pylint: disable=import-outside-toplevel - from platformio.app import get_user_agent - - kwargs["headers"] = kwargs.get("headers", {}) - if "User-Agent" not in kwargs["headers"]: - kwargs["headers"]["User-Agent"] = get_user_agent() - - if "timeout" not in kwargs: - kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT - - r = requests.get(*args, **kwargs) - r.raise_for_status() - return r.text +def pioversion_to_intstr(): + vermatch = re.match(r"^([\d\.]+)", __version__) + assert vermatch + return [int(i) for i in vermatch.group(1).split(".")[:3]] def pepver_to_semver(pepver): return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) +def get_original_version(version): + if version.count(".") != 2: + return None + _, raw = version.split(".")[:2] + if int(raw) <= 99: + return None + if int(raw) <= 9999: + return "%s.%s" % (raw[:-2], int(raw[-2:])) + return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) + + def items_to_list(items): if isinstance(items, list): return items @@ -472,14 +409,3 @@ def humanize_duration_time(duration): tokens.append(int(round(duration) if multiplier == 1 else fraction)) duration -= fraction * multiplier return "{:02d}:{:02d}:{:02d}.{:03d}".format(*tokens) - - -def get_original_version(version): - if version.count(".") != 2: - return None - _, raw = version.split(".")[:2] - if int(raw) <= 99: - return None - if int(raw) <= 9999: - return "%s.%s" % (raw[:-2], int(raw[-2:])) - return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 655449b002..596c0f297a 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -389,7 +389,7 @@ def test_check_pvs_studio_free_license(clirunner, tmpdir): assert style == 0 -def test_check_embedded_platform_all_tools(clirunner, tmpdir): +def test_check_embedded_platform_all_tools(clirunner, validate_cliresult, tmpdir): config = """ [env:test] platform = ststm32 @@ -422,11 +422,9 @@ def test_check_embedded_platform_all_tools(clirunner, tmpdir): for framework in frameworks: for tool in ("cppcheck", "clangtidy", "pvs-studio"): tmpdir.join("platformio.ini").write(config % (framework, tool)) - result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) - + validate_cliresult(result) defects = sum(count_defects(result.output)) - assert result.exit_code == 0 and defects > 0, "Failed %s with %s" % ( framework, tool, diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 9f07286813..e0a64a8cf0 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -16,12 +16,12 @@ import pytest -from platformio import util +from platformio import proc from platformio.commands.test.command import cli as cmd_test def test_local_env(): - result = util.exec_command( + result = proc.exec_command( [ "platformio", "test", diff --git a/tests/conftest.py b/tests/conftest.py index 56a59cbd9b..d81f0e8ad4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,7 +20,7 @@ import pytest from click.testing import CliRunner -from platformio import util +from platformio.clients import http def pytest_configure(config): @@ -74,7 +74,7 @@ def fin(): @pytest.fixture(scope="function") def without_internet(monkeypatch): - monkeypatch.setattr(util, "_internet_on", lambda: False) + monkeypatch.setattr(http, "_internet_on", lambda: False) @pytest.fixture diff --git a/tests/test_examples.py b/tests/test_examples.py index ada20d35c1..994eb8c014 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -19,7 +19,7 @@ import pytest -from platformio import util +from platformio import fs, proc from platformio.compat import PY2 from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory @@ -64,14 +64,14 @@ def pytest_generate_tests(metafunc): def test_run(pioproject_dir): - with util.cd(pioproject_dir): + with fs.cd(pioproject_dir): config = ProjectConfig() build_dir = config.get_optional_dir("build") if isdir(build_dir): - util.rmtree_(build_dir) + fs.rmtree(build_dir) env_names = config.envs() - result = util.exec_command( + result = proc.exec_command( ["platformio", "run", "-e", random.choice(env_names)] ) if result["returncode"] != 0: diff --git a/tests/test_misc.py b/tests/test_misc.py index ae019f6b97..f816fe6daf 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -17,29 +17,33 @@ import pytest import requests -from platformio import exception, util +from platformio import proc +from platformio.clients import http +from platformio.clients.registry import RegistryClient def test_platformio_cli(): - result = util.exec_command(["pio", "--help"]) + result = proc.exec_command(["pio", "--help"]) assert result["returncode"] == 0 # pylint: disable=unsupported-membership-test assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result["out"] def test_ping_internet_ips(): - for host in util.PING_REMOTE_HOSTS: + for host in http.PING_REMOTE_HOSTS: requests.get("http://%s" % host, allow_redirects=False, timeout=2) def test_api_internet_offline(without_internet, isolated_pio_core): - with pytest.raises(exception.InternetIsOffline): - util.get_api_result("/stats") + regclient = RegistryClient() + with pytest.raises(http.InternetIsOffline): + regclient.fetch_json_data("get", "/v2/stats") def test_api_cache(monkeypatch, isolated_pio_core): - api_kwargs = {"url": "/stats", "cache_valid": "10s"} - result = util.get_api_result(**api_kwargs) + regclient = RegistryClient() + api_kwargs = {"method": "get", "path": "/v2/stats", "cache_valid": "10s"} + result = regclient.fetch_json_data(**api_kwargs) assert result and "boards" in result - monkeypatch.setattr(util, "_internet_on", lambda: False) - assert util.get_api_result(**api_kwargs) == result + monkeypatch.setattr(http, "_internet_on", lambda: False) + assert regclient.fetch_json_data(**api_kwargs) == result From 102aa5f22bec5c7c79de71f3db3b8b252e7123f8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 17:49:29 +0300 Subject: [PATCH 170/223] Port legacy API requests to the new registry client --- platformio/__init__.py | 2 - platformio/clients/registry.py | 15 ++-- platformio/commands/lib/command.py | 24 ++++--- platformio/commands/platform.py | 6 +- platformio/package/manager/platform.py | 7 +- platformio/util.py | 98 -------------------------- 6 files changed, 28 insertions(+), 124 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b51a0be079..0d25faf8ba 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -39,8 +39,6 @@ __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" -__apiurl__ = "https://api.platformio.org" - __accounts_api__ = "https://api.accounts.platformio.org" __registry_api__ = "https://api.registry.platformio.org" __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index f8130c60bc..6111846d34 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -17,11 +17,6 @@ from platformio.clients.http import HTTPClient, HTTPClientError from platformio.package.meta import PackageType -try: - from urllib.parse import quote -except ImportError: - from urllib import quote - # pylint: disable=too-many-arguments @@ -35,7 +30,7 @@ def send_auth_request(self, *args, **kwargs): token = AccountClient().fetch_authentication_token() headers["Authorization"] = "Bearer %s" % token kwargs["headers"] = headers - return self.request_json_data(*args, **kwargs) + return self.fetch_json_data(*args, **kwargs) def publish_package( self, archive_path, owner=None, released_at=None, private=False, notify=True @@ -123,17 +118,17 @@ def list_packages(self, query=None, filters=None, page=None): search_query.append('%s:"%s"' % (name[:-1], value)) if query: search_query.append(query) - params = dict(query=quote(" ".join(search_query))) + params = dict(query=" ".join(search_query)) if page: params["page"] = int(page) - return self.request_json_data("get", "/v3/packages", params=params) + return self.fetch_json_data("get", "/v3/packages", params=params) def get_package(self, type_, owner, name, version=None): try: - return self.request_json_data( + return self.fetch_json_data( "get", "/v3/packages/{owner}/{type}/{name}".format( - type=type_, owner=owner.lower(), name=quote(name.lower()) + type=type_, owner=owner.lower(), name=name.lower() ), params=dict(version=version) if version else None, ) diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 6ca0ee773f..96d39814b2 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -347,6 +347,7 @@ def lib_list(ctx, json_output): help="Do not prompt, automatically paginate with delay", ) def lib_search(query, json_output, page, noninteractive, **filters): + regclient = LibraryPackageManager().get_registry_client_instance() if not query: query = [] if not isinstance(query, list): @@ -356,8 +357,11 @@ def lib_search(query, json_output, page, noninteractive, **filters): for value in values: query.append('%s:"%s"' % (key, value)) - result = util.get_api_result( - "/v2/lib/search", dict(query=" ".join(query), page=page), cache_valid="1d" + result = regclient.fetch_json_data( + "get", + "/v2/lib/search", + params=dict(query=" ".join(query), page=page), + cache_valid="1d", ) if json_output: @@ -406,9 +410,10 @@ def lib_search(query, json_output, page, noninteractive, **filters): time.sleep(5) elif not click.confirm("Show next libraries?"): break - result = util.get_api_result( + result = regclient.fetch_json_data( + "get", "/v2/lib/search", - {"query": " ".join(query), "page": int(result["page"]) + 1}, + params=dict(query=" ".join(query), page=int(result["page"]) + 1), cache_valid="1d", ) @@ -438,10 +443,10 @@ def lib_builtin(storage, json_output): @click.argument("library", metavar="[LIBRARY]") @click.option("--json-output", is_flag=True) def lib_show(library, json_output): - lib_id = LibraryPackageManager().reveal_registry_package_id( - library, silent=json_output - ) - lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") + lm = LibraryPackageManager() + lib_id = lm.reveal_registry_package_id(library, silent=json_output) + regclient = lm.get_registry_client_instance() + lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h") if json_output: return click.echo(dump_json_to_unicode(lib)) @@ -534,7 +539,8 @@ def lib_register(config_url): # pylint: disable=unused-argument @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) def lib_stats(json_output): - result = util.get_api_result("/lib/stats", cache_valid="1h") + regclient = LibraryPackageManager().get_registry_client_instance() + result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h") if json_output: return click.echo(dump_json_to_unicode(result)) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index b996a16afa..803149b111 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -57,7 +57,8 @@ def _print_platforms(platforms): def _get_registry_platforms(): - return util.get_api_result("/platforms", cache_valid="7d") + regclient = PlatformPackageManager().get_registry_client_instance() + return regclient.fetch_json_data("get", "/v2/platforms", cache_valid="1d") def _get_platform_data(*args, **kwargs): @@ -188,8 +189,9 @@ def platform_search(query, json_output): @click.argument("query", required=False) @click.option("--json-output", is_flag=True) def platform_frameworks(query, json_output): + regclient = PlatformPackageManager().get_registry_client_instance() frameworks = [] - for framework in util.get_api_result("/frameworks", cache_valid="7d"): + for framework in regclient.fetch_json_data("get", "/v2/frameworks", cache_valid="1d"): if query == "all": query = "" search_data = dump_json_to_unicode(framework) diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index 2172e67233..71e8c5fbfe 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -164,9 +164,10 @@ def get_installed_boards(self): boards.append(board) return boards - @staticmethod - def get_registered_boards(): - return util.get_api_result("/boards", cache_valid="7d") + def get_registered_boards(self): + return self.get_registry_client_instance().fetch_json_data( + "get", "/v2/boards", cache_valid="1d" + ) def get_all_boards(self): boards = self.get_installed_boards() diff --git a/platformio/util.py b/platformio/util.py index a0686377b3..04576b2444 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -242,104 +242,6 @@ def get_services(self): return items -@memoized(expire="60s") -def _api_request_session(): - return requests.Session() - - -@throttle(500) -def _get_api_result( - url, params=None, data=None, auth=None # pylint: disable=too-many-branches -): - # pylint: disable=import-outside-toplevel - from platformio.app import get_user_agent, get_setting - - result = {} - r = None - verify_ssl = sys.version_info >= (2, 7, 9) - - if not url.startswith("http"): - url = __apiurl__ + url - if not get_setting("strict_ssl"): - url = url.replace("https://", "http://") - - headers = {"User-Agent": get_user_agent()} - try: - if data: - r = _api_request_session().post( - url, - params=params, - data=data, - headers=headers, - auth=auth, - verify=verify_ssl, - timeout=DEFAULT_REQUESTS_TIMEOUT, - ) - else: - r = _api_request_session().get( - url, - params=params, - headers=headers, - auth=auth, - verify=verify_ssl, - timeout=DEFAULT_REQUESTS_TIMEOUT, - ) - result = r.json() - r.raise_for_status() - return r.text - except requests.exceptions.HTTPError as e: - if result and "message" in result: - raise exception.APIRequestError(result["message"]) - if result and "errors" in result: - raise exception.APIRequestError(result["errors"][0]["title"]) - raise exception.APIRequestError(e) - except ValueError: - raise exception.APIRequestError("Invalid response: %s" % r.text.encode("utf-8")) - finally: - if r: - r.close() - return None - - -def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): - from platformio.app import ContentCache # pylint: disable=import-outside-toplevel - - total = 0 - max_retries = 5 - cache_key = ( - ContentCache.key_from_args(url, params, data, auth) if cache_valid else None - ) - while total < max_retries: - try: - with ContentCache() as cc: - if cache_key: - result = cc.get(cache_key) - if result is not None: - return json.loads(result) - - # check internet before and resolve issue with 60 seconds timeout - internet_on(raise_exception=True) - - result = _get_api_result(url, params, data) - if cache_valid: - with ContentCache() as cc: - cc.set(cache_key, result, cache_valid) - return json.loads(result) - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - total += 1 - if not PlatformioCLI.in_silence(): - click.secho( - "[API] ConnectionError: {0} (incremented retry: max={1}, " - "total={2})".format(e, max_retries, total), - fg="yellow", - ) - time.sleep(2 * total) - - raise exception.APIRequestError( - "Could not connect to PlatformIO API Service. Please try later." - ) - - def pioversion_to_intstr(): vermatch = re.match(r"^([\d\.]+)", __version__) assert vermatch From abae9c7e771492673efdabaed8f008daca390bc3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 17:52:12 +0300 Subject: [PATCH 171/223] Cache base registry requests --- platformio/clients/registry.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 6111846d34..e990a65f7e 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -121,7 +121,9 @@ def list_packages(self, query=None, filters=None, page=None): params = dict(query=" ".join(search_query)) if page: params["page"] = int(page) - return self.fetch_json_data("get", "/v3/packages", params=params) + return self.fetch_json_data( + "get", "/v3/packages", params=params, cache_valid="1h" + ) def get_package(self, type_, owner, name, version=None): try: @@ -131,6 +133,7 @@ def get_package(self, type_, owner, name, version=None): type=type_, owner=owner.lower(), name=name.lower() ), params=dict(version=version) if version else None, + cache_valid="1h", ) except HTTPClientError as e: if e.response.status_code == 404: From 7e4bfb1959b1876ed54da2c6d2e193de06d124b5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 20:05:14 +0300 Subject: [PATCH 172/223] Move CacheContent API to "cache.py" module --- README.rst | 1 - platformio/app.py | 168 ++---------------- platformio/cache.py | 165 +++++++++++++++++ platformio/clients/account.py | 4 +- platformio/clients/http.py | 16 +- platformio/commands/debug/process/client.py | 9 +- platformio/commands/home/rpc/handlers/misc.py | 8 +- platformio/commands/home/rpc/handlers/os.py | 9 +- platformio/commands/platform.py | 10 +- platformio/commands/update.py | 4 +- platformio/maintenance.py | 3 +- platformio/package/manager/_download.py | 2 +- platformio/package/manager/_update.py | 2 +- platformio/package/manager/core.py | 2 +- platformio/telemetry.py | 4 +- 15 files changed, 215 insertions(+), 192 deletions(-) create mode 100644 platformio/cache.py diff --git a/README.rst b/README.rst index fcff06c8d4..c4ab3d5f53 100644 --- a/README.rst +++ b/README.rst @@ -147,7 +147,6 @@ Share minimal diagnostics and usage information to help us make PlatformIO bette It is enabled by default. For more information see: * `Telemetry Setting `_ -* `SSL Setting `_ License ------- diff --git a/platformio/app.py b/platformio/app.py index 0933892b28..5990050065 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -14,7 +14,6 @@ from __future__ import absolute_import -import codecs import getpass import hashlib import json @@ -22,18 +21,12 @@ import platform import socket import uuid -from os import environ, getenv, listdir, remove from os.path import dirname, isdir, isfile, join, realpath -from time import time from platformio import __version__, exception, fs, proc from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.package.lockfile import LockFile -from platformio.project.helpers import ( - get_default_projects_dir, - get_project_cache_dir, - get_project_core_dir, -) +from platformio.project.helpers import get_default_projects_dir, get_project_core_dir def projects_dir_validate(projects_dir): @@ -63,10 +56,9 @@ def projects_dir_validate(projects_dir): "value": 7, }, "enable_cache": { - "description": "Enable caching for API requests and Library Manager", + "description": "Enable caching for HTTP API requests", "value": True, }, - "strict_ssl": {"description": "Strict SSL for PlatformIO Services", "value": False}, "enable_telemetry": { "description": ("Telemetry service (Yes/No)"), "value": True, @@ -173,146 +165,6 @@ def __contains__(self, item): return item in self._storage -class ContentCache(object): - def __init__(self, cache_dir=None): - self.cache_dir = None - self._db_path = None - self._lockfile = None - - self.cache_dir = cache_dir or get_project_cache_dir() - self._db_path = join(self.cache_dir, "db.data") - - def __enter__(self): - self.delete() - return self - - def __exit__(self, type_, value, traceback): - pass - - def _lock_dbindex(self): - if not self.cache_dir: - os.makedirs(self.cache_dir) - self._lockfile = LockFile(self.cache_dir) - try: - self._lockfile.acquire() - except: # pylint: disable=bare-except - return False - - return True - - def _unlock_dbindex(self): - if self._lockfile: - self._lockfile.release() - return True - - def get_cache_path(self, key): - assert "/" not in key and "\\" not in key - key = str(key) - assert len(key) > 3 - return join(self.cache_dir, key[-2:], key) - - @staticmethod - def key_from_args(*args): - h = hashlib.md5() - for arg in args: - if arg: - h.update(hashlib_encode_data(arg)) - return h.hexdigest() - - def get(self, key): - cache_path = self.get_cache_path(key) - if not isfile(cache_path): - return None - with codecs.open(cache_path, "rb", encoding="utf8") as fp: - return fp.read() - - def set(self, key, data, valid): - if not get_setting("enable_cache"): - return False - cache_path = self.get_cache_path(key) - if isfile(cache_path): - self.delete(key) - if not data: - return False - if not isdir(self.cache_dir): - os.makedirs(self.cache_dir) - tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} - assert valid.endswith(tuple(tdmap)) - expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) - - if not self._lock_dbindex(): - return False - - if not isdir(dirname(cache_path)): - os.makedirs(dirname(cache_path)) - try: - with codecs.open(cache_path, "wb", encoding="utf8") as fp: - fp.write(data) - with open(self._db_path, "a") as fp: - fp.write("%s=%s\n" % (str(expire_time), cache_path)) - except UnicodeError: - if isfile(cache_path): - try: - remove(cache_path) - except OSError: - pass - - return self._unlock_dbindex() - - def delete(self, keys=None): - """ Keys=None, delete expired items """ - if not isfile(self._db_path): - return None - if not keys: - keys = [] - if not isinstance(keys, list): - keys = [keys] - paths_for_delete = [self.get_cache_path(k) for k in keys] - found = False - newlines = [] - with open(self._db_path) as fp: - for line in fp.readlines(): - line = line.strip() - if "=" not in line: - continue - expire, path = line.split("=") - try: - if ( - time() < int(expire) - and isfile(path) - and path not in paths_for_delete - ): - newlines.append(line) - continue - except ValueError: - pass - found = True - if isfile(path): - try: - remove(path) - if not listdir(dirname(path)): - fs.rmtree(dirname(path)) - except OSError: - pass - - if found and self._lock_dbindex(): - with open(self._db_path, "w") as fp: - fp.write("\n".join(newlines) + "\n") - self._unlock_dbindex() - - return True - - def clean(self): - if not self.cache_dir or not isdir(self.cache_dir): - return - fs.rmtree(self.cache_dir) - - -def clean_cache(): - with ContentCache() as cc: - cc.clean() - - def sanitize_setting(name, value): if name not in DEFAULT_SETTINGS: raise exception.InvalidSettingName(name) @@ -350,8 +202,8 @@ def delete_state_item(name): def get_setting(name): _env_name = "PLATFORMIO_SETTING_%s" % name.upper() - if _env_name in environ: - return sanitize_setting(name, getenv(_env_name)) + if _env_name in os.environ: + return sanitize_setting(name, os.getenv(_env_name)) with State() as state: if "settings" in state and name in state["settings"]: @@ -388,7 +240,7 @@ def is_disabled_progressbar(): [ get_session_var("force_option"), proc.is_ci(), - getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", + os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", ] ) @@ -401,15 +253,15 @@ def get_cid(): if cid: return cid uid = None - if getenv("C9_UID"): - uid = getenv("C9_UID") - elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): + if os.getenv("C9_UID"): + uid = os.getenv("C9_UID") + elif os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")): try: uid = json.loads( fetch_remote_content( "{api}/user?token={token}".format( - api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), - token=getenv("USER_TOKEN"), + api=os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")), + token=os.getenv("USER_TOKEN"), ) ) ).get("id") diff --git a/platformio/cache.py b/platformio/cache.py new file mode 100644 index 0000000000..bc817f61ef --- /dev/null +++ b/platformio/cache.py @@ -0,0 +1,165 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import codecs +import hashlib +import os +from time import time + +from platformio import app, fs +from platformio.compat import hashlib_encode_data +from platformio.package.lockfile import LockFile +from platformio.project.helpers import get_project_cache_dir + + +class ContentCache(object): + def __init__(self, namespace=None): + self.cache_dir = os.path.join(get_project_cache_dir(), namespace or "content") + self._db_path = os.path.join(self.cache_dir, "db.data") + self._lockfile = None + if not os.path.isdir(self.cache_dir): + os.makedirs(self.cache_dir) + + def __enter__(self): + # cleanup obsolete items + self.delete() + return self + + def __exit__(self, type_, value, traceback): + pass + + @staticmethod + def key_from_args(*args): + h = hashlib.sha1() + for arg in args: + if arg: + h.update(hashlib_encode_data(arg)) + return h.hexdigest() + + def get_cache_path(self, key): + assert "/" not in key and "\\" not in key + key = str(key) + assert len(key) > 3 + return os.path.join(self.cache_dir, key) + + def get(self, key): + cache_path = self.get_cache_path(key) + if not os.path.isfile(cache_path): + return None + with codecs.open(cache_path, "rb", encoding="utf8") as fp: + return fp.read() + + def set(self, key, data, valid): + if not app.get_setting("enable_cache"): + return False + cache_path = self.get_cache_path(key) + if os.path.isfile(cache_path): + self.delete(key) + if not data: + return False + tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} + assert valid.endswith(tuple(tdmap)) + expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) + + if not self._lock_dbindex(): + return False + + if not os.path.isdir(os.path.dirname(cache_path)): + os.makedirs(os.path.dirname(cache_path)) + try: + with codecs.open(cache_path, "wb", encoding="utf8") as fp: + fp.write(data) + with open(self._db_path, "a") as fp: + fp.write("%s=%s\n" % (str(expire_time), os.path.basename(cache_path))) + except UnicodeError: + if os.path.isfile(cache_path): + try: + os.remove(cache_path) + except OSError: + pass + + return self._unlock_dbindex() + + def delete(self, keys=None): + """ Keys=None, delete expired items """ + if not os.path.isfile(self._db_path): + return None + if not keys: + keys = [] + if not isinstance(keys, list): + keys = [keys] + paths_for_delete = [self.get_cache_path(k) for k in keys] + found = False + newlines = [] + with open(self._db_path) as fp: + for line in fp.readlines(): + line = line.strip() + if "=" not in line: + continue + expire, fname = line.split("=") + path = os.path.join(self.cache_dir, fname) + try: + if ( + time() < int(expire) + and os.path.isfile(path) + and path not in paths_for_delete + ): + newlines.append(line) + continue + except ValueError: + pass + found = True + if os.path.isfile(path): + try: + os.remove(path) + if not os.listdir(os.path.dirname(path)): + fs.rmtree(os.path.dirname(path)) + except OSError: + pass + + if found and self._lock_dbindex(): + with open(self._db_path, "w") as fp: + fp.write("\n".join(newlines) + "\n") + self._unlock_dbindex() + + return True + + def clean(self): + if not os.path.isdir(self.cache_dir): + return + fs.rmtree(self.cache_dir) + + def _lock_dbindex(self): + self._lockfile = LockFile(self.cache_dir) + try: + self._lockfile.acquire() + except: # pylint: disable=bare-except + return False + + return True + + def _unlock_dbindex(self): + if self._lockfile: + self._lockfile.release() + return True + + +# +# Helpers +# + + +def cleanup_content_cache(namespace=None): + with ContentCache(namespace) as cc: + cc.clean() diff --git a/platformio/clients/account.py b/platformio/clients/account.py index d492b65df4..ba2c34516f 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -153,9 +153,7 @@ def auth_token(self, password, regenerate): ).get("auth_token") def forgot_password(self, username): - return self.fetch_json_data( - "post", "/v1/forgot", data={"username": username}, - ) + return self.fetch_json_data("post", "/v1/forgot", data={"username": username},) def get_profile(self): return self.send_auth_request("get", "/v1/profile",) diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 0b4ca3737b..b1330f2ea3 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -20,9 +20,9 @@ from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util +from platformio.cache import ContentCache from platformio.exception import PlatformioException, UserSideException - PING_REMOTE_HOSTS = [ "140.82.118.3", # Github.com "35.231.145.151", # Gitlab.com @@ -90,16 +90,20 @@ def send_request(self, method, path, **kwargs): except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: raise HTTPClientError(str(e)) - def fetch_json_data(self, *args, **kwargs): + def fetch_json_data(self, method, path, **kwargs): cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None if not cache_valid: - return self.raise_error_from_response(self.send_request(*args, **kwargs)) - cache_key = app.ContentCache.key_from_args(*args, kwargs) - with app.ContentCache() as cc: + return self.raise_error_from_response( + self.send_request(method, path, **kwargs) + ) + cache_key = ContentCache.key_from_args( + method, path, kwargs.get("params"), kwargs.get("data") + ) + with ContentCache("http") as cc: result = cc.get(cache_key) if result is not None: return json.loads(result) - response = self.send_request(*args, **kwargs) + response = self.send_request(method, path, **kwargs) cc.set(cache_key, response.text, cache_valid) return self.raise_error_from_response(response) diff --git a/platformio/commands/debug/process/client.py b/platformio/commands/debug/process/client.py index a58438b755..45374727c1 100644 --- a/platformio/commands/debug/process/client.py +++ b/platformio/commands/debug/process/client.py @@ -26,7 +26,8 @@ from twisted.internet import stdio # pylint: disable=import-error from twisted.internet import task # pylint: disable=import-error -from platformio import app, fs, proc, telemetry, util +from platformio import fs, proc, telemetry, util +from platformio.cache import ContentCache from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.debug.initcfgs import get_gdb_init_config @@ -252,7 +253,7 @@ def _handle_error(self, data): def _kill_previous_session(self): assert self._session_id pid = None - with app.ContentCache() as cc: + with ContentCache() as cc: pid = cc.get(self._session_id) cc.delete(self._session_id) if not pid: @@ -269,11 +270,11 @@ def _kill_previous_session(self): def _lock_session(self, pid): if not self._session_id: return - with app.ContentCache() as cc: + with ContentCache() as cc: cc.set(self._session_id, str(pid), "1h") def _unlock_session(self): if not self._session_id: return - with app.ContentCache() as cc: + with ContentCache() as cc: cc.delete(self._session_id) diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index a216344e96..a4bdc6522a 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -17,15 +17,15 @@ from twisted.internet import defer, reactor # pylint: disable=import-error -from platformio import app +from platformio.cache import ContentCache from platformio.commands.home.rpc.handlers.os import OSRPC class MiscRPC(object): def load_latest_tweets(self, data_url): - cache_key = app.ContentCache.key_from_args(data_url, "tweets") + cache_key = ContentCache.key_from_args(data_url, "tweets") cache_valid = "180d" - with app.ContentCache() as cc: + with ContentCache() as cc: cache_data = cc.get(cache_key) if cache_data: cache_data = json.loads(cache_data) @@ -43,7 +43,7 @@ def load_latest_tweets(self, data_url): @defer.inlineCallbacks def _preload_latest_tweets(data_url, cache_key, cache_valid): result = json.loads((yield OSRPC.fetch_content(data_url))) - with app.ContentCache() as cc: + with ContentCache() as cc: cc.set( cache_key, json.dumps({"time": int(time.time()), "result": result}), diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 7c8331805f..38bdcb8acc 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -22,7 +22,8 @@ import click from twisted.internet import defer # pylint: disable=import-error -from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs, util +from platformio import DEFAULT_REQUESTS_TIMEOUT, fs, util +from platformio.cache import ContentCache from platformio.clients.http import ensure_internet_on from platformio.commands.home import helpers from platformio.compat import PY2, get_filesystem_encoding, glob_recursive @@ -40,8 +41,8 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None): "Safari/603.3.8" ) } - cache_key = app.ContentCache.key_from_args(uri, data) if cache_valid else None - with app.ContentCache() as cc: + cache_key = ContentCache.key_from_args(uri, data) if cache_valid else None + with ContentCache() as cc: if cache_key: result = cc.get(cache_key) if result is not None: @@ -63,7 +64,7 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None): r.raise_for_status() result = r.text if cache_valid: - with app.ContentCache() as cc: + with ContentCache() as cc: cc.set(cache_key, result, cache_valid) defer.returnValue(result) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 803149b111..2bfe9ebbec 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -16,7 +16,8 @@ import click -from platformio import app, util +from platformio import util +from platformio.cache import cleanup_content_cache from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode from platformio.package.manager.platform import PlatformPackageManager @@ -191,7 +192,9 @@ def platform_search(query, json_output): def platform_frameworks(query, json_output): regclient = PlatformPackageManager().get_registry_client_instance() frameworks = [] - for framework in regclient.fetch_json_data("get", "/v2/frameworks", cache_valid="1d"): + for framework in regclient.fetch_json_data( + "get", "/v2/frameworks", cache_valid="1d" + ): if query == "all": query = "" search_data = dump_json_to_unicode(framework) @@ -401,7 +404,8 @@ def platform_update( # pylint: disable=too-many-locals, too-many-arguments return click.echo(dump_json_to_unicode(result)) # cleanup cached board and platform lists - app.clean_cache() + cleanup_content_cache("http") + for platform in platforms: click.echo( "Platform %s" diff --git a/platformio/commands/update.py b/platformio/commands/update.py index b1e15a4345..ff88723efc 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -14,7 +14,7 @@ import click -from platformio import app +from platformio.cache import cleanup_content_cache from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update @@ -38,7 +38,7 @@ @click.pass_context def cli(ctx, core_packages, only_check, dry_run): # cleanup lib search results, cached board and platform lists - app.clean_cache() + cleanup_content_cache("http") only_check = dry_run or only_check diff --git a/platformio/maintenance.py b/platformio/maintenance.py index d70c5ca8f4..b8dd67fd52 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -20,6 +20,7 @@ import semantic_version from platformio import __version__, app, exception, fs, telemetry, util +from platformio.cache import cleanup_content_cache from platformio.clients import http from platformio.commands import PlatformioCLI from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY @@ -160,7 +161,7 @@ def after_upgrade(ctx): else: click.secho("Please wait while upgrading PlatformIO...", fg="yellow") try: - app.clean_cache() + cleanup_content_cache("http") except: # pylint: disable=bare-except pass diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 34295287c4..4039568b61 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -27,7 +27,7 @@ class PackageManagerDownloadMixin(object): DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month def compute_download_path(self, *args): - request_hash = hashlib.new("sha256") + request_hash = hashlib.new("sha1") for arg in args: request_hash.update(compat.hashlib_encode_data(arg)) dl_path = os.path.join(self.get_download_dir(), request_hash.hexdigest()) diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py index 10fdd978d3..3b6dd2d44d 100644 --- a/platformio/package/manager/_update.py +++ b/platformio/package/manager/_update.py @@ -16,10 +16,10 @@ import click +from platformio.clients.http import ensure_internet_on from platformio.package.exception import UnknownPackageError from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageSpec from platformio.package.vcsclient import VCSBaseException, VCSClientFactory -from platformio.clients.http import ensure_internet_on class PackageManagerUpdateMixin(object): diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 9f02b84604..7eed98215a 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -17,7 +17,7 @@ import subprocess import sys -from platformio import __core_packages__, fs, exception, util +from platformio import __core_packages__, exception, fs, util from platformio.compat import PY2 from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 5e5878c8fd..4c5a67066b 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -123,9 +123,7 @@ def _filter_args(items): caller_id = str(app.get_session_var("caller_id")) self["cd1"] = util.get_systype() - self["cd4"] = ( - 1 if (not is_ci() and (caller_id or not is_container())) else 0 - ) + self["cd4"] = 1 if (not is_ci() and (caller_id or not is_container())) else 0 if caller_id: self["cd5"] = caller_id.lower() From 95151062f5054203d0d4aa3abddd27e021910809 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 22:52:29 +0300 Subject: [PATCH 173/223] Implement mirroring for HTTP client --- platformio/__init__.py | 5 +- platformio/clients/account.py | 2 +- platformio/clients/http.py | 83 ++++++++++++++++++++----- platformio/clients/registry.py | 2 +- platformio/package/manager/_registry.py | 24 +++---- 5 files changed, 84 insertions(+), 32 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0d25faf8ba..b68f3371a2 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -40,7 +40,10 @@ __copyright__ = "Copyright 2014-present PlatformIO" __accounts_api__ = "https://api.accounts.platformio.org" -__registry_api__ = "https://api.registry.platformio.org" +__registry_api__ = [ + "https://api.registry.platformio.org", + "https://api.registry.ns1.platformio.org", +] __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __core_packages__ = { diff --git a/platformio/clients/account.py b/platformio/clients/account.py index ba2c34516f..e2abde1798 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -40,7 +40,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__(self): - super(AccountClient, self).__init__(base_url=__accounts_api__) + super(AccountClient, self).__init__(__accounts_api__) @staticmethod def get_refresh_token(): diff --git a/platformio/clients/http.py b/platformio/clients/http.py index b1330f2ea3..8ee15b3530 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -13,6 +13,7 @@ # limitations under the License. import json +import math import os import socket @@ -23,6 +24,12 @@ from platformio.cache import ContentCache from platformio.exception import PlatformioException, UserSideException +try: + from urllib.parse import urljoin +except ImportError: + from urlparse import urljoin + + PING_REMOTE_HOSTS = [ "140.82.118.3", # Github.com "35.231.145.151", # Gitlab.com @@ -51,23 +58,54 @@ class InternetIsOffline(UserSideException): ) -class HTTPClient(object): - def __init__( - self, base_url, - ): - if base_url.endswith("/"): - base_url = base_url[:-1] +class EndpointSession(requests.Session): + def __init__(self, base_url, *args, **kwargs): + super(EndpointSession, self).__init__(*args, **kwargs) self.base_url = base_url - self._session = requests.Session() - self._session.headers.update({"User-Agent": app.get_user_agent()}) - retry = Retry( - total=5, + + def request( # pylint: disable=signature-differs,arguments-differ + self, method, url, *args, **kwargs + ): + print(self.base_url, method, url, args, kwargs) + return super(EndpointSession, self).request( + method, urljoin(self.base_url, url), *args, **kwargs + ) + + +class EndpointSessionIterator(object): + def __init__(self, endpoints): + if not isinstance(endpoints, list): + endpoints = [endpoints] + self.endpoints = endpoints + self.endpoints_iter = iter(endpoints) + self.retry = Retry( + total=math.ceil(6 / len(self.endpoints)), backoff_factor=1, # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], status_forcelist=[413, 429, 500, 502, 503, 504], ) - adapter = requests.adapters.HTTPAdapter(max_retries=retry) - self._session.mount(base_url, adapter) + + def __iter__(self): # pylint: disable=non-iterator-returned + return self + + def next(self): + """ For Python 2 compatibility """ + return self.__next__() + + def __next__(self): + base_url = next(self.endpoints_iter) + session = EndpointSession(base_url) + session.headers.update({"User-Agent": app.get_user_agent()}) + adapter = requests.adapters.HTTPAdapter(max_retries=self.retry) + session.mount(base_url, adapter) + return session + + +class HTTPClient(object): + def __init__(self, endpoints): + self._session_iter = EndpointSessionIterator(endpoints) + self._session = None + self._next_session() def __del__(self): if not self._session: @@ -75,20 +113,31 @@ def __del__(self): self._session.close() self._session = None + def _next_session(self): + if self._session: + self._session.close() + self._session = next(self._session_iter) + @util.throttle(500) def send_request(self, method, path, **kwargs): # check Internet before and resolve issue with 60 seconds timeout - # print(self, method, path, kwargs) ensure_internet_on(raise_exception=True) # set default timeout if "timeout" not in kwargs: kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT - try: - return getattr(self._session, method)(self.base_url + path, **kwargs) - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - raise HTTPClientError(str(e)) + while True: + try: + return getattr(self._session, method)(path, **kwargs) + except ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + ) as e: + try: + self._next_session() + except: # pylint: disable=bare-except + raise HTTPClientError(str(e)) def fetch_json_data(self, method, path, **kwargs): cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index e990a65f7e..c8fbeeeadf 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -22,7 +22,7 @@ class RegistryClient(HTTPClient): def __init__(self): - super(RegistryClient, self).__init__(base_url=__registry_api__) + super(RegistryClient, self).__init__(__registry_api__) def send_auth_request(self, *args, **kwargs): headers = kwargs.get("headers", {}) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 72f189fb27..415a997797 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -27,19 +27,23 @@ from urlparse import urlparse -class RegistryFileMirrorsIterator(object): +class RegistryFileMirrorIterator(object): HTTP_CLIENT_INSTANCES = {} def __init__(self, download_url): self.download_url = download_url self._url_parts = urlparse(download_url) - self._base_url = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) + self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) self._visited_mirrors = [] def __iter__(self): # pylint: disable=non-iterator-returned return self + def next(self): + """ For Python 2 compatibility """ + return self.__next__() + def __next__(self): http = self.get_http_client() response = http.send_request( @@ -64,16 +68,12 @@ def __next__(self): response.headers.get("X-PIO-Content-SHA256"), ) - def next(self): - """ For Python 2 compatibility """ - return self.__next__() - def get_http_client(self): - if self._base_url not in RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES: - RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[ - self._base_url - ] = HTTPClient(self._base_url) - return RegistryFileMirrorsIterator.HTTP_CLIENT_INSTANCES[self._base_url] + if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES: + RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient( + self._mirror + ) + return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] class PackageManageRegistryMixin(object): @@ -98,7 +98,7 @@ def install_from_registry(self, spec, search_filters=None, silent=False): if not pkgfile: raise UnknownPackageError(spec.humanize()) - for url, checksum in RegistryFileMirrorsIterator(pkgfile["download_url"]): + for url, checksum in RegistryFileMirrorIterator(pkgfile["download_url"]): try: return self.install_from_url( url, From c2caf8b839981de390c435b60fe41f18d6aef9fc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 22:53:41 +0300 Subject: [PATCH 174/223] Bump version to 4.4.0b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b68f3371a2..edf20f6e5b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -16,7 +16,7 @@ DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) -VERSION = (4, 4, "0b1") +VERSION = (4, 4, "0b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From dcf91c49acd267654ac2c38baf56b197638e517d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 22 Aug 2020 22:56:26 +0300 Subject: [PATCH 175/223] Remove debug code --- platformio/clients/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 8ee15b3530..318448c721 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -66,7 +66,7 @@ def __init__(self, base_url, *args, **kwargs): def request( # pylint: disable=signature-differs,arguments-differ self, method, url, *args, **kwargs ): - print(self.base_url, method, url, args, kwargs) + # print(self.base_url, method, url, args, kwargs) return super(EndpointSession, self).request( method, urljoin(self.base_url, url), *args, **kwargs ) From e2bb81bae4ec00887dd734136da262734c0eaec5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 13:22:11 +0300 Subject: [PATCH 176/223] Restore legacy util.cd API --- platformio/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/util.py b/platformio/util.py index 04576b2444..f2c485a6a5 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -28,6 +28,7 @@ from platformio import __version__, exception, proc from platformio.compat import PY2, WINDOWS +from platformio.fs import cd # pylint: disable=unused-import from platformio.fs import load_json # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import From 8ea10a18d3de95311f67397b4fc71850d33dbea0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 13:22:38 +0300 Subject: [PATCH 177/223] Bump version to 4.4.0b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index edf20f6e5b..f50be5feef 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -16,7 +16,7 @@ DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) -VERSION = (4, 4, "0b2") +VERSION = (4, 4, "0b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From da179cb33fb615bf907f71dd1619fd70c9381df3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 14:29:31 +0300 Subject: [PATCH 178/223] Enhance configuration variables --- platformio/__init__.py | 12 ++++++++++-- platformio/clients/http.py | 17 ++++------------- platformio/commands/home/rpc/handlers/os.py | 6 +++--- platformio/package/download.py | 4 ++-- tests/test_misc.py | 4 ++-- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index f50be5feef..83f86b3bb8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,8 +14,6 @@ import sys -DEFAULT_REQUESTS_TIMEOUT = (10, None) # (connect, read) - VERSION = (4, 4, "0b3") __version__ = ".".join([str(s) for s in VERSION]) @@ -46,6 +44,8 @@ ] __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" +__default_requests_timeout__ = (10, None) # (connect, read) + __core_packages__ = { "contrib-piohome": "~3.2.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), @@ -55,3 +55,11 @@ "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.7.0", } + +__check_internet_hosts__ = [ + "140.82.118.3", # Github.com + "35.231.145.151", # Gitlab.com + "88.198.170.159", # platformio.org + "github.com", + "platformio.org", +] diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 318448c721..8e73295853 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -20,7 +20,7 @@ import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error -from platformio import DEFAULT_REQUESTS_TIMEOUT, app, util +from platformio import __check_internet_hosts__, __default_requests_timeout__, app, util from platformio.cache import ContentCache from platformio.exception import PlatformioException, UserSideException @@ -30,15 +30,6 @@ from urlparse import urljoin -PING_REMOTE_HOSTS = [ - "140.82.118.3", # Github.com - "35.231.145.151", # Gitlab.com - "88.198.170.159", # platformio.org - "github.com", - "platformio.org", -] - - class HTTPClientError(PlatformioException): def __init__(self, message, response=None): super(HTTPClientError, self).__init__() @@ -125,7 +116,7 @@ def send_request(self, method, path, **kwargs): # set default timeout if "timeout" not in kwargs: - kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + kwargs["timeout"] = __default_requests_timeout__ while True: try: @@ -179,7 +170,7 @@ def raise_error_from_response(response, expected_codes=(200, 201, 202)): def _internet_on(): timeout = 2 socket.setdefaulttimeout(timeout) - for host in PING_REMOTE_HOSTS: + for host in __check_internet_hosts__: try: for var in ("HTTP_PROXY", "HTTPS_PROXY"): if not os.getenv(var) and not os.getenv(var.lower()): @@ -208,7 +199,7 @@ def fetch_remote_content(*args, **kwargs): kwargs["headers"]["User-Agent"] = app.get_user_agent() if "timeout" not in kwargs: - kwargs["timeout"] = DEFAULT_REQUESTS_TIMEOUT + kwargs["timeout"] = __default_requests_timeout__ r = requests.get(*args, **kwargs) r.raise_for_status() diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 38bdcb8acc..448c633a50 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -22,7 +22,7 @@ import click from twisted.internet import defer # pylint: disable=import-error -from platformio import DEFAULT_REQUESTS_TIMEOUT, fs, util +from platformio import __default_requests_timeout__, fs, util from platformio.cache import ContentCache from platformio.clients.http import ensure_internet_on from platformio.commands.home import helpers @@ -54,11 +54,11 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None): session = helpers.requests_session() if data: r = yield session.post( - uri, data=data, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT + uri, data=data, headers=headers, timeout=__default_requests_timeout__ ) else: r = yield session.get( - uri, headers=headers, timeout=DEFAULT_REQUESTS_TIMEOUT + uri, headers=headers, timeout=__default_requests_timeout__ ) r.raise_for_status() diff --git a/platformio/package/download.py b/platformio/package/download.py index 0f40fd0dc8..bd425ac630 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -21,7 +21,7 @@ import click import requests -from platformio import DEFAULT_REQUESTS_TIMEOUT, app, fs +from platformio import __default_requests_timeout__, app, fs from platformio.package.exception import PackageException @@ -33,7 +33,7 @@ def __init__(self, url, dest_dir=None): url, stream=True, headers={"User-Agent": app.get_user_agent()}, - timeout=DEFAULT_REQUESTS_TIMEOUT, + timeout=__default_requests_timeout__, ) if self._request.status_code != 200: raise PackageException( diff --git a/tests/test_misc.py b/tests/test_misc.py index f816fe6daf..36574ee426 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -17,7 +17,7 @@ import pytest import requests -from platformio import proc +from platformio import __check_internet_hosts__, proc from platformio.clients import http from platformio.clients.registry import RegistryClient @@ -30,7 +30,7 @@ def test_platformio_cli(): def test_ping_internet_ips(): - for host in http.PING_REMOTE_HOSTS: + for host in __check_internet_hosts__: requests.get("http://%s" % host, allow_redirects=False, timeout=2) From 620241e067a26e17a35d2e95a2c9aeae266999f4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 15:24:31 +0300 Subject: [PATCH 179/223] Move package "version" related things to "platformio.package.version" module --- platformio/builder/tools/pioplatform.py | 3 +- platformio/builder/tools/platformio.py | 11 +++-- platformio/commands/platform.py | 4 +- platformio/maintenance.py | 19 +++------ platformio/package/meta.py | 21 +--------- platformio/package/version.py | 53 +++++++++++++++++++++++++ platformio/platform/base.py | 9 +++-- platformio/util.py | 16 +------- 8 files changed, 78 insertions(+), 58 deletions(-) create mode 100644 platformio/package/version.py diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 5ca7794f3e..ec5c6c4cca 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -23,6 +23,7 @@ from platformio import fs, util from platformio.compat import WINDOWS from platformio.package.meta import PackageItem +from platformio.package.version import get_original_version from platformio.platform.exception import UnknownBoard from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectOptions @@ -210,7 +211,7 @@ def _get_debug_data(): def _get_packages_data(): data = [] for item in platform.dump_used_packages(): - original_version = util.get_original_version(item["version"]) + original_version = get_original_version(item["version"]) info = "%s %s" % (item["name"], item["version"]) extra = [] if original_version: diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 5d8f8e8bca..aac4742621 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -26,9 +26,9 @@ from SCons.Script import Export # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error -from platformio import fs +from platformio import __version__, fs from platformio.compat import string_types -from platformio.util import pioversion_to_intstr +from platformio.package.version import pepver_to_semver SRC_HEADER_EXT = ["h", "hpp"] SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"] @@ -94,11 +94,16 @@ def BuildProgram(env): def ProcessProgramDeps(env): def _append_pio_macros(): + core_version = pepver_to_semver(__version__) env.AppendUnique( CPPDEFINES=[ ( "PLATFORMIO", - int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())), + int( + "{0:02d}{1:02d}{2:02d}".format( + core_version.major, core_version.minor, core_version.patch + ) + ), ) ] ) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 2bfe9ebbec..7725be392e 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -16,12 +16,12 @@ import click -from platformio import util from platformio.cache import cleanup_content_cache from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode from platformio.package.manager.platform import PlatformPackageManager from platformio.package.meta import PackageItem, PackageSpec +from platformio.package.version import get_original_version from platformio.platform.exception import UnknownPlatform from platformio.platform.factory import PlatformFactory @@ -121,7 +121,7 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru continue item[key] = value if key == "version": - item["originalVersion"] = util.get_original_version(value) + item["originalVersion"] = get_original_version(value) data["packages"].append(item) return data diff --git a/platformio/maintenance.py b/platformio/maintenance.py index b8dd67fd52..1900db4930 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -19,7 +19,7 @@ import click import semantic_version -from platformio import __version__, app, exception, fs, telemetry, util +from platformio import __version__, app, exception, fs, telemetry from platformio.cache import cleanup_content_cache from platformio.clients import http from platformio.commands import PlatformioCLI @@ -32,6 +32,7 @@ from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec +from platformio.package.version import pepver_to_semver from platformio.platform.factory import PlatformFactory from platformio.proc import is_container @@ -87,12 +88,8 @@ def set_caller(caller=None): class Upgrader(object): def __init__(self, from_version, to_version): - self.from_version = semantic_version.Version.coerce( - util.pepver_to_semver(from_version) - ) - self.to_version = semantic_version.Version.coerce( - util.pepver_to_semver(to_version) - ) + self.from_version = pepver_to_semver(from_version) + self.to_version = pepver_to_semver(to_version) self._upgraders = [ (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms), @@ -141,9 +138,7 @@ def after_upgrade(ctx): if last_version == "0.0.0": app.set_state_item("last_version", __version__) - elif semantic_version.Version.coerce( - util.pepver_to_semver(last_version) - ) > semantic_version.Version.coerce(util.pepver_to_semver(__version__)): + elif pepver_to_semver(last_version) > pepver_to_semver(__version__): click.secho("*" * terminal_width, fg="yellow") click.secho( "Obsolete PIO Core v%s is used (previous was %s)" @@ -229,9 +224,7 @@ def check_platformio_upgrade(): update_core_packages(silent=True) latest_version = get_latest_version() - if semantic_version.Version.coerce( - util.pepver_to_semver(latest_version) - ) <= semantic_version.Version.coerce(util.pepver_to_semver(__version__)): + if pepver_to_semver(latest_version) <= pepver_to_semver(__version__): return terminal_width, _ = click.get_terminal_size() diff --git a/platformio/package/meta.py b/platformio/package/meta.py index fa93780e3f..2715206fef 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -357,28 +357,9 @@ def version(self, value): self._version = ( value if isinstance(value, semantic_version.Version) - else self.to_semver(value) + else cast_version_to_semver(value) ) - @staticmethod - def to_semver(value, force=True, raise_exception=False): - assert value - try: - return semantic_version.Version(value) - except ValueError: - pass - if force: - try: - return semantic_version.Version.coerce(value) - except ValueError: - pass - if raise_exception: - raise ValueError("Invalid SemVer version %s" % value) - # parse commit hash - if re.match(r"^[\da-f]+$", value, flags=re.I): - return semantic_version.Version("0.0.0+sha." + value) - return semantic_version.Version("0.0.0+" + value) - def as_dict(self): return dict( type=self.type, diff --git a/platformio/package/version.py b/platformio/package/version.py new file mode 100644 index 0000000000..770be9e4b2 --- /dev/null +++ b/platformio/package/version.py @@ -0,0 +1,53 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import semantic_version + + +def cast_version_to_semver(value, force=True, raise_exception=False): + assert value + try: + return semantic_version.Version(value) + except ValueError: + pass + if force: + try: + return semantic_version.Version.coerce(value) + except ValueError: + pass + if raise_exception: + raise ValueError("Invalid SemVer version %s" % value) + # parse commit hash + if re.match(r"^[\da-f]+$", value, flags=re.I): + return semantic_version.Version("0.0.0+sha." + value) + return semantic_version.Version("0.0.0+" + value) + + +def pepver_to_semver(pepver): + return cast_version_to_semver( + re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) + ) + + +def get_original_version(version): + if version.count(".") != 2: + return None + _, raw = version.split(".")[:2] + if int(raw) <= 99: + return None + if int(raw) <= 9999: + return "%s.%s" % (raw[:-2], int(raw[-2:])) + return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 8e49288d10..0c061a110a 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -18,8 +18,9 @@ import click import semantic_version -from platformio import __version__, fs, proc, util +from platformio import __version__, fs, proc from platformio.package.manager.tool import ToolPackageManager +from platformio.package.version import pepver_to_semver from platformio.platform._packages import PlatformPackagesMixin from platformio.platform._run import PlatformRunMixin from platformio.platform.board import PlatformBoardConfig @@ -31,7 +32,7 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub PlatformPackagesMixin, PlatformRunMixin ): - PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) + CORE_SEMVER = pepver_to_semver(__version__) _BOARDS_CACHE = {} def __init__(self, manifest_path): @@ -110,10 +111,10 @@ def python_packages(self): def ensure_engine_compatible(self): if not self.engines or "platformio" not in self.engines: return True - if self.PIO_VERSION in semantic_version.SimpleSpec(self.engines["platformio"]): + if self.CORE_SEMVER in semantic_version.SimpleSpec(self.engines["platformio"]): return True raise IncompatiblePlatform( - self.name, str(self.PIO_VERSION), self.engines["platformio"] + self.name, str(self.CORE_SEMVER), self.engines["platformio"] ) def get_dir(self): diff --git a/platformio/util.py b/platformio/util.py index f2c485a6a5..aeeaf55bce 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -244,26 +244,12 @@ def get_services(self): def pioversion_to_intstr(): + """ Legacy for framework-zephyr/scripts/platformio/platformio-build-pre.py""" vermatch = re.match(r"^([\d\.]+)", __version__) assert vermatch return [int(i) for i in vermatch.group(1).split(".")[:3]] -def pepver_to_semver(pepver): - return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) - - -def get_original_version(version): - if version.count(".") != 2: - return None - _, raw = version.split(".")[:2] - if int(raw) <= 99: - return None - if int(raw) <= 9999: - return "%s.%s" % (raw[:-2], int(raw[-2:])) - return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) - - def items_to_list(items): if isinstance(items, list): return items From 1c8aca2f6a63a18149fe5e0e5e9069f218f83a42 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 15:25:03 +0300 Subject: [PATCH 180/223] Check ALL possible version for the first matched package --- platformio/package/manager/_registry.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 415a997797..f8af2ece3a 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -19,7 +19,8 @@ from platformio.clients.http import HTTPClient from platformio.clients.registry import RegistryClient from platformio.package.exception import UnknownPackageError -from platformio.package.meta import PackageMetaData, PackageSpec +from platformio.package.meta import PackageSpec +from platformio.package.version import cast_version_to_semver try: from urllib.parse import urlparse @@ -185,17 +186,13 @@ def print_multi_package_issue(self, packages, spec): ) def find_best_registry_version(self, packages, spec): - # find compatible version within the latest package versions for package in packages: + # find compatible version within the latest package versions version = self.pick_best_registry_version([package["version"]], spec) if version: return (package, version) - if not spec.requirements: - return (None, None) - - # if the custom version requirements, check ALL package versions - for package in packages: + # if the custom version requirements, check ALL package versions version = self.pick_best_registry_version( self.fetch_registry_package( PackageSpec( @@ -215,14 +212,14 @@ def pick_best_registry_version(self, versions, spec=None): assert not spec or isinstance(spec, PackageSpec) best = None for version in versions: - semver = PackageMetaData.to_semver(version["name"]) + semver = cast_version_to_semver(version["name"]) if spec and spec.requirements and semver not in spec.requirements: continue if not any( self.is_system_compatible(f.get("system")) for f in version["files"] ): continue - if not best or (semver > PackageMetaData.to_semver(best["name"])): + if not best or (semver > cast_version_to_semver(best["name"])): best = version return best From a069bae1fbb99449b6ac975d95e8b188529bec4d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 15:26:58 +0300 Subject: [PATCH 181/223] Fix a bug with package updating when version is not in SemVer format // Resolve #3635 --- platformio/package/meta.py | 3 ++- tests/package/test_manager.py | 16 +++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 2715206fef..147a1faf2d 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -22,6 +22,7 @@ from platformio.compat import get_object_members, hashlib_encode_data, string_types from platformio.package.manifest.parser import ManifestFileType +from platformio.package.version import cast_version_to_semver try: from urllib.parse import urlparse @@ -89,7 +90,7 @@ def __setattr__(self, name, value): and name in ("current", "latest", "wanted") and not isinstance(value, semantic_version.Version) ): - value = semantic_version.Version(str(value)) + value = cast_version_to_semver(str(value)) return super(PackageOutdatedResult, self).__setattr__(name, value) def is_outdated(self, allow_incompatible=False): diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 2c331dbea1..f5939f156e 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -392,8 +392,14 @@ def test_registry(isolated_pio_core): def test_update_with_metadata(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") lm = LibraryPackageManager(str(storage_dir)) - pkg = lm.install("ArduinoJson @ 5.10.1", silent=True) + # test non SemVer in registry + pkg = lm.install("RadioHead @ <1.90", silent=True) + outdated = lm.outdated(pkg) + assert str(outdated.current) == "1.89.0" + assert outdated.latest > semantic_version.Version("1.100.0") + + pkg = lm.install("ArduinoJson @ 5.10.1", silent=True) # tesy latest outdated = lm.outdated(pkg) assert str(outdated.current) == "5.10.1" @@ -411,7 +417,7 @@ def test_update_with_metadata(isolated_pio_core, tmpdir_factory): new_pkg = lm.update("ArduinoJson@^5", PackageSpec("ArduinoJson@^5"), silent=True) assert str(new_pkg.metadata.version) == "5.13.4" # check that old version is removed - assert len(lm.get_installed()) == 1 + assert len(lm.get_installed()) == 2 # update to the latest lm = LibraryPackageManager(str(storage_dir)) @@ -422,7 +428,7 @@ def test_update_with_metadata(isolated_pio_core, tmpdir_factory): def test_update_without_metadata(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") storage_dir.join("legacy-package").mkdir().join("library.json").write( - '{"name": "AsyncMqttClient-esphome", "version": "0.8.2"}' + '{"name": "AsyncMqttClient-esphome", "version": "0.8"}' ) storage_dir.join("legacy-dep").mkdir().join("library.json").write( '{"name": "AsyncTCP-esphome", "version": "1.1.1"}' @@ -431,8 +437,8 @@ def test_update_without_metadata(isolated_pio_core, tmpdir_factory): pkg = lm.get_package("AsyncMqttClient-esphome") outdated = lm.outdated(pkg) assert len(lm.get_installed()) == 2 - assert str(pkg.metadata.version) == "0.8.2" - assert outdated.latest > semantic_version.Version("0.8.2") + assert str(pkg.metadata.version) == "0.8.0" + assert outdated.latest > semantic_version.Version("0.8.0") # update lm = LibraryPackageManager(str(storage_dir)) From 24f85a337fade368e2a218cb8e4cb7d5f05a4db2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 21:07:14 +0300 Subject: [PATCH 182/223] Fix "AttributeError: module 'platformio.exception' has no attribute 'InternetIsOffline'" --- platformio/builder/tools/piolib.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 8cc1ad58ce..45371ececa 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=no-member, no-self-use, unused-argument, too-many-lines +# pylint: disable=no-self-use, unused-argument, too-many-lines # pylint: disable=too-many-instance-attributes, too-many-public-methods # pylint: disable=assignment-from-no-return @@ -33,6 +33,7 @@ from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool +from platformio.clients.http import InternetIsOffline from platformio.compat import WINDOWS, hashlib_encode_data, string_types from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager @@ -882,7 +883,7 @@ def _is_builtin(spec): try: lm.install(spec) did_install = True - except (UnknownPackageError, exception.InternetIsOffline) as e: + except (UnknownPackageError, InternetIsOffline) as e: click.secho("Warning! %s" % e, fg="yellow") # reset cache From f39c9fb597facb92701afbfc687e37898e7bf43d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 21:07:40 +0300 Subject: [PATCH 183/223] Bump version to 4.4.0b4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 83f86b3bb8..4567a0ad0d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (4, 4, "0b3") +VERSION = (4, 4, "0b4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From b44bc80bd1ffe328b3d9dfe8130ba09e236487f5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Aug 2020 21:41:53 +0300 Subject: [PATCH 184/223] PyLint fix for PY2 --- platformio/builder/tools/piolib.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 45371ececa..35d1462e1a 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -484,6 +484,7 @@ def get_include_dirs(self): def src_filter(self): src_dir = join(self.path, "src") if isdir(src_dir): + # pylint: disable=no-member src_filter = LibBuilderBase.src_filter.fget(self) for root, _, files in os.walk(src_dir, followlinks=True): found = False @@ -518,6 +519,7 @@ def dependencies(self): @property def lib_ldf_mode(self): + # pylint: disable=no-member if not self._manifest.get("dependencies"): return LibBuilderBase.lib_ldf_mode.fget(self) missing = object() @@ -554,7 +556,7 @@ def include_dir(self): def src_dir(self): if isdir(join(self.path, "source")): return join(self.path, "source") - return LibBuilderBase.src_dir.fget(self) + return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) @@ -700,17 +702,18 @@ def include_dir(self): if "includeDir" in self._manifest.get("build", {}): with fs.cd(self.path): return realpath(self._manifest.get("build").get("includeDir")) - return LibBuilderBase.include_dir.fget(self) + return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): with fs.cd(self.path): return realpath(self._manifest.get("build").get("srcDir")) - return LibBuilderBase.src_dir.fget(self) + return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member @property def src_filter(self): + # pylint: disable=no-member if "srcFilter" in self._manifest.get("build", {}): return self._manifest.get("build").get("srcFilter") if self.env["SRC_FILTER"]: @@ -723,19 +726,19 @@ def src_filter(self): def build_flags(self): if "flags" in self._manifest.get("build", {}): return self._manifest.get("build").get("flags") - return LibBuilderBase.build_flags.fget(self) + return LibBuilderBase.build_flags.fget(self) # pylint: disable=no-member @property def build_unflags(self): if "unflags" in self._manifest.get("build", {}): return self._manifest.get("build").get("unflags") - return LibBuilderBase.build_unflags.fget(self) + return LibBuilderBase.build_unflags.fget(self) # pylint: disable=no-member @property def extra_script(self): if "extraScript" in self._manifest.get("build", {}): return self._manifest.get("build").get("extraScript") - return LibBuilderBase.extra_script.fget(self) + return LibBuilderBase.extra_script.fget(self) # pylint: disable=no-member @property def lib_archive(self): @@ -747,12 +750,14 @@ def lib_archive(self): return self.env.GetProjectConfig().get( "env:" + self.env["PIOENV"], "lib_archive" ) + # pylint: disable=no-member return self._manifest.get("build", {}).get( "libArchive", LibBuilderBase.lib_archive.fget(self) ) @property def lib_ldf_mode(self): + # pylint: disable=no-member return self.validate_ldf_mode( self._manifest.get("build", {}).get( "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self) @@ -761,6 +766,7 @@ def lib_ldf_mode(self): @property def lib_compat_mode(self): + # pylint: disable=no-member return self.validate_compat_mode( self._manifest.get("build", {}).get( "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self) @@ -835,7 +841,7 @@ def get_search_files(self): @property def lib_ldf_mode(self): - mode = LibBuilderBase.lib_ldf_mode.fget(self) + mode = LibBuilderBase.lib_ldf_mode.fget(self) # pylint: disable=no-member if not mode.startswith("chain"): return mode # parse all project files @@ -843,6 +849,7 @@ def lib_ldf_mode(self): @property def src_filter(self): + # pylint: disable=no-member return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self) @property @@ -1037,7 +1044,7 @@ def _print_deps_tree(root, level=0): _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") - ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) + ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member click.echo("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") click.echo( From d6d95e05e8991d635f072be07f5b8c2e32c96747 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 15:09:37 +0300 Subject: [PATCH 185/223] Rename "fs.format_filesize" to "fs.humanize_file_size" --- platformio/builder/tools/pioplatform.py | 3 ++- platformio/commands/boards.py | 4 ++-- platformio/fs.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index ec5c6c4cca..7dd36a64ef 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -179,7 +179,8 @@ def _get_hardware_data(): ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") data.append( - "%s RAM, %s Flash" % (fs.format_filesize(ram), fs.format_filesize(flash)) + "%s RAM, %s Flash" + % (fs.humanize_file_size(ram), fs.humanize_file_size(flash)) ) return data diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 21614b1356..962ab504d6 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -59,8 +59,8 @@ def print_boards(boards): click.style(b["id"], fg="cyan"), b["mcu"], "%dMHz" % (b["fcpu"] / 1000000), - fs.format_filesize(b["rom"]), - fs.format_filesize(b["ram"]), + fs.humanize_file_size(b["rom"]), + fs.humanize_file_size(b["ram"]), b["name"], ) for b in boards diff --git a/platformio/fs.py b/platformio/fs.py index a4dc6ee4d5..75bf959c42 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -57,7 +57,7 @@ def load_json(file_path): raise exception.InvalidJSONFile(file_path) -def format_filesize(filesize): +def humanize_file_size(filesize): base = 1024 unit = 0 suffix = "B" From 13db51a5560d8c7dfd6dd87622fd671b9aa8fa3e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 15:10:38 +0300 Subject: [PATCH 186/223] Install/Uninstall dependencies only for library-type packages // Resolve #3637 --- platformio/package/manager/_install.py | 27 ++------------ platformio/package/manager/_uninstall.py | 18 ++-------- platformio/package/manager/library.py | 45 +++++++++++++++++++++++- 3 files changed, 48 insertions(+), 42 deletions(-) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 003c238644..1bafcf8c32 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -21,7 +21,7 @@ from platformio import app, compat, fs, util from platformio.package.exception import PackageException -from platformio.package.meta import PackageItem, PackageSpec +from platformio.package.meta import PackageItem from platformio.package.unpack import FileUnpacker from platformio.package.vcsclient import VCSClientFactory @@ -119,30 +119,7 @@ def _install( # pylint: disable=too-many-arguments return pkg def install_dependencies(self, pkg, silent=False): - assert isinstance(pkg, PackageItem) - manifest = self.load_manifest(pkg) - if not manifest.get("dependencies"): - return - if not silent: - self.print_message("Installing dependencies...") - for dependency in manifest.get("dependencies"): - if not self._install_dependency(dependency, silent) and not silent: - self.print_message( - "Warning! Could not install dependency %s for package '%s'" - % (dependency, pkg.metadata.name), - fg="yellow", - ) - - def _install_dependency(self, dependency, silent=False): - spec = PackageSpec( - name=dependency.get("name"), requirements=dependency.get("version") - ) - search_filters = { - key: value - for key, value in dependency.items() - if key in ("authors", "platforms", "frameworks") - } - return self._install(spec, search_filters=search_filters or None, silent=silent) + pass def install_from_url(self, url, spec, checksum=None, silent=False): spec = self.ensure_spec(spec) diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 2cca850535..322eced6dc 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -19,7 +19,7 @@ from platformio import fs from platformio.package.exception import UnknownPackageError -from platformio.package.meta import PackageItem, PackageSpec +from platformio.package.meta import PackageSpec class PackageManagerUninstallMixin(object): @@ -73,18 +73,4 @@ def _uninstall(self, spec, silent=False, skip_dependencies=False): return pkg def uninstall_dependencies(self, pkg, silent=False): - assert isinstance(pkg, PackageItem) - manifest = self.load_manifest(pkg) - if not manifest.get("dependencies"): - return - if not silent: - self.print_message("Removing dependencies...", fg="yellow") - for dependency in manifest.get("dependencies"): - pkg = self.get_package( - PackageSpec( - name=dependency.get("name"), requirements=dependency.get("version") - ) - ) - if not pkg: - continue - self._uninstall(pkg, silent=silent) + pass diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index 1375e84ee2..a0d1407f9c 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -17,7 +17,7 @@ from platformio.package.exception import MissingPackageManifestError from platformio.package.manager.base import BasePackageManager -from platformio.package.meta import PackageSpec, PackageType +from platformio.package.meta import PackageItem, PackageSpec, PackageType from platformio.project.helpers import get_project_global_lib_dir @@ -62,3 +62,46 @@ def find_library_root(path): return os.path.dirname(root) return root return path + + def install_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message("Installing dependencies...") + for dependency in manifest.get("dependencies"): + if not self._install_dependency(dependency, silent) and not silent: + self.print_message( + "Warning! Could not install dependency %s for package '%s'" + % (dependency, pkg.metadata.name), + fg="yellow", + ) + + def _install_dependency(self, dependency, silent=False): + spec = PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + search_filters = { + key: value + for key, value in dependency.items() + if key in ("authors", "platforms", "frameworks") + } + return self._install(spec, search_filters=search_filters or None, silent=silent) + + def uninstall_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message("Removing dependencies...", fg="yellow") + for dependency in manifest.get("dependencies"): + pkg = self.get_package( + PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + ) + if not pkg: + continue + self._uninstall(pkg, silent=silent) From 3e7e9e2b3d1b75cd76fc623b6adf9927f35f4225 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 15:22:05 +0300 Subject: [PATCH 187/223] Remove unused data using a new `pio system prune // Resolve #3522 --- HISTORY.rst | 17 +++++++++-------- docs | 2 +- platformio/commands/system/command.py | 25 ++++++++++++++++++++++++- platformio/fs.py | 11 +++++++++++ 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 17a8786b0c..538649fa71 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,8 +30,8 @@ PlatformIO Core 4 - Built-in fine-grained access control (role based, teams, organizations) - Command Line Interface: - * `platformio package publish `__ – publish a personal or organization package - * `platformio package unpublish `__ – remove a pushed package from the registry + * `pio package publish `__ – publish a personal or organization package + * `pio package unpublish `__ – remove a pushed package from the registry * Grant package access to the team members or maintainers * New **Package Management System** @@ -46,7 +46,7 @@ PlatformIO Core 4 - Command launcher with own arguments - Launch command with custom options declared in `"platformio.ini" `__ - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) - - List available project targets (including dev-platform specific and custom targets) with a new `platformio run --list-targets `__ command (`issue #3544 `_) + - List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) * **PlatformIO Build System** @@ -58,12 +58,13 @@ PlatformIO Core 4 * **Miscellaneous** - - Display system-wide information using a new `platformio system info `__ command (`issue #3521 `_) - - Dump data intended for IDE extensions/plugins using a new `platformio project idedata `__ command - - Added a new ``-e, --environment`` option to `platformio project init `__ command that helps to update a PlatformIO project using existing environment - - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. - - Do not generate ".travis.yml" for a new project, let the user have a choice + - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) + - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) + - Dump data intended for IDE extensions/plugins using a new `pio project idedata `__ command + - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment + - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. - Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 + - Do not generate ".travis.yml" for a new project, let the user have a choice - Do not escape compiler arguments in VSCode template on Windows - Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) diff --git a/docs b/docs index d01bbede6c..be04ee45c8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d01bbede6ca90420ed24736be4963a48228eff42 +Subproject commit be04ee45c8d037c8eecbbe9a178926593fd620a0 diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index 2fee54710c..cb31120566 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -13,6 +13,7 @@ # limitations under the License. import json +import os import platform import subprocess import sys @@ -20,7 +21,7 @@ import click from tabulate import tabulate -from platformio import __version__, compat, proc, util +from platformio import __version__, compat, fs, proc, util from platformio.commands.system.completion import ( get_completion_install_path, install_completion_code, @@ -30,6 +31,7 @@ from platformio.package.manager.platform import PlatformPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_cache_dir @click.group("system", short_help="Miscellaneous system commands") @@ -95,6 +97,27 @@ def system_info(json_output): ) +@cli.command("prune", short_help="Remove unused data") +@click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation") +def system_prune(force): + click.secho("WARNING! This will remove:", fg="yellow") + click.echo(" - cached API requests") + click.echo(" - cached package downloads") + click.echo(" - temporary data") + if not force: + click.confirm("Do you want to continue?", abort=True) + + reclaimed_total = 0 + cache_dir = get_project_cache_dir() + if os.path.isdir(cache_dir): + reclaimed_total += fs.calculate_folder_size(cache_dir) + fs.rmtree(cache_dir) + + click.secho( + "Total reclaimed space: %s" % fs.humanize_file_size(reclaimed_total), fg="green" + ) + + @cli.group("completion", short_help="Shell completion support") def completion(): # pylint: disable=import-error,import-outside-toplevel diff --git a/platformio/fs.py b/platformio/fs.py index 75bf959c42..da2101c5fb 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -85,6 +85,17 @@ def calculate_file_hashsum(algorithm, path): return h.hexdigest() +def calculate_folder_size(path): + assert os.path.isdir(path) + result = 0 + for root, __, files in os.walk(path): + for f in files: + file_path = os.path.join(root, f) + if not os.path.islink(file_path): + result += os.path.getsize(file_path) + return result + + def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel From 6af2bad12320bc264a8ea3f070d25d5c09f4b75a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 22:56:31 +0300 Subject: [PATCH 188/223] Make PIO Core 4.0 automatically compatible with dev-platforms for PIO Core 2.0 & 3.0 // Resolve #3638 --- platformio/platform/base.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 0c061a110a..ffafd61c2d 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -111,11 +111,13 @@ def python_packages(self): def ensure_engine_compatible(self): if not self.engines or "platformio" not in self.engines: return True - if self.CORE_SEMVER in semantic_version.SimpleSpec(self.engines["platformio"]): + core_spec = semantic_version.SimpleSpec(self.engines["platformio"]) + if self.CORE_SEMVER in core_spec: return True - raise IncompatiblePlatform( - self.name, str(self.CORE_SEMVER), self.engines["platformio"] - ) + # PIO Core 4 is compatible with dev-platforms for PIO Core 2.0 & 3.0 + if any(semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3)): + return True + raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER), str(core_spec)) def get_dir(self): return os.path.dirname(self.manifest_path) From c6a37ef88061c11b6362ff4acc575b7bfa361f8e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 23:04:17 +0300 Subject: [PATCH 189/223] Get real path of just installed core-package --- platformio/package/manager/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 7eed98215a..62b884a8d3 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -34,9 +34,9 @@ def get_core_package_dir(name): pkg = pm.get_package(spec) if pkg: return pkg.path - pkg = pm.install(spec).path + assert pm.install(spec).path _remove_unnecessary_packages() - return pkg + return pm.get_package(spec) def update_core_packages(only_check=False, silent=False): From 655e2856d1c93412162d1de6677e251ce9a50e7b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 23:05:01 +0300 Subject: [PATCH 190/223] Bump version to 4.4.0b5 --- platformio/__init__.py | 2 +- platformio/package/manager/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 4567a0ad0d..735d427638 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (4, 4, "0b4") +VERSION = (4, 4, "0b5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 62b884a8d3..098bfdea3e 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -34,7 +34,7 @@ def get_core_package_dir(name): pkg = pm.get_package(spec) if pkg: return pkg.path - assert pm.install(spec).path + assert pm.install(spec) _remove_unnecessary_packages() return pm.get_package(spec) From e43176e33a5fceac98e7ebf03b03e04ad232fd3c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 23:11:24 +0300 Subject: [PATCH 191/223] Typo fix --- platformio/package/manager/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 098bfdea3e..2d01b1559f 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -36,7 +36,7 @@ def get_core_package_dir(name): return pkg.path assert pm.install(spec) _remove_unnecessary_packages() - return pm.get_package(spec) + return pm.get_package(spec).path def update_core_packages(only_check=False, silent=False): From 091ba4346dbe928887fb5eec4ce3a2e48f946c07 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 24 Aug 2020 23:11:43 +0300 Subject: [PATCH 192/223] Bump version to 4.4.0b6 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 735d427638..071e87bf10 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (4, 4, "0b5") +VERSION = (4, 4, "0b6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From ff19109787fdb5430a16b1590cb0718bac200522 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 14:34:03 +0300 Subject: [PATCH 193/223] Fix test --- tests/commands/test_platform.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 2c197ec3e6..088f4b3884 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -51,12 +51,19 @@ def test_install_unknown_from_registry(clirunner): assert isinstance(result.exception, UnknownPackageError) -def test_install_incompatbile(clirunner, validate_cliresult, isolated_pio_core): +# def test_install_incompatbile(clirunner, validate_cliresult, isolated_pio_core): +# result = clirunner.invoke( +# cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], +# ) +# assert result.exit_code != 0 +# assert isinstance(result.exception, IncompatiblePlatform) + + +def test_install_core_3_dev_platform(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], ) - assert result.exit_code != 0 - assert isinstance(result.exception, IncompatiblePlatform) + assert result.exit_code == 0 def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core): @@ -120,7 +127,7 @@ def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): def test_uninstall(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( - cli_platform.platform_uninstall, ["atmelavr", "espressif8266"] + cli_platform.platform_uninstall, ["atmelavr@1.2.0", "atmelavr", "espressif8266"] ) validate_cliresult(result) assert not isolated_pio_core.join("platforms").listdir() From fa9025171415e8b23ee8dcf98946748db631dbd5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 14:35:01 +0300 Subject: [PATCH 194/223] Fixed an issue when Unit Testing engine fails with a custom project configuration file // Resolve #3583 --- HISTORY.rst | 3 ++- platformio/commands/test/processor.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 538649fa71..4328fe11f2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -66,7 +66,8 @@ PlatformIO Core 4 - Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 - Do not generate ".travis.yml" for a new project, let the user have a choice - Do not escape compiler arguments in VSCode template on Windows - - Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 `_) + - Fixed an issue with Unit Testing engine when running multiple environments (`issue #3523 < https://github.com/platformio/platformio-core/issues/3523>`_) + - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 334db858df..de09b5f9a4 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -138,6 +138,7 @@ def build_or_upload(self, target): return self.cmd_ctx.invoke( cmd_run, project_dir=self.options["project_dir"], + project_conf=self.options["project_config"].path, upload_port=self.options["upload_port"], verbose=self.options["verbose"], silent=self.options["silent"], From 2ea80d91f8eb2cdbd0048583b2d96a2441c5ff33 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 15:55:17 +0300 Subject: [PATCH 195/223] Minor fixes --- HISTORY.rst | 6 +++--- tests/commands/test_platform.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4328fe11f2..e1ddddc4df 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -62,11 +62,11 @@ PlatformIO Core 4 - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) - Dump data intended for IDE extensions/plugins using a new `pio project idedata `__ command - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment - - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required. - - Updated PIO Unit Testing support for Mbed framework. Added compatibility with Mbed OS 6 + - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required + - Updated PIO Unit Testing support for Mbed framework and added compatibility with Mbed OS 6 - Do not generate ".travis.yml" for a new project, let the user have a choice - Do not escape compiler arguments in VSCode template on Windows - - Fixed an issue with Unit Testing engine when running multiple environments (`issue #3523 < https://github.com/platformio/platformio-core/issues/3523>`_) + - Fixed an issue with Unit Testing engine when running multiple environments (`issue #3523 `_) - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) 4.3.4 (2020-05-23) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 088f4b3884..39afbeb5e0 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -18,7 +18,6 @@ from platformio.commands import platform as cli_platform from platformio.package.exception import UnknownPackageError -from platformio.platform.exception import IncompatiblePlatform def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): From 79bfac29baa3000d0254f407feb6fe042f29bc59 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 18:57:20 +0300 Subject: [PATCH 196/223] Update history and sync docs --- HISTORY.rst | 47 +++++++++++++++++++++++++++-------------------- docs | 2 +- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e1ddddc4df..108cc259a8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,12 +11,6 @@ PlatformIO Core 4 **A professional collaborative platform for embedded development** -* Integration with the new `Account Management System `__ - - - Manage own organizations - - Manage organization teams - - Manage resource access - * Integration with the new **PlatformIO Trusted Registry** - Enterprise-grade package storage with high availability (multi replicas) @@ -34,21 +28,27 @@ PlatformIO Core 4 * `pio package unpublish `__ – remove a pushed package from the registry * Grant package access to the team members or maintainers +* Integration with the new `Account Management System `__ + + - Manage own organizations + - Manage organization teams + - Manage resource access + * New **Package Management System** - Integrated PlatformIO Core with the new PlatformIO Trusted Registry - Strict dependency declaration using owner name (resolves name conflicts) (`issue #1824 `_) - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) -* New `Custom Targets `__ +* **PlatformIO Build System** - - Pre/Post processing based on a dependent sources (other target, source file, etc.) - - Command launcher with own arguments - - Launch command with custom options declared in `"platformio.ini" `__ - - Python callback as a target (use the power of Python interpreter and PlatformIO Build API) - - List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) + - New `Custom Targets `__ -* **PlatformIO Build System** + * Pre/Post processing based on a dependent sources (other target, source file, etc.) + * Command launcher with own arguments + * Launch command with custom options declared in `"platformio.ini" `__ + * Python callback as a target (use the power of Python interpreter and PlatformIO Build API) + * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) - Upgraded to `SCons 4.0 - a next-generation software construction tool `__ - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) @@ -56,18 +56,25 @@ PlatformIO Core 4 - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) - Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) +* **Project Management** + + - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required + - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment + - Dump data intended for IDE extensions/plugins using a new `pio project idedata `__ command + - Do not generate ".travis.yml" for a new project, let the user have a choice + +* **Unit Testing** + + - Updated PIO Unit Testing support for Mbed framework and added compatibility with Mbed OS 6 + - Fixed an issue when running multiple test environments (`issue #3523 `_) + - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) + * **Miscellaneous** - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) - - Dump data intended for IDE extensions/plugins using a new `pio project idedata `__ command - - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment - - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required - - Updated PIO Unit Testing support for Mbed framework and added compatibility with Mbed OS 6 - - Do not generate ".travis.yml" for a new project, let the user have a choice - Do not escape compiler arguments in VSCode template on Windows - - Fixed an issue with Unit Testing engine when running multiple environments (`issue #3523 `_) - - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) + 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index be04ee45c8..dfa6701b70 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit be04ee45c8d037c8eecbbe9a178926593fd620a0 +Subproject commit dfa6701b70dac8b8a1449cdff99879f79151589e From b9fe4933365a5f2b9807bbc9baa5f4a4a4b67671 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 19:18:26 +0300 Subject: [PATCH 197/223] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index dfa6701b70..cc09d98135 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit dfa6701b70dac8b8a1449cdff99879f79151589e +Subproject commit cc09d981358a438ad474485a89ee5ddbc1842c65 From 3e72f098fe6e2c52ba1e1094e2b11dd39a9a02c1 Mon Sep 17 00:00:00 2001 From: Valerii Koval Date: Tue, 25 Aug 2020 21:19:21 +0300 Subject: [PATCH 198/223] Updates for PIO Check (#3640) * Update check tools to the latest versions * Use language standard when exporting defines to check tools * Buffer Cppcheck output to detect multiline messages * Add new test for PIO Check * Pass include paths to Clang-Tidy as individual compiler arguments Clang-tidy doesn't support response files which can exceed command length limitations on Windows * Simplify tests for PIO Check * Update history * Sync changelog --- HISTORY.rst | 1 + platformio/__init__.py | 4 +- platformio/commands/check/tools/base.py | 4 +- platformio/commands/check/tools/clangtidy.py | 7 +- platformio/commands/check/tools/cppcheck.py | 18 ++- tests/commands/test_check.py | 117 +++++++++++++------ 6 files changed, 101 insertions(+), 50 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 108cc259a8..24963d3e4e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -82,6 +82,7 @@ PlatformIO Core 4 * Added `PlatformIO CLI Shell Completion `__ for Fish, Zsh, Bash, and PowerShell (`issue #3435 `_) * Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 `_) * Fixed an issue on Windows when installing a library dependency from Git repository (`issue #2844 `_, `issue #3328 `_) +* Fixed an issue with PIO Check when a defect with multiline error message is not reported in verbose mode (`issue #3631 `_) 4.3.3 (2020-04-28) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__init__.py b/platformio/__init__.py index 071e87bf10..0d8d3f17fb 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -51,9 +51,9 @@ "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", - "tool-cppcheck": "~1.190.0", + "tool-cppcheck": "~1.210.0", "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.7.0", + "tool-pvs-studio": "~7.8.0", } __check_internet_hosts__ = [ diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index d873810d42..dc9f476fc4 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -83,7 +83,9 @@ def _extract_defines(language, includes_file): cmd = "echo | %s -x %s %s %s -dM -E -" % ( self.cc_path, language, - " ".join([f for f in build_flags if f.startswith(("-m", "-f"))]), + " ".join( + [f for f in build_flags if f.startswith(("-m", "-f", "-std"))] + ), includes_file, ) result = proc.exec_command(cmd, shell=True) diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index 05be67b4ab..06f3ff7616 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -63,10 +63,7 @@ def configure_command(self): for scope in project_files: src_files.extend(project_files[scope]) - cmd.extend(flags) - cmd.extend(src_files) - cmd.append("--") - + cmd.extend(flags + src_files + ["--"]) cmd.extend( ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines["c++"]] ) @@ -79,6 +76,6 @@ def configure_command(self): continue includes.append(inc) - cmd.append("--extra-arg=" + self._long_includes_hook(includes)) + cmd.extend(["-I%s" % inc for inc in includes]) return cmd diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 931b16edd9..b38bb8d69f 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -24,6 +24,8 @@ class CppcheckCheckTool(CheckToolBase): def __init__(self, *args, **kwargs): + self._field_delimiter = "<&PIO&>" + self._buffer = "" self.defect_fields = [ "severity", "message", @@ -55,13 +57,15 @@ def tool_output_filter(self, line): return line def parse_defect(self, raw_line): - if "<&PIO&>" not in raw_line or any( - f not in raw_line for f in self.defect_fields - ): + if self._field_delimiter not in raw_line: + return None + + self._buffer += raw_line + if any(f not in self._buffer for f in self.defect_fields): return None args = dict() - for field in raw_line.split("<&PIO&>"): + for field in self._buffer.split(self._field_delimiter): field = field.strip().replace('"', "") name, value = field.split("=", 1) args[name] = value @@ -94,6 +98,7 @@ def parse_defect(self, raw_line): self._bad_input = True return None + self._buffer = "" return DefectItem(**args) def configure_command( @@ -103,13 +108,16 @@ def configure_command( cmd = [ tool_path, + "--addon-python=%s" % proc.get_pythonexe_path(), "--error-exitcode=1", "--verbose" if self.options.get("verbose") else "--quiet", ] cmd.append( '--template="%s"' - % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields]) + % self._field_delimiter.join( + ["{0}={{{0}}}".format(f) for f in self.defect_fields] + ) ) flags = self.get_flags("cppcheck") diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 596c0f297a..6d4c6878b6 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -61,6 +61,12 @@ } """ + +PVS_STUDIO_FREE_LICENSE_HEADER = """ +// This is an open source non-commercial project. Dear PVS-Studio, please check it. +// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com +""" + EXPECTED_ERRORS = 4 EXPECTED_WARNINGS = 1 EXPECTED_STYLE = 1 @@ -87,19 +93,21 @@ def count_defects(output): return error, warning, style -def test_check_cli_output(clirunner, check_dir): +def test_check_cli_output(clirunner, validate_cliresult, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir)]) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors + warnings + style == EXPECTED_DEFECTS -def test_check_json_output(clirunner, check_dir): +def test_check_json_output(clirunner, validate_cliresult, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), "--json-output"] ) + validate_cliresult(result) + output = json.loads(result.stdout.strip()) assert isinstance(output, list) @@ -114,14 +122,24 @@ def test_check_tool_defines_passed(clirunner, check_dir): assert "__GNUC__" in output -def test_check_severity_threshold(clirunner, check_dir): +def test_check_language_standard_definition_passed(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\nbuild_flags = -std=c++17" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert "__cplusplus=201703L" in result.output + assert "--std=c++17" in result.output + + +def test_check_severity_threshold(clirunner, validate_cliresult, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), "--severity=high"] ) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == 0 assert style == 0 @@ -129,10 +147,9 @@ def test_check_severity_threshold(clirunner, check_dir): def test_check_includes_passed(clirunner, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) - output = result.output inc_count = 0 - for l in output.split("\n"): + for l in result.output.split("\n"): if l.startswith("Includes:"): inc_count = l.count("-I") @@ -140,18 +157,18 @@ def test_check_includes_passed(clirunner, check_dir): assert inc_count > 1 -def test_check_silent_mode(clirunner, check_dir): +def test_check_silent_mode(clirunner, validate_cliresult, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--silent"]) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == 0 assert style == 0 -def test_check_custom_pattern_absolute_path(clirunner, tmpdir_factory): +def test_check_custom_pattern_absolute_path(clirunner, validate_cliresult, tmpdir_factory): project_dir = tmpdir_factory.mktemp("project") project_dir.join("platformio.ini").write(DEFAULT_CONFIG) @@ -161,16 +178,16 @@ def test_check_custom_pattern_absolute_path(clirunner, tmpdir_factory): result = clirunner.invoke( cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] ) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == EXPECTED_WARNINGS assert style == EXPECTED_STYLE -def test_check_custom_pattern_relative_path(clirunner, tmpdir_factory): +def test_check_custom_pattern_relative_path(clirunner, validate_cliresult, tmpdir_factory): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) @@ -180,10 +197,10 @@ def test_check_custom_pattern_relative_path(clirunner, tmpdir_factory): result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--pattern=app", "--pattern=prj"] ) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors + warnings + style == EXPECTED_DEFECTS * 2 @@ -214,7 +231,7 @@ def test_check_bad_flag_passed(clirunner, check_dir): assert style == 0 -def test_check_success_if_no_errors(clirunner, tmpdir): +def test_check_success_if_no_errors(clirunner, validate_cliresult, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src").join("main.c").write( """ @@ -232,26 +249,28 @@ def test_check_success_if_no_errors(clirunner, tmpdir): ) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert "[PASSED]" in result.output - assert result.exit_code == 0 assert errors == 0 assert warnings == 1 assert style == 1 -def test_check_individual_flags_passed(clirunner, tmpdir): +def test_check_individual_flags_passed(clirunner, validate_cliresult, tmpdir): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy, pvs-studio" config += """\ncheck_flags = cppcheck: --std=c++11 clangtidy: --fix-errors pvs-studio: --analysis-mode=4 """ + tmpdir.join("platformio.ini").write(config) - tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + tmpdir.mkdir("src").join("main.cpp").write(PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + validate_cliresult(result) clang_flags_found = cppcheck_flags_found = pvs_flags_found = False for l in result.output.split("\n"): @@ -269,7 +288,7 @@ def test_check_individual_flags_passed(clirunner, tmpdir): assert pvs_flags_found -def test_check_cppcheck_misra_addon(clirunner, check_dir): +def test_check_cppcheck_misra_addon(clirunner, validate_cliresult, check_dir): check_dir.join("misra.json").write( """ { @@ -309,12 +328,12 @@ def test_check_cppcheck_misra_addon(clirunner, check_dir): cmd_check, ["--project-dir", str(check_dir), "--flags=--addon=misra.json"] ) - assert result.exit_code == 0 + validate_cliresult(result) assert "R21.3 Found MISRA defect" in result.output assert not isfile(join(str(check_dir), "src", "main.cpp.dump")) -def test_check_fails_on_defects_only_with_flag(clirunner, tmpdir): +def test_check_fails_on_defects_only_with_flag(clirunner, validate_cliresult, tmpdir): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) @@ -325,11 +344,13 @@ def test_check_fails_on_defects_only_with_flag(clirunner, tmpdir): cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] ) - assert default_result.exit_code == 0 + validate_cliresult(default_result) assert result_with_flag.exit_code != 0 -def test_check_fails_on_defects_only_on_specified_level(clirunner, tmpdir): +def test_check_fails_on_defects_only_on_specified_level( + clirunner, validate_cliresult, tmpdir +): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write( @@ -350,12 +371,12 @@ def test_check_fails_on_defects_only_on_specified_level(clirunner, tmpdir): high_result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] ) + validate_cliresult(high_result) low_result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=low"] ) - assert high_result.exit_code == 0 assert low_result.exit_code != 0 @@ -367,15 +388,9 @@ def test_check_pvs_studio_free_license(clirunner, tmpdir): framework = arduino check_tool = pvs-studio """ - code = ( - """// This is an open source non-commercial project. Dear PVS-Studio, please check it. -// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com -""" - + TEST_CODE - ) tmpdir.join("platformio.ini").write(config) - tmpdir.mkdir("src").join("main.c").write(code) + tmpdir.mkdir("src").join("main.c").write(PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE) result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high", "-v"] @@ -399,8 +414,7 @@ def test_check_embedded_platform_all_tools(clirunner, validate_cliresult, tmpdir """ # tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write( - """// This is an open source non-commercial project. Dear PVS-Studio, please check it. -// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com + PVS_STUDIO_FREE_LICENSE_HEADER + """ #include void unused_function(int val){ @@ -425,13 +439,13 @@ def test_check_embedded_platform_all_tools(clirunner, validate_cliresult, tmpdir result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) defects = sum(count_defects(result.output)) - assert result.exit_code == 0 and defects > 0, "Failed %s with %s" % ( + assert defects > 0, "Failed %s with %s" % ( framework, tool, ) -def test_check_skip_includes_from_packages(clirunner, tmpdir): +def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir): config = """ [env:test] platform = nordicnrf52 @@ -445,13 +459,42 @@ def test_check_skip_includes_from_packages(clirunner, tmpdir): result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--skip-packages", "-v"] ) - - output = result.output + validate_cliresult(result) project_path = fs.to_unix_path(str(tmpdir)) - for l in output.split("\n"): + for l in result.output.split("\n"): if not l.startswith("Includes:"): continue for inc in l.split(" "): if inc.startswith("-I") and project_path not in inc: pytest.fail("Detected an include path from packages: " + inc) + + +def test_check_multiline_error(clirunner, tmpdir_factory): + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write(DEFAULT_CONFIG) + + project_dir.mkdir("include").join("main.h").write( + """ +#error This is a multiline error message \\ +that should be correctly reported \\ +in both default and verbose modes. +""" + ) + + project_dir.mkdir("src").join("main.c").write( + """ +#include +#include "main.h" + +int main() {} +""" + ) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir)]) + errors, _, _ = count_defects(result.output) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir), "-v"]) + verbose_errors, _, _ = count_defects(result.output) + + assert verbose_errors == errors == 1 From f77978a295b8208d36a5162fd40063fc6b53dedd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 22:01:08 +0300 Subject: [PATCH 199/223] Apply formatting --- tests/commands/test_check.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index 6d4c6878b6..fa33af6874 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -168,7 +168,9 @@ def test_check_silent_mode(clirunner, validate_cliresult, check_dir): assert style == 0 -def test_check_custom_pattern_absolute_path(clirunner, validate_cliresult, tmpdir_factory): +def test_check_custom_pattern_absolute_path( + clirunner, validate_cliresult, tmpdir_factory +): project_dir = tmpdir_factory.mktemp("project") project_dir.join("platformio.ini").write(DEFAULT_CONFIG) @@ -187,7 +189,9 @@ def test_check_custom_pattern_absolute_path(clirunner, validate_cliresult, tmpdi assert style == EXPECTED_STYLE -def test_check_custom_pattern_relative_path(clirunner, validate_cliresult, tmpdir_factory): +def test_check_custom_pattern_relative_path( + clirunner, validate_cliresult, tmpdir_factory +): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) @@ -268,7 +272,9 @@ def test_check_individual_flags_passed(clirunner, validate_cliresult, tmpdir): """ tmpdir.join("platformio.ini").write(config) - tmpdir.mkdir("src").join("main.cpp").write(PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE) + tmpdir.mkdir("src").join("main.cpp").write( + PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE + ) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) validate_cliresult(result) @@ -414,7 +420,8 @@ def test_check_embedded_platform_all_tools(clirunner, validate_cliresult, tmpdir """ # tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write( - PVS_STUDIO_FREE_LICENSE_HEADER + """ + PVS_STUDIO_FREE_LICENSE_HEADER + + """ #include void unused_function(int val){ @@ -439,10 +446,7 @@ def test_check_embedded_platform_all_tools(clirunner, validate_cliresult, tmpdir result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) defects = sum(count_defects(result.output)) - assert defects > 0, "Failed %s with %s" % ( - framework, - tool, - ) + assert defects > 0, "Failed %s with %s" % (framework, tool,) def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir): From 210cd760424bbb94bc888d5f65b0fa8f90962c39 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 22:01:22 +0300 Subject: [PATCH 200/223] Rename "idedata" sub-command to "data" --- docs | 2 +- platformio/commands/project.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs b/docs index cc09d98135..ccce6f04e2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit cc09d981358a438ad474485a89ee5ddbc1842c65 +Subproject commit ccce6f04e2e54dd3e08c83f23480a49823c8f5c7 diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 6900ce74ac..bd37175a49 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -69,7 +69,7 @@ def project_config(project_dir, json_output): return None -@cli.command("idedata", short_help="Dump data intended for IDE extensions/plugins") +@cli.command("data", short_help="Dump data intended for IDE extensions/plugins") @click.option( "-d", "--project-dir", @@ -78,7 +78,7 @@ def project_config(project_dir, json_output): ) @click.option("-e", "--environment", multiple=True) @click.option("--json-output", is_flag=True) -def project_idedata(project_dir, environment, json_output): +def project_data(project_dir, environment, json_output): if not is_platformio_project(project_dir): raise NotPlatformIOProjectError(project_dir) with fs.cd(project_dir): From 5086b96edea9ad67e909d39d28194ae14f083c94 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 25 Aug 2020 22:22:35 +0300 Subject: [PATCH 201/223] Bump version to 5.0.0b1 --- HISTORY.rst | 18 ++++++++++++------ docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 24963d3e4e..4f82d8d8d3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,15 +1,16 @@ Release Notes ============= -.. _release_notes_4: +.. _release_notes_5: -PlatformIO Core 4 +PlatformIO Core 5 ----------------- -4.4.0 (2020-??-??) +**A professional collaborative platform for embedded development** + +5.0.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ -**A professional collaborative platform for embedded development** * Integration with the new **PlatformIO Trusted Registry** @@ -42,6 +43,7 @@ PlatformIO Core 4 * **PlatformIO Build System** + - Upgraded to `SCons 4.0 - a next-generation software construction tool `__ - New `Custom Targets `__ * Pre/Post processing based on a dependent sources (other target, source file, etc.) @@ -50,7 +52,6 @@ PlatformIO Core 4 * Python callback as a target (use the power of Python interpreter and PlatformIO Build API) * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) - - Upgraded to `SCons 4.0 - a next-generation software construction tool `__ - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) @@ -60,7 +61,7 @@ PlatformIO Core 4 - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment - - Dump data intended for IDE extensions/plugins using a new `pio project idedata `__ command + - Dump build system data intended for IDE extensions/plugins using a new `pio project data `__ command - Do not generate ".travis.yml" for a new project, let the user have a choice * **Unit Testing** @@ -76,6 +77,11 @@ PlatformIO Core 4 - Do not escape compiler arguments in VSCode template on Windows +.. _release_notes_4: + +PlatformIO Core 4 +----------------- + 4.3.4 (2020-05-23) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index ccce6f04e2..c536bff835 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ccce6f04e2e54dd3e08c83f23480a49823c8f5c7 +Subproject commit c536bff8352fc0a26366d1d9ca73b8e60af8d205 diff --git a/platformio/__init__.py b/platformio/__init__.py index 0d8d3f17fb..9e5c7d31c7 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (4, 4, "0b6") +VERSION = (5, 0, "0b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 0db39ccfbd853f4ba9863aeffbc2976ee8b72f87 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 06:40:22 +0300 Subject: [PATCH 202/223] Automatically accept PIO. Core 4.0 compatible dev-platforms --- platformio/platform/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index ffafd61c2d..d5bcbcc2c3 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -114,8 +114,8 @@ def ensure_engine_compatible(self): core_spec = semantic_version.SimpleSpec(self.engines["platformio"]) if self.CORE_SEMVER in core_spec: return True - # PIO Core 4 is compatible with dev-platforms for PIO Core 2.0 & 3.0 - if any(semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3)): + # PIO Core 4 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0 + if any(semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3, 4)): return True raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER), str(core_spec)) From 1560fb724c0c3471a994eab47c7e80c24f1e2f83 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 06:40:46 +0300 Subject: [PATCH 203/223] Bump version to 5.0.0b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 9e5c7d31c7..82133bd490 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "0b1") +VERSION = (5, 0, "0b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 3c91e3c1e13b9fa9bcfc22c416a37fef7e849d53 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 14:51:01 +0300 Subject: [PATCH 204/223] Move build dir to the disk root (should fix issue with long path for Zephyr RTOS on WIndows) --- .github/workflows/examples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index f1db5c388b..e3bb201f3a 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -49,6 +49,7 @@ jobs: if: startsWith(matrix.os, 'windows') env: PLATFORMIO_CORE_DIR: C:/pio + PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap" run: | tox -e testexamples From 8625fdc571748c7cc873c3c16a406eb73b80dced Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 14:51:53 +0300 Subject: [PATCH 205/223] Minor imperovements --- platformio/package/manager/_registry.py | 7 +- platformio/platform/base.py | 2 +- scripts/docspregen.py | 560 +++++++++++++++--------- 3 files changed, 346 insertions(+), 223 deletions(-) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index f8af2ece3a..6d243d7623 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -139,14 +139,13 @@ def search_registry_packages(self, spec, filters=None): def fetch_registry_package(self, spec): assert isinstance(spec, PackageSpec) result = None + regclient = self.get_registry_client_instance() if spec.owner and spec.name: - result = self.get_registry_client_instance().get_package( - self.pkg_type, spec.owner, spec.name - ) + result = regclient.get_package(self.pkg_type, spec.owner, spec.name) if not result and (spec.id or (spec.name and not spec.owner)): packages = self.search_registry_packages(spec) if packages: - result = self.get_registry_client_instance().get_package( + result = regclient.get_package( self.pkg_type, packages[0]["owner"]["username"], packages[0]["name"] ) if not result: diff --git a/platformio/platform/base.py b/platformio/platform/base.py index d5bcbcc2c3..b29a9d7b82 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -114,7 +114,7 @@ def ensure_engine_compatible(self): core_spec = semantic_version.SimpleSpec(self.engines["platformio"]) if self.CORE_SEMVER in core_spec: return True - # PIO Core 4 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0 + # PIO Core 5 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0 if any(semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3, 4)): return True raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER), str(core_spec)) diff --git a/scripts/docspregen.py b/scripts/docspregen.py index f2dc275761..3698aa0bc7 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -22,7 +22,8 @@ import click from platformio import fs, util -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory try: from urlparse import ParseResult, urlparse, urlunparse @@ -41,15 +42,16 @@ limitations under the License. """ -API_PACKAGES = util.get_api_result("/packages") -API_FRAMEWORKS = util.get_api_result("/frameworks") -BOARDS = PlatformManager().get_installed_boards() -PLATFORM_MANIFESTS = PlatformManager().get_installed() +REGCLIENT = regclient = PlatformPackageManager().get_registry_client_instance() +API_PACKAGES = regclient.fetch_json_data("get", "/v2/packages") +API_FRAMEWORKS = regclient.fetch_json_data("get", "/v2/frameworks") +BOARDS = PlatformPackageManager().get_installed_boards() +PLATFORM_MANIFESTS = PlatformPackageManager().legacy_get_installed() DOCS_ROOT_DIR = realpath(join(dirname(realpath(__file__)), "..", "docs")) def is_compat_platform_and_framework(platform, framework): - p = PlatformFactory.newPlatform(platform) + p = PlatformFactory.new(platform) return framework in (p.frameworks or {}).keys() @@ -60,8 +62,10 @@ def campaign_url(url, source="platformio.org", medium="docs"): query += "&" query += "utm_source=%s&utm_medium=%s" % (source, medium) return urlunparse( - ParseResult(data.scheme, data.netloc, data.path, data.params, query, - data.fragment)) + ParseResult( + data.scheme, data.netloc, data.path, data.params, query, data.fragment + ) + ) def generate_boards_table(boards, skip_columns=None): @@ -75,10 +79,12 @@ def generate_boards_table(boards, skip_columns=None): ("RAM", "{ram}"), ] lines = [] - lines.append(""" + lines.append( + """ .. list-table:: :header-rows: 1 -""") +""" + ) # add header for (name, template) in columns: @@ -87,23 +93,26 @@ def generate_boards_table(boards, skip_columns=None): prefix = " * - " if name == "Name" else " - " lines.append(prefix + name) - for data in sorted(boards, key=lambda item: item['name']): - has_onboard_debug = (data.get('debug') and any( - t.get("onboard") for (_, t) in data['debug']['tools'].items())) + for data in sorted(boards, key=lambda item: item["name"]): + has_onboard_debug = data.get("debug") and any( + t.get("onboard") for (_, t) in data["debug"]["tools"].items() + ) debug = "No" if has_onboard_debug: debug = "On-board" - elif data.get('debug'): + elif data.get("debug"): debug = "External" - variables = dict(id=data['id'], - name=data['name'], - platform=data['platform'], - debug=debug, - mcu=data['mcu'].upper(), - f_cpu=int(data['fcpu'] / 1000000.0), - ram=fs.format_filesize(data['ram']), - rom=fs.format_filesize(data['rom'])) + variables = dict( + id=data["id"], + name=data["name"], + platform=data["platform"], + debug=debug, + mcu=data["mcu"].upper(), + f_cpu=int(data["fcpu"] / 1000000.0), + ram=fs.humanize_file_size(data["ram"]), + rom=fs.humanize_file_size(data["rom"]), + ) for (name, template) in columns: if skip_columns and name in skip_columns: @@ -121,25 +130,30 @@ def generate_frameworks_contents(frameworks): if not frameworks: return [] lines = [] - lines.append(""" + lines.append( + """ Frameworks ---------- .. list-table:: :header-rows: 1 * - Name - - Description""") + - Description""" + ) known = set() for framework in API_FRAMEWORKS: - known.add(framework['name']) - if framework['name'] not in frameworks: + known.add(framework["name"]) + if framework["name"] not in frameworks: continue - lines.append(""" + lines.append( + """ * - :ref:`framework_{name}` - - {description}""".format(**framework)) + - {description}""".format( + **framework + ) + ) if set(frameworks) - known: - click.secho("Unknown frameworks %s " % ( - set(frameworks) - known), fg="red") + click.secho("Unknown frameworks %s " % (set(frameworks) - known), fg="red") return lines @@ -147,20 +161,26 @@ def generate_platforms_contents(platforms): if not platforms: return [] lines = [] - lines.append(""" + lines.append( + """ Platforms --------- .. list-table:: :header-rows: 1 * - Name - - Description""") + - Description""" + ) for name in sorted(platforms): - p = PlatformFactory.newPlatform(name) - lines.append(""" + p = PlatformFactory.new(name) + lines.append( + """ * - :ref:`platform_{name}` - - {description}""".format(name=p.name, description=p.description)) + - {description}""".format( + name=p.name, description=p.description + ) + ) return lines @@ -170,16 +190,17 @@ def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): skip_board_columns.append("Debug") lines = [] onboard_debug = [ - b for b in boards if b.get('debug') and any( - t.get("onboard") for (_, t) in b['debug']['tools'].items()) - ] - external_debug = [ - b for b in boards if b.get('debug') and b not in onboard_debug + b + for b in boards + if b.get("debug") + and any(t.get("onboard") for (_, t) in b["debug"]["tools"].items()) ] + external_debug = [b for b in boards if b.get("debug") and b not in onboard_debug] if not onboard_debug and not external_debug: return lines - lines.append(""" + lines.append( + """ Debugging --------- @@ -187,11 +208,13 @@ def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): .. contents:: :local: -""") +""" + ) if extra_rst: lines.append(".. include:: %s" % extra_rst) - lines.append(""" + lines.append( + """ Tools & Debug Probes ~~~~~~~~~~~~~~~~~~~~ @@ -203,31 +226,36 @@ def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): .. warning:: You will need to install debug tool drivers depending on your system. Please click on compatible debug tool below for the further instructions. -""") +""" + ) if onboard_debug: - lines.append(""" + lines.append( + """ On-Board Debug Tools ^^^^^^^^^^^^^^^^^^^^ Boards listed below have on-board debug probe and **ARE READY** for debugging! You do not need to use/buy external debug probe. -""") +""" + ) lines.extend( - generate_boards_table(onboard_debug, - skip_columns=skip_board_columns)) + generate_boards_table(onboard_debug, skip_columns=skip_board_columns) + ) if external_debug: - lines.append(""" + lines.append( + """ External Debug Tools ^^^^^^^^^^^^^^^^^^^^ Boards listed below are compatible with :ref:`piodebug` but **DEPEND ON** external debug probe. They **ARE NOT READY** for debugging. Please click on board name for the further details. -""") +""" + ) lines.extend( - generate_boards_table(external_debug, - skip_columns=skip_board_columns)) + generate_boards_table(external_debug, skip_columns=skip_board_columns) + ) return lines @@ -235,41 +263,56 @@ def generate_packages(platform, packagenames, is_embedded): if not packagenames: return lines = [] - lines.append(""" + lines.append( + """ Packages -------- -""") - lines.append(""".. list-table:: +""" + ) + lines.append( + """.. list-table:: :header-rows: 1 * - Name - - Description""") + - Description""" + ) for name in sorted(packagenames): if name not in API_PACKAGES: click.secho("Unknown package `%s`" % name, fg="red") - lines.append(""" + lines.append( + """ * - {name} - - """.format(name=name)) + """.format( + name=name + ) + ) else: - lines.append(""" + lines.append( + """ * - `{name} <{url}>`__ - - {description}""".format(name=name, - url=campaign_url(API_PACKAGES[name]['url']), - description=API_PACKAGES[name]['description'])) + - {description}""".format( + name=name, + url=campaign_url(API_PACKAGES[name]["url"]), + description=API_PACKAGES[name]["description"], + ) + ) if is_embedded: - lines.append(""" + lines.append( + """ .. warning:: **Linux Users**: * Install "udev" rules :ref:`faq_udev_rules` * Raspberry Pi users, please read this article `Enable serial port on Raspberry Pi `__. -""") +""" + ) if platform == "teensy": - lines.append(""" + lines.append( + """ **Windows Users:** Teensy programming uses only Windows built-in HID @@ -278,14 +321,17 @@ def generate_packages(platform, packagenames, is_embedded): `_ is needed to access the COM port your program uses. No special driver installation is necessary on Windows 10. -""") +""" + ) else: - lines.append(""" + lines.append( + """ **Windows Users:** Please check that you have a correctly installed USB driver from board manufacturer -""") +""" + ) return "\n".join(lines) @@ -293,14 +339,12 @@ def generate_packages(platform, packagenames, is_embedded): def generate_platform(name, rst_dir): print("Processing platform: %s" % name) - compatible_boards = [ - board for board in BOARDS if name == board['platform'] - ] + compatible_boards = [board for board in BOARDS if name == board["platform"]] lines = [] lines.append(RST_COPYRIGHT) - p = PlatformFactory.newPlatform(name) + p = PlatformFactory.new(name) assert p.repository_url.endswith(".git") github_url = p.repository_url[:-4] @@ -314,14 +358,18 @@ def generate_platform(name, rst_dir): lines.append(" :ref:`projectconf_env_platform` = ``%s``" % p.name) lines.append("") lines.append(p.description) - lines.append(""" -For more detailed information please visit `vendor site <%s>`_.""" % - campaign_url(p.homepage)) - lines.append(""" + lines.append( + """ +For more detailed information please visit `vendor site <%s>`_.""" + % campaign_url(p.homepage) + ) + lines.append( + """ .. contents:: Contents :local: :depth: 1 -""") +""" + ) # # Extra @@ -332,12 +380,15 @@ def generate_platform(name, rst_dir): # # Examples # - lines.append(""" + lines.append( + """ Examples -------- Examples are listed from `%s development platform repository <%s>`_: -""" % (p.title, campaign_url("%s/tree/master/examples" % github_url))) +""" + % (p.title, campaign_url("%s/tree/master/examples" % github_url)) + ) examples_dir = join(p.get_dir(), "examples") if isdir(examples_dir): for eitem in os.listdir(examples_dir): @@ -355,14 +406,17 @@ def generate_platform(name, rst_dir): generate_debug_contents( compatible_boards, skip_board_columns=["Platform"], - extra_rst="%s_debug.rst" % - name if isfile(join(rst_dir, "%s_debug.rst" % - name)) else None)) + extra_rst="%s_debug.rst" % name + if isfile(join(rst_dir, "%s_debug.rst" % name)) + else None, + ) + ) # # Development version of dev/platform # - lines.append(""" + lines.append( + """ Stable and upstream versions ---------------------------- @@ -393,13 +447,15 @@ def generate_platform(name, rst_dir): [env:upstream_develop] platform = {github_url}.git board = ... -""".format(name=p.name, title=p.title, github_url=github_url)) +""".format( + name=p.name, title=p.title, github_url=github_url + ) + ) # # Packages # - _packages_content = generate_packages(name, p.packages.keys(), - p.is_embedded()) + _packages_content = generate_packages(name, p.packages.keys(), p.is_embedded()) if _packages_content: lines.append(_packages_content) @@ -408,8 +464,8 @@ def generate_platform(name, rst_dir): # compatible_frameworks = [] for framework in API_FRAMEWORKS: - if is_compat_platform_and_framework(name, framework['name']): - compatible_frameworks.append(framework['name']) + if is_compat_platform_and_framework(name, framework["name"]): + compatible_frameworks.append(framework["name"]) lines.extend(generate_frameworks_contents(compatible_frameworks)) # @@ -418,11 +474,12 @@ def generate_platform(name, rst_dir): if compatible_boards: vendors = {} for board in compatible_boards: - if board['vendor'] not in vendors: - vendors[board['vendor']] = [] - vendors[board['vendor']].append(board) + if board["vendor"] not in vendors: + vendors[board["vendor"]] = [] + vendors[board["vendor"]].append(board) - lines.append(""" + lines.append( + """ Boards ------ @@ -431,20 +488,20 @@ def generate_platform(name, rst_dir): `PlatformIO Boards Explorer `_ * For more detailed ``board`` information please scroll the tables below by horizontally. -""") +""" + ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) - lines.extend( - generate_boards_table(boards, skip_columns=["Platform"])) + lines.extend(generate_boards_table(boards, skip_columns=["Platform"])) return "\n".join(lines) def update_platform_docs(): for manifest in PLATFORM_MANIFESTS: - name = manifest['name'] + name = manifest["name"] platforms_dir = join(DOCS_ROOT_DIR, "platforms") rst_path = join(platforms_dir, "%s.rst" % name) with open(rst_path, "w") as f: @@ -455,12 +512,11 @@ def generate_framework(type_, data, rst_dir=None): print("Processing framework: %s" % type_) compatible_platforms = [ - m for m in PLATFORM_MANIFESTS - if is_compat_platform_and_framework(m['name'], type_) - ] - compatible_boards = [ - board for board in BOARDS if type_ in board['frameworks'] + m + for m in PLATFORM_MANIFESTS + if is_compat_platform_and_framework(m["name"], type_) ] + compatible_boards = [board for board in BOARDS if type_ in board["frameworks"]] lines = [] @@ -468,21 +524,26 @@ def generate_framework(type_, data, rst_dir=None): lines.append(".. _framework_%s:" % type_) lines.append("") - lines.append(data['title']) - lines.append("=" * len(data['title'])) + lines.append(data["title"]) + lines.append("=" * len(data["title"])) lines.append("") lines.append(":Configuration:") lines.append(" :ref:`projectconf_env_framework` = ``%s``" % type_) lines.append("") - lines.append(data['description']) - lines.append(""" + lines.append(data["description"]) + lines.append( + """ For more detailed information please visit `vendor site <%s>`_. -""" % campaign_url(data['url'])) +""" + % campaign_url(data["url"]) + ) - lines.append(""" + lines.append( + """ .. contents:: Contents :local: - :depth: 1""") + :depth: 1""" + ) # Extra if isfile(join(rst_dir, "%s_extra.rst" % type_)): @@ -495,27 +556,37 @@ def generate_framework(type_, data, rst_dir=None): lines.extend( generate_debug_contents( compatible_boards, - extra_rst="%s_debug.rst" % - type_ if isfile(join(rst_dir, "%s_debug.rst" % - type_)) else None)) + extra_rst="%s_debug.rst" % type_ + if isfile(join(rst_dir, "%s_debug.rst" % type_)) + else None, + ) + ) if compatible_platforms: # examples - lines.append(""" + lines.append( + """ Examples -------- -""") +""" + ) for manifest in compatible_platforms: - p = PlatformFactory.newPlatform(manifest['name']) - lines.append("* `%s for %s <%s>`_" % - (data['title'], manifest['title'], - campaign_url("%s/tree/master/examples" % - p.repository_url[:-4]))) + p = PlatformFactory.new(manifest["name"]) + lines.append( + "* `%s for %s <%s>`_" + % ( + data["title"], + manifest["title"], + campaign_url("%s/tree/master/examples" % p.repository_url[:-4]), + ) + ) # Platforms lines.extend( generate_platforms_contents( - [manifest['name'] for manifest in compatible_platforms])) + [manifest["name"] for manifest in compatible_platforms] + ) + ) # # Boards @@ -523,10 +594,11 @@ def generate_framework(type_, data, rst_dir=None): if compatible_boards: vendors = {} for board in compatible_boards: - if board['vendor'] not in vendors: - vendors[board['vendor']] = [] - vendors[board['vendor']].append(board) - lines.append(""" + if board["vendor"] not in vendors: + vendors[board["vendor"]] = [] + vendors[board["vendor"]].append(board) + lines.append( + """ Boards ------ @@ -534,7 +606,8 @@ def generate_framework(type_, data, rst_dir=None): * You can list pre-configured boards by :ref:`cmd_boards` command or `PlatformIO Boards Explorer `_ * For more detailed ``board`` information please scroll the tables below by horizontally. -""") +""" + ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) @@ -544,7 +617,7 @@ def generate_framework(type_, data, rst_dir=None): def update_framework_docs(): for framework in API_FRAMEWORKS: - name = framework['name'] + name = framework["name"] frameworks_dir = join(DOCS_ROOT_DIR, "frameworks") rst_path = join(frameworks_dir, "%s.rst" % name) with open(rst_path, "w") as f: @@ -561,7 +634,8 @@ def update_boards(): lines.append("Boards") lines.append("======") - lines.append(""" + lines.append( + """ Rapid Embedded Development, Continuous and IDE integration in a few steps with PlatformIO thanks to built-in project generator for the most popular embedded boards and IDE. @@ -570,25 +644,28 @@ def update_boards(): * You can list pre-configured boards by :ref:`cmd_boards` command or `PlatformIO Boards Explorer `_ * For more detailed ``board`` information please scroll tables below by horizontal. -""") +""" + ) platforms = {} for data in BOARDS: - platform = data['platform'] + platform = data["platform"] if platform in platforms: platforms[platform].append(data) else: platforms[platform] = [data] for platform, boards in sorted(platforms.items()): - p = PlatformFactory.newPlatform(platform) + p = PlatformFactory.new(platform) lines.append(p.title) lines.append("-" * len(p.title)) - lines.append(""" + lines.append( + """ .. toctree:: :maxdepth: 1 - """) - for board in sorted(boards, key=lambda item: item['name']): + """ + ) + for board in sorted(boards, key=lambda item: item["name"]): lines.append(" %s/%s" % (platform, board["id"])) lines.append("") @@ -600,44 +677,48 @@ def update_boards(): for data in BOARDS: # if data['id'] != "m5stack-core-esp32": # continue - rst_path = join(DOCS_ROOT_DIR, "boards", data["platform"], - "%s.rst" % data["id"]) + rst_path = join( + DOCS_ROOT_DIR, "boards", data["platform"], "%s.rst" % data["id"] + ) if not isdir(dirname(rst_path)): os.makedirs(dirname(rst_path)) update_embedded_board(rst_path, data) def update_embedded_board(rst_path, board): - platform = PlatformFactory.newPlatform(board['platform']) - board_config = platform.board_config(board['id']) + platform = PlatformFactory.new(board["platform"]) + board_config = platform.board_config(board["id"]) board_manifest_url = platform.repository_url assert board_manifest_url if board_manifest_url.endswith(".git"): board_manifest_url = board_manifest_url[:-4] - board_manifest_url += "/blob/master/boards/%s.json" % board['id'] - - variables = dict(id=board['id'], - name=board['name'], - platform=board['platform'], - platform_description=platform.description, - url=campaign_url(board['url']), - mcu=board_config.get("build", {}).get("mcu", ""), - mcu_upper=board['mcu'].upper(), - f_cpu=board['fcpu'], - f_cpu_mhz=int(int(board['fcpu']) / 1000000), - ram=fs.format_filesize(board['ram']), - rom=fs.format_filesize(board['rom']), - vendor=board['vendor'], - board_manifest_url=board_manifest_url, - upload_protocol=board_config.get("upload.protocol", "")) + board_manifest_url += "/blob/master/boards/%s.json" % board["id"] + + variables = dict( + id=board["id"], + name=board["name"], + platform=board["platform"], + platform_description=platform.description, + url=campaign_url(board["url"]), + mcu=board_config.get("build", {}).get("mcu", ""), + mcu_upper=board["mcu"].upper(), + f_cpu=board["fcpu"], + f_cpu_mhz=int(int(board["fcpu"]) / 1000000), + ram=fs.humanize_file_size(board["ram"]), + rom=fs.humanize_file_size(board["rom"]), + vendor=board["vendor"], + board_manifest_url=board_manifest_url, + upload_protocol=board_config.get("upload.protocol", ""), + ) lines = [RST_COPYRIGHT] lines.append(".. _board_{platform}_{id}:".format(**variables)) lines.append("") - lines.append(board['name']) - lines.append("=" * len(board['name'])) - lines.append(""" + lines.append(board["name"]) + lines.append("=" * len(board["name"])) + lines.append( + """ .. contents:: Hardware @@ -657,12 +738,16 @@ def update_embedded_board(rst_path, board): - {ram} * - **Vendor** - `{vendor} <{url}>`__ -""".format(**variables)) +""".format( + **variables + ) + ) # # Configuration # - lines.append(""" + lines.append( + """ Configuration ------------- @@ -690,23 +775,33 @@ def update_embedded_board(rst_path, board): ; change MCU frequency board_build.f_cpu = {f_cpu}L -""".format(**variables)) +""".format( + **variables + ) + ) # # Uploading # upload_protocols = board_config.get("upload.protocols", []) if len(upload_protocols) > 1: - lines.append(""" + lines.append( + """ Uploading --------- %s supports the next uploading protocols: -""" % board['name']) +""" + % board["name"] + ) for protocol in sorted(upload_protocols): lines.append("* ``%s``" % protocol) - lines.append(""" -Default protocol is ``%s``""" % variables['upload_protocol']) - lines.append(""" + lines.append( + """ +Default protocol is ``%s``""" + % variables["upload_protocol"] + ) + lines.append( + """ You can change upload protocol using :ref:`projectconf_upload_protocol` option: .. code-block:: ini @@ -716,22 +811,29 @@ def update_embedded_board(rst_path, board): board = {id} upload_protocol = {upload_protocol} -""".format(**variables)) +""".format( + **variables + ) + ) # # Debugging # lines.append("Debugging") lines.append("---------") - if not board.get('debug'): + if not board.get("debug"): lines.append( ":ref:`piodebug` currently does not support {name} board.".format( - **variables)) + **variables + ) + ) else: default_debug_tool = board_config.get_debug_tool_name() has_onboard_debug = any( - t.get("onboard") for (_, t) in board['debug']['tools'].items()) - lines.append(""" + t.get("onboard") for (_, t) in board["debug"]["tools"].items() + ) + lines.append( + """ :ref:`piodebug` - "1-click" solution for debugging with a zero configuration. .. warning:: @@ -741,34 +843,43 @@ def update_embedded_board(rst_path, board): You can switch between debugging :ref:`debugging_tools` using :ref:`projectconf_debug_tool` option in :ref:`projectconf`. -""") +""" + ) if has_onboard_debug: lines.append( "{name} has on-board debug probe and **IS READY** for " - "debugging. You don't need to use/buy external debug probe.". - format(**variables)) + "debugging. You don't need to use/buy external debug probe.".format( + **variables + ) + ) else: lines.append( "{name} does not have on-board debug probe and **IS NOT " "READY** for debugging. You will need to use/buy one of " - "external probe listed below.".format(**variables)) - lines.append(""" + "external probe listed below.".format(**variables) + ) + lines.append( + """ .. list-table:: :header-rows: 1 * - Compatible Tools - On-board - - Default""") - for (tool_name, tool_data) in sorted(board['debug']['tools'].items()): - lines.append(""" * - :ref:`debugging_tool_{name}` + - Default""" + ) + for (tool_name, tool_data) in sorted(board["debug"]["tools"].items()): + lines.append( + """ * - :ref:`debugging_tool_{name}` - {onboard} - {default}""".format( - name=tool_name, - onboard="Yes" if tool_data.get("onboard") else "", - default="Yes" if tool_name == default_debug_tool else "")) + name=tool_name, + onboard="Yes" if tool_data.get("onboard") else "", + default="Yes" if tool_name == default_debug_tool else "", + ) + ) - if board['frameworks']: - lines.extend(generate_frameworks_contents(board['frameworks'])) + if board["frameworks"]: + lines.extend(generate_frameworks_contents(board["frameworks"])) with open(rst_path, "w") as f: f.write("\n".join(lines)) @@ -781,21 +892,21 @@ def update_debugging(): platforms = [] frameworks = [] for data in BOARDS: - if not data.get('debug'): + if not data.get("debug"): continue - for tool in data['debug']['tools']: + for tool in data["debug"]["tools"]: tool = str(tool) if tool not in tool_to_platforms: tool_to_platforms[tool] = [] - tool_to_platforms[tool].append(data['platform']) + tool_to_platforms[tool].append(data["platform"]) if tool not in tool_to_boards: tool_to_boards[tool] = [] - tool_to_boards[tool].append(data['id']) + tool_to_boards[tool].append(data["id"]) - platforms.append(data['platform']) - frameworks.extend(data['frameworks']) - vendor = data['vendor'] + platforms.append(data["platform"]) + frameworks.extend(data["frameworks"]) + vendor = data["vendor"] if vendor in vendors: vendors[vendor].append(data) else: @@ -809,26 +920,30 @@ def update_debugging(): lines.extend(generate_frameworks_contents(frameworks)) # Boards - lines.append(""" + lines.append( + """ Boards ------ .. note:: For more detailed ``board`` information please scroll tables below by horizontal. -""") +""" + ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) lines.extend(generate_boards_table(boards)) # save - with open(join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), - "r+") as fp: + with open( + join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), "r+" + ) as fp: content = fp.read() fp.seek(0) fp.truncate() - fp.write(content[:content.index(".. _debugging_platforms:")] + - "\n".join(lines)) + fp.write( + content[: content.index(".. _debugging_platforms:")] + "\n".join(lines) + ) # Debug tools for tool, platforms in tool_to_platforms.items(): @@ -847,24 +962,27 @@ def update_debugging(): tool_frameworks.append(framework) lines.extend(generate_frameworks_contents(tool_frameworks)) - lines.append(""" + lines.append( + """ Boards ------ .. note:: For more detailed ``board`` information please scroll tables below by horizontal. -""") +""" + ) lines.extend( generate_boards_table( - [b for b in BOARDS if b['id'] in tool_to_boards[tool]], - skip_columns=None)) + [b for b in BOARDS if b["id"] in tool_to_boards[tool]], + skip_columns=None, + ) + ) with open(tool_path, "r+") as fp: content = fp.read() fp.seek(0) fp.truncate() - fp.write(content[:content.index(".. begin_platforms")] + - "\n".join(lines)) + fp.write(content[: content.index(".. begin_platforms")] + "\n".join(lines)) def update_project_examples(): @@ -899,7 +1017,7 @@ def update_project_examples(): desktop = [] for manifest in PLATFORM_MANIFESTS: - p = PlatformFactory.newPlatform(manifest['name']) + p = PlatformFactory.new(manifest["name"]) github_url = p.repository_url[:-4] # Platform README @@ -922,19 +1040,21 @@ def update_project_examples(): name=p.name, title=p.title, description=p.description, - examples="\n".join(examples_md_lines))) + examples="\n".join(examples_md_lines), + ) + ) # Framework README for framework in API_FRAMEWORKS: - if not is_compat_platform_and_framework(p.name, framework['name']): + if not is_compat_platform_and_framework(p.name, framework["name"]): continue - if framework['name'] not in framework_examples_md_lines: - framework_examples_md_lines[framework['name']] = [] + if framework["name"] not in framework_examples_md_lines: + framework_examples_md_lines[framework["name"]] = [] lines = [] lines.append("- [%s](%s)" % (p.title, github_url)) lines.extend(" %s" % l for l in examples_md_lines) lines.append("") - framework_examples_md_lines[framework['name']].extend(lines) + framework_examples_md_lines[framework["name"]].extend(lines) # Root README line = "* [%s](%s)" % (p.title, "%s/tree/master/examples" % github_url) @@ -946,27 +1066,29 @@ def update_project_examples(): # Frameworks frameworks = [] for framework in API_FRAMEWORKS: - readme_dir = join(project_examples_dir, "frameworks", - framework['name']) + readme_dir = join(project_examples_dir, "frameworks", framework["name"]) if not isdir(readme_dir): os.makedirs(readme_dir) with open(join(readme_dir, "README.md"), "w") as fp: fp.write( framework_readme_tpl.format( - name=framework['name'], - title=framework['title'], - description=framework['description'], - examples="\n".join( - framework_examples_md_lines[framework['name']]))) + name=framework["name"], + title=framework["title"], + description=framework["description"], + examples="\n".join(framework_examples_md_lines[framework["name"]]), + ) + ) url = campaign_url( "https://docs.platformio.org/en/latest/frameworks/%s.html#examples" - % framework['name'], + % framework["name"], source="github", - medium="examples") - frameworks.append("* [%s](%s)" % (framework['title'], url)) + medium="examples", + ) + frameworks.append("* [%s](%s)" % (framework["title"], url)) with open(join(project_examples_dir, "README.md"), "w") as fp: - fp.write("""# PlatformIO Project Examples + fp.write( + """# PlatformIO Project Examples - [Development platforms](#development-platforms): - [Embedded](#embedded) @@ -986,7 +1108,9 @@ def update_project_examples(): ## Frameworks %s -""" % ("\n".join(embedded), "\n".join(desktop), "\n".join(frameworks))) +""" + % ("\n".join(embedded), "\n".join(desktop), "\n".join(frameworks)) + ) def main(): From d59416431dae2d4b22f0116c5057e5ec78b0e7d2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 15:40:03 +0300 Subject: [PATCH 206/223] Parse npm-like "repository" data from a package manifest // Resolve #3637 --- platformio/package/manifest/parser.py | 12 ++++++++++++ platformio/util.py | 4 ++-- tests/package/test_manifest.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 689de80b7e..8949f43e53 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -662,6 +662,7 @@ def parse(self, contents): data["keywords"] = self.str_to_list(data["keywords"], sep=",") data = self._parse_system(data) data = self._parse_homepage(data) + data = self._parse_repository(data) return data @staticmethod @@ -682,3 +683,14 @@ def _parse_homepage(data): data["homepage"] = data["url"] del data["url"] return data + + @staticmethod + def _parse_repository(data): + if isinstance(data.get("repository", {}), dict): + return data + data["repository"] = dict(type="git", url=str(data["repository"])) + if data["repository"]["url"].startswith(("github:", "gitlab:", "bitbucket:")): + data["repository"]["url"] = "https://{0}.com/{1}".format( + *(data["repository"]["url"].split(":", 1)) + ) + return data diff --git a/platformio/util.py b/platformio/util.py index aeeaf55bce..f950a36485 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -28,8 +28,8 @@ from platformio import __version__, exception, proc from platformio.compat import PY2, WINDOWS -from platformio.fs import cd # pylint: disable=unused-import -from platformio.fs import load_json # pylint: disable=unused-import +from platformio.fs import cd, load_json # pylint: disable=unused-import +from platformio.package.version import pepver_to_semver # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 9dd5b8789a..426cbdf17a 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -672,6 +672,20 @@ def test_package_json_schema(): ) assert mp.as_dict()["system"] == ["darwin_x86_64"] + # shortcut repository syntax (npm-style) + contents = """ +{ + "name": "tool-github", + "version": "1.2.0", + "repository": "github:user/repo" +} +""" + raw_data = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.PACKAGE_JSON + ).as_dict() + data = ManifestSchema().load_manifest(raw_data) + assert data["repository"]["url"] == "https://github.com/user/repo.git" + def test_parser_from_dir(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") From 4a7f578649781252f1466f0725bf31b239a9fdaa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Aug 2020 15:40:24 +0300 Subject: [PATCH 207/223] Sync docs and history --- HISTORY.rst | 2010 +-------------------------------------------------- docs | 2 +- examples | 2 +- 3 files changed, 7 insertions(+), 2007 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4f82d8d8d3..53a933d377 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -82,2024 +82,24 @@ PlatformIO Core 5 PlatformIO Core 4 ----------------- -4.3.4 (2020-05-23) -~~~~~~~~~~~~~~~~~~ - -* Added `PlatformIO CLI Shell Completion `__ for Fish, Zsh, Bash, and PowerShell (`issue #3435 `_) -* Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 `_) -* Fixed an issue on Windows when installing a library dependency from Git repository (`issue #2844 `_, `issue #3328 `_) -* Fixed an issue with PIO Check when a defect with multiline error message is not reported in verbose mode (`issue #3631 `_) - -4.3.3 (2020-04-28) -~~~~~~~~~~~~~~~~~~ - -* Fixed "UnicodeDecodeError: 'utf-8' codec can't decode byte" when non-Latin chars are used in project path (`issue #3481 `_) - -4.3.2 (2020-04-28) -~~~~~~~~~~~~~~~~~~ - -* New `Account Management System `__ (preview) -* Open source `PIO Remote `__ client -* Improved `PIO Check `__ with more accurate project processing -* Echo what is typed when ``send_on_enter`` `device monitor filter `__ is used (`issue #3452 `_) -* Fixed PIO Unit Testing for Zephyr RTOS -* Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) -* Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 `_) -* Fixed an incorrect node path used for pattern matching when processing middleware nodes -* Fixed an issue with missing ``lib_extra_dirs`` option in SRC_LIST for CLion (`issue #3460 `_) - -4.3.1 (2020-03-20) -~~~~~~~~~~~~~~~~~~ - -* Fixed a SyntaxError "'return' with argument inside generator" for PIO Unified Debugger when Python 2.7 is used -* Fixed an issue when ``lib_archive = no`` was not honored in `"platformio.ini" `__ -* Fixed a TypeError "super(type, obj): obj must be an instance or subtype of type" when device monitor is used with a custom dev-platform filter (`issue #3431 `_) - -4.3.0 (2020-03-19) -~~~~~~~~~~~~~~~~~~ - -* Initial support for an official `PlatformIO for CLion IDE `__ plugin: - - - Smart C and C++ editor - - Code refactoring - - On-the-fly code analysis - - "New PlatformIO Project" wizard - - Building, Uploading, Testing - - Integrated debugger (inline variable view, conditional breakpoints, expressions, watchpoints, peripheral registers, multi-thread support, etc.) - -* `Device Monitor 2.0 `__ - - - Added **PlatformIO Device Monitor Filter API** (dev-platforms can extend base device monitor with a custom functionality, such as exception decoding) (`pull #3383 `_) - - Configure project device monitor with `monitor_filters `__ option - - `Capture device monitor output to a file `__ with ``log2file`` filter (`issue #670 `_) - - Show a timestamp for each new line with ``time`` filter (`issue #981 `_) - - Send a text to device on ENTER with ``send_on_enter`` filter (`issue #926 `_) - - Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter - -* New standalone (1-script) `PlatformIO Core Installer `_ -* Initial support for `Renode `__ simulation framework (`issue #3401 `_) -* Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) -* Improved support for Arduino "library.properties" ``depends`` field -* Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server -* Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) -* Fixed an issue when Python 2 does not keep encoding when converting ".ino" (`issue #3393 `_) -* Fixed an issue when ``"libArchive": false`` in "library.json" does not work (`issue #3403 `_) -* Fixed an issue when not all commands in `compilation database "compile_commands.json" `__ use absolute paths (`pull #3415 `_) -* Fixed an issue when unknown transport is used for `PIO Unit Testing `__ engine (`issue #3422 `_) - -4.2.1 (2020-02-17) -~~~~~~~~~~~~~~~~~~ - -* Improved VSCode template with special ``forceInclude`` field for direct includes via ``-include`` flag (`issue #3379 `_) -* Improved support of PIO Home on card-sized PC (Raspberry Pi, etc.) (`issue #3313 `_) -* Froze "marshmallow" dependency to 2.X for Python 2 (`issue #3380 `_) -* Fixed "TypeError: unsupported operand type(s)" when system environment variable is used by project configuration parser (`issue #3377 `_) -* Fixed an issue when Library Dependency Finder (LDF) ignores custom "libLDFMode" and "libCompatMode" options in `library.json `__ -* Fixed an issue when generating of compilation database "compile_commands.json" does not work with Python 2.7 (`issue #3378 `_) - - -4.2.0 (2020-02-12) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home 3.1 `__: - - - Project Manager - - Project Configuration UI for `"platformio.ini" `__ - -* `PIO Check `__ – automated code analysis without hassle: - - - Added support for `PVS-Studio `__ static code analyzer - -* Initial support for `Project Manager `_ CLI: - - - Show computed project configuration with a new `platformio project config `_ command or dump to JSON with ``platformio project config --json-output`` (`issue #3335 `_) - - Moved ``platformio init`` command to `platformio project init `_ - -* Generate `compilation database "compile_commands.json" `__ (`issue #2990 `_) -* Control debug flags and optimization level with a new `debug_build_flags `__ option -* Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install `__ command (`issue #3345 `_) -* Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) -* Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) -* Added support for Arduino's library.properties ``depends`` field (`issue #2781 `_) -* Autodetect monitor port for boards with specified HWIDs (`issue #3349 `_) -* Updated SCons tool to 3.1.2 -* Updated Unity tool to 2.5.0 -* Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) -* Warn about broken library manifest when scanning dependencies (`issue #3268 `_) -* Do not overwrite custom items in VSCode's "extensions.json" (`issue #3374 `_) -* Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) -* Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) -* Fixed default PIO Unified Debugger configuration for `J-Link probe `__ -* Fixed an issue with LDF when header files not found if "libdeps_dir" is within a subdirectory of "lib_extra_dirs" (`issue #3311 `_) -* Fixed an issue "Import of non-existent variable 'projenv''" when development platform does not call "env.BuildProgram()" (`issue #3315 `_) -* Fixed an issue when invalid CLI command does not return non-zero exit code -* Fixed an issue when Project Inspector crashes when flash use > 100% (`issue #3368 `_) -* Fixed a "UnicodeDecodeError" when listing built-in libraries on macOS with Python 2.7 (`issue #3370 `_) -* Fixed an issue with improperly handled compiler flags with space symbols in VSCode template (`issue #3364 `_) -* Fixed an issue when no error is raised if referred parameter (interpolation) is missing in a project configuration file (`issue #3279 `_) - - -4.1.0 (2019-11-07) -~~~~~~~~~~~~~~~~~~ - -* `PIO Check `__ – automated code analysis without hassle: - - - Potential NULL pointer dereferences - - Possible indexing beyond array bounds - - Suspicious assignments - - Reads of potentially uninitialized objects - - Unused variables or functions - - Out of scope memory usage. - -* `PlatformIO Home 3.0 `__: - - - Project Inspection - - Static Code Analysis - - Firmware File Explorer - - Firmware Memory Inspection - - Firmware Sections & Symbols Viewer. - -* Added support for `Build Middlewares `__: configure custom build flags per specific file, skip any build nodes from a framework, replace build file with another on-the-fly, etc. -* Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) -* Generate ``.ccls`` LSP file for `Emacs `__ cross references, hierarchies, completion and semantic highlighting -* Added ``--no-ansi`` flag for `PIO Core `__ to disable ANSI control characters -* Added ``--shutdown-timeout`` option to `PIO Home Server `__ -* Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) -* Fixed default PIO Unified Debugger configuration for `J-Link probe `__ -* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 `_) -* Fixed an issue when installing a package using custom Git tag and submodules were not updated correctly (`issue #3060 `_) -* Fixed an issue with linking process when ``$LDSCRIPT`` contains a space in path -* Fixed security issue when extracting items from TAR archive (`issue #2995 `_) -* Fixed an issue with project generator when ``src_build_flags`` were not respected (`issue #3137 `_) -* Fixed an issue when booleans in "platformio.ini" are not parsed properly (`issue #3022 `_) -* Fixed an issue with invalid encoding when generating project for Visual Studio (`issue #3183 `_) -* Fixed an issue when Project Config Parser does not remove in-line comments when Python 3 is used (`issue #3213 `_) -* Fixed an issue with a GCC Linter for PlatformIO IDE for Atom (`issue #3218 `_) - -4.0.3 (2019-08-30) -~~~~~~~~~~~~~~~~~~ - -* Added support for multi-environment PlatformIO project for `CLion IDE `__ (`issue #2824 `_) -* Generate ``.ccls`` LSP file for `Vim `__ cross references, hierarchies, completion and semantic highlighting (`issue #2952 `_) -* Added support for `PLATFORMIO_DISABLE_COLOR `__ system environment variable which disables color ANSI-codes in a terminal output (`issue #2956 `_) -* Updated SCons tool to 3.1.1 -* Remove ProjectConfig cache when "platformio.ini" was modified outside -* Fixed an issue with PIO Unified Debugger on Windows OS when debug server is piped -* Fixed an issue when `--upload-port `__ CLI flag does not override declared `upload_port `__ option in `"platformio.ini" (Project Configuration File) `__ - -4.0.2 (2019-08-23) -~~~~~~~~~~~~~~~~~~ - -* Fixed an issue with a broken `LDF `__ when checking for framework compatibility (`issue #2940 `_) - -4.0.1 (2019-08-22) -~~~~~~~~~~~~~~~~~~ - -* Print `debug tool `__ name for the active debugging session -* Do not shutdown PIO Home Server for "upgrade" operations (`issue #2784 `_) -* Improved computing of project check sum (structure, configuration) and avoid unnecessary rebuilding -* Improved printing of tabulated results -* Automatically normalize file system paths to UNIX-style for Project Generator (`issue #2857 `_) -* Ability to set "databaseFilename" for VSCode and C/C++ extension (`issue #2825 `_) -* Renamed "enable_ssl" setting to `strict_ssl `__ -* Fixed an issue with incorrect escaping of Windows slashes when using `PIO Unified Debugger `__ and "piped" openOCD -* Fixed an issue when "debug", "home", "run", and "test" commands were not shown in "platformio --help" CLI -* Fixed an issue with PIO Home's "No JSON object could be decoded" (`issue #2823 `_) -* Fixed an issue when `library.json `__ had priority over project configuration for `LDF `__ (`issue #2867 `_) - -4.0.0 (2019-07-10) -~~~~~~~~~~~~~~~~~~ - -`Migration Guide from 3.0 to 4.0 `__. - -* `PlatformIO Plus Goes Open Source `__ - - - Built-in `PIO Unified Debugger `__ - - Built-in `PIO Unit Testing `__ - -* **Project Configuration** - - - New project configuration parser with a strict options typing (`API `__) - - Unified workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 `_) - - Share common (global) options between project environments using `[env] `__ section (`issue #1643 `_) - - Include external configuration files with `extra_configs `__ option (`issue #1590 `_) - - Custom project ``***_dir`` options declared in `platformio `__ section have higher priority than `Environment variables `__ - - Added support for Unix shell-style wildcards for `monitor_port `__ option (`issue #2541 `_) - - Added new `monitor_flags `__ option which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) - - Added support for `PLATFORMIO_DEFAULT_ENVS `__ system environment variable (`issue #1967 `_) - - Added support for `shared_dir `__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a `PIO Remote `__ machine - -* **Library Management** - - - Switched to workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` - - Save libraries passed to `platformio lib install `__ command into the project dependency list (`lib_deps `__) with a new ``--save`` flag (`issue #1028 `_) - - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - - Use isolated library dependency storage per project build environment (`issue #1696 `_) - - Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry (`issue #1654 `_) - - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 `_) - - Fixed an issue when `lib_compat_mode = strict `__ does not ignore libraries incompatible with a project framework - -* **Build System** - - - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` - - Switch between `Build Configurations `__ (``release`` and ``debug``) with a new project configuration option `build_type `__ - - Custom `platform_packages `__ per a build environment with an option to override default (`issue #1367 `_) - - Print platform package details, such as version, VSC source and commit (`issue #2155 `_) - - Control a number of parallel build jobs with a new `-j, --jobs `__ option - - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) - - Configure a shared folder for the derived files (objects, firmwares, ELFs) from a build system using `build_cache_dir `__ option (`issue #2674 `_) - - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) - -* **Infrastructure** - - - Python 3 support (`issue #895 `_) - - Significantly speedup back-end for PIO Home. It works super fast now! - - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) - - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) - - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #143 `_, `issue #1342 `_, `issue #1959 `_, `issue #2100 `_) - -* **Integration** - - - Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file - - Fixed an issue with hardcoded C standard version when generating project for CLion IDE (`issue #2527 `_) - - Fixed an issue with Project Generator when an include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) - - Fixed an issue when generating invalid "Eclipse CDT Cross GCC Built-in Compiler Settings" if a custom `PLATFORMIO_CORE_DIR `__ is used (`issue #806 `_) - -* **Miscellaneous** - - - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead - - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) - - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 `_) +See `PlatformIO Core 4.0 history `__. PlatformIO Core 3 ----------------- -3.6.7 (2019-04-23) -~~~~~~~~~~~~~~~~~~ - -* `PIO Unified Debugger `__: improved debugging in ``debug_load_mode = modified`` and fixed an issue with useless project rebuilding -* Project Generator: fixed a VSCode C/C++'s "Cannot find" warning when CPPPATH folder does not exist -* Fixed an "IndexError: list index out of range" for Arduino sketch preprocessor - (`issue #2268 `_) -* Fixed an issue when invalid "env_default" in `"platformio.ini" (Project Configuration File) `__ results into unhandled errors - (`issue #2265 `_) - -3.6.6 (2019-03-29) -~~~~~~~~~~~~~~~~~~ - -* Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode - (`issue #2243 `_) -* Fixed an issue when PlatformIO Build System does not pick up "mbed_lib.json" files from libraries - (`issue #2164 `_) -* Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor) -* Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option -* Fixed an issue with incorrect order of project "include" and "src" paths in ``CPPPATH`` - (`issue #1914 `_) - -3.6.5 (2019-03-07) -~~~~~~~~~~~~~~~~~~ - -* Project Generator: added new targets for CLion IDE "BUILD_VERBOSE" and "MONITOR" (serial port monitor) - (`issue #359 `_) -* Fixed an issue with slow updating of PlatformIO Core packages on Windows -* Fixed an issue when `platformio ci `__ recompiles project if ``--keep-build-dir`` option is passed - (`issue #2109 `_) -* Fixed an issue when ``$PROJECT_HASH`` template was not expanded for the other directory ``***_dir`` options in `"platformio.ini" (Project Configuration File) `__ - (`issue #2170 `_) - -3.6.4 (2019-01-23) -~~~~~~~~~~~~~~~~~~ - -* Improved Project Generator for IDEs: - - - Use full path to PlatformIO CLI when generating a project - (`issue #1674 `_) - - CLion: Improved project portability using "${CMAKE_CURRENT_LIST_DIR}" instead of full path - - Eclipse: Provide language standard to a project C/C++ indexer - (`issue #1010 `_) - -* Fixed an issue with incorrect detecting of compatibility (LDF) between generic library and Arduino or ARM mbed frameworks -* Fixed "Runtime Error: Dictionary size changed during iteration" - (`issue #2003 `_) -* Fixed an error "Could not extract item..." when extracting TAR archive with symbolic items on Windows platform - (`issue #2015 `_) - -3.6.3 (2018-12-12) -~~~~~~~~~~~~~~~~~~ - -* Ignore ``*.asm`` and ``*.ASM`` files when building Arduino-based library (compatibility with Arduino builder) -* Fixed spurious project's "Problems" for `PlatformIO IDE for VSCode `__ when ARM mbed framework is used -* Fixed an issue with a broken headers list when generating ".clang_complete" for `Emacs `__ - (`issue #1960 `_) - -3.6.2 (2018-11-29) -~~~~~~~~~~~~~~~~~~ - -* Improved IntelliSense for `PlatformIO IDE for VSCode `__ via passing extra compiler information for C/C++ Code Parser (resolves issues with spurious project's "Problems") -* Fixed an issue with VSCode IntelliSense warning about the missed headers located in `include `__ folder -* Fixed incorrect wording when initializing/updating project -* Fixed an issue with incorrect order for library dependencies ``CPPPATH`` - (`issue #1914 `_) -* Fixed an issue when Library Dependency Finder (LDF) does not handle project `src_filter `__ - (`issue #1905 `_) -* Fixed an issue when Library Dependency Finder (LDF) finds spurious dependencies in ``chain+`` and ``deep+`` modes - (`issue #1930 `_) - -3.6.1 (2018-10-29) -~~~~~~~~~~~~~~~~~~ - -* Generate an `include `__ and `test `__ directories with a README file when initializing a new project -* Support in-line comments for multi-line value (``lib_deps``, ``build_flags``, etc) in `"platformio.ini" (Project Configuration File) `__ -* Added ``$PROJECT_HASH`` template variable for `build_dir `__. One of the use cases is setting a global storage for project artifacts using `PLATFORMIO_BUILD_DIR `__ system environment variable. For example, ``/tmp/pio-build/$PROJECT_HASH`` (Unix) or ``$[sysenv.TEMP}/pio-build/$PROJECT_HASH`` (Windows) -* Improved a loading speed of PIO Home "Recent News" -* Improved `PIO Unified Debugger `__ for "mbed" framework and fixed issue with missed local variables -* Introduced `"Release" and "Debug" Build Configurations `__ -* Build project in "Debug Mode" including debugging information with a new ``debug`` target using `platformio run `__ command or `targets `__ option in ``platformio.ini``. The last option allows avoiding project rebuilding between "Run/Debug" modes. - (`issue #1833 `_) -* Process ``build_unflags`` for the cloned environment when building a static library -* Report on outdated `99-platformio-udev.rules `__ - (`issue #1823 `_) -* Show a valid error when the Internet is off-line while initializing a new project - (`issue #1784 `_) -* Do not re-create ".gitignore" and ".travis.yml" files if they were removed from a project -* Fixed an issue when dynamic build flags were not handled correctly - (`issue #1799 `_) -* Fixed an issue when ``pio run -t monitor`` always uses the first ``monitor_port`` even with multiple environments - (`issue #1841 `_) -* Fixed an issue with broken includes when generating ``.clang_complete`` and space is used in a path - (`issue #1873 `_) -* Fixed an issue with incorrect handling of a custom package name when using `platformio lib install `__ or `platformio platform install `__ commands - -3.6.0 (2018-08-06) -~~~~~~~~~~~~~~~~~~ - -* `Program Memory Usage `_ - - - Print human-readable memory usage information after a build and before uploading - - Print detailed memory usage information with "sections" and "addresses" - in `verbose mode `__ - - Check maximum allowed "program" and "data" sizes before uploading/programming - (`issue #1412 `_) - -* `PIO Unit Testing `__: - - - Documented `Project Shared Code `__ - - Force building of project source code using `test_build_project_src `__ option - - Fixed missed ``UNIT_TEST`` macro for unit test components/libraries - -* Check package structure after unpacking and raise error when antivirus tool - blocks PlatformIO package manager - (`issue #1462 `_) -* Lock interprocess requests to PlatformIO Package Manager for - install/uninstall operations - (`issue #1594 `_) -* Fixed an issue with `PIO Remote `__ - when upload process depends on the source code of a project framework -* Fixed an issue when ``srcFilter`` field in `library.json `__ - breaks a library build - (`issue #1735 `_) - -3.5.4 (2018-07-03) -~~~~~~~~~~~~~~~~~~ - -* Improved removing of default build flags using `build_unflags `__ option - (`issue #1712 `_) -* Export ``LIBS``, ``LIBPATH``, and ``LINKFLAGS`` data from project dependent - libraries to the global build environment -* Don't export ``CPPPATH`` data of project dependent libraries to framework's - build environment - (`issue #1665 `_) -* Handle "architectures" data from "library.properties" manifest in - `lib_compat_mode = strict `__ -* Added workaround for Python SemVer package's `issue #61 `_ with caret range and pre-releases -* Replaced conflicted "env" pattern by "sysenv" for `"platformio.ini" Dynamic Variables" `__ - (`issue #1705 `_) -* Removed "date&time" when processing project with `platformio run `__ command - (`issue #1343 `_) -* Fixed issue with invalid LD script if path contains space -* Fixed preprocessor for Arduino sketch when function returns certain type - (`issue #1683 `_) -* Fixed issue when `platformio lib uninstall `__ - removes initial source code - (`issue #1023 `_) - -3.5.3 (2018-06-01) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home `__ - - interact with PlatformIO ecosystem using modern and cross-platform GUI: - - - "Recent News" block on "Welcome" page - - Direct import of development platform's example - -* Simplify configuration for `PIO Unit Testing `__: separate main program from a test build process, drop - requirement for ``#ifdef UNIT_TEST`` guard -* Override any option from board manifest in `"platformio.ini" (Project Configuration File) `__ - (`issue #1612 `_) -* Configure a custom path to SVD file using `debug_svd_path `__ - option -* Custom project `description `_ - which will be used by `PlatformIO Home `_ -* Updated Unity tool to 2.4.3 -* Improved support for Black Magic Probe in "uploader" mode -* Renamed "monitor_baud" option to "monitor_speed" -* Fixed issue when a custom `lib_dir `__ - was not handled correctly - (`issue #1473 `_) -* Fixed issue with useless project rebuilding for case insensitive file - systems (Windows) -* Fixed issue with ``build_unflags`` option when a macro contains value - (e.g., ``-DNAME=VALUE``) -* Fixed issue which did not allow to override runtime build environment using - extra POST script -* Fixed "RuntimeError: maximum recursion depth exceeded" for library manager - (`issue #1528 `_) - -3.5.2 (2018-03-13) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home `__ - - interact with PlatformIO ecosystem using modern and cross-platform GUI: - - - Multiple themes (Dark & Light) - - Ability to specify a name for new project - -* Control `PIO Unified Debugger `__ - and its firmware loading mode using - `debug_load_mode `__ option -* Added aliases (off, light, strict) for - `LDF Compatibility Mode `__ -* Search for a library using PIO Library Registry ID ``id:X`` (e.g. ``pio lib search id:13``) -* Show device system information (MCU, Frequency, RAM, Flash, Debugging tools) - in a build log -* Show all available upload protocols before firmware uploading in a build log -* Handle "os.mbed.com" URL as a Mercurial (hg) repository -* Improved support for old mbed libraries without manifest -* Fixed project generator for Qt Creator IDE - (`issue #1303 `_, - `issue #1323 `_) -* Mark project source and library directories for CLion IDE - (`issue #1359 `_, - `issue #1345 `_, - `issue #897 `_) -* Fixed issue with duplicated "include" records when generating data for IDE - (`issue #1301 `_) - -3.5.1 (2018-01-18) -~~~~~~~~~~~~~~~~~~ - -* New ``test_speed`` option to control a communication baudrate/speed between - `PIO Unit Testing `__ - engine and a target device - (`issue #1273 `_) -* Show full library version in "Library Dependency Graph" including VCS - information - (`issue #1274 `_) -* Configure a custom firmware/program name in build directory (`example `__) -* Renamed ``envs_dir`` option to ``build_dir`` - in `"platformio.ini" (Project Configuration File) `__ -* Refactored code without "arrow" dependency (resolve issue with "ImportError: - No module named backports.functools_lru_cache") -* Improved support of PIO Unified Debugger for Eclipse Oxygen -* Improved a work in off-line mode -* Fixed project generator for CLion and Qt Creator IDE - (`issue #1299 `_) -* Fixed PIO Unified Debugger for mbed framework -* Fixed library updates when a version is declared in VCS format (not SemVer) - -3.5.0 (2017-12-28) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home `__ - - interact with PlatformIO ecosystem using modern and cross-platform GUI: - - - Library Manager: - - * Search for new libraries in PlatformIO Registry - * "1-click" library installation, per-project libraries, extra storages - * List installed libraries in multiple storages - * List built-in libraries (by frameworks) - * Updates for installed libraries - * Multiple examples, trending libraries, and more. - - - PlatformIO Projects - - PIO Account - - Development platforms, frameworks and board explorer - - Device Manager: serial, logical, and multicast DNS services - -* Integration with `Jenkins CI `_ -* New `include `__ - folder for project's header files - (`issue #1107 `_) -* Depend on development platform using VCS URL (Git, Mercurial and Subversion) - instead of a name in `"platformio.ini" (Project Configuration File) `__. - Drop support for ``*_stage`` dev/platform names (use VCS URL instead). -* Reinstall/redownload package with a new ``-f, --force`` option for - `platformio lib install `__ - and `platformio platform install `__ - commands - (`issue #778 `_) -* Handle missed dependencies and provide a solution based on PlatformIO Library - Registry - (`issue #781 `_) -* New setting `projects_dir `__ - that allows to override a default PIO Home Projects location - (`issue #1161 `_) - -* `Library Dependency Finder (LDF) `__: - - - Search for dependencies used in `PIO Unit Testing `__ - (`issue #953 `_) - - Parse library source file in pair with a header when they have the same name - (`issue #1175 `_) - - Handle library dependencies defined as VCS or SemVer in - `"platformio.ini" (Project Configuration File) `__ - (`issue #1155 `_) - - Added option to configure library `Compatible Mode `__ - using `library.json `__ - -* New options for `platformio device list `__ - command: - - - ``--serial`` list available serial ports (default) - - ``--logical`` list logical devices - - ``--mdns`` discover multicast DNS services - (`issue #463 `_) - -* Fixed platforms, packages, and libraries updating behind proxy - (`issue #1061 `_) -* Fixed missing toolchain include paths for project generator - (`issue #1154 `_) -* Fixed "Super-Quick (Mac / Linux)" installation in "get-platformio.py" script - (`issue #1017 `_) -* Fixed "get-platformio.py" script which hangs on Windows 10 - (`issue #1118 `_) -* Other bug fixes and performance improvements - -3.4.1 (2017-08-02) -~~~~~~~~~~~~~~~~~~ - -* Pre/Post extra scripting for advanced control of PIO Build System - (`issue #891 `_) -* New `lib_archive `_ - option to control library archiving and linking behavior - (`issue #993 `_) -* Add "inc" folder automatically to CPPPATH when "src" is available (works for project and library) - (`issue #1003 `_) -* Use a root of library when filtering source code using - `library.json `__ - and ``srcFilter`` field -* Added ``monitor_*`` options to white-list for `"platformio.ini" (Project Configuration File) `__ - (`issue #982 `_) -* Do not ask for board ID when initialize project for desktop platform -* Handle broken PIO Core state and create new one -* Fixed an issue with a custom transport for `PIO Unit Testing `__ - when multiple tests are present -* Fixed an issue when can not upload firmware to SAM-BA based board (Due) - -3.4.0 (2017-06-26) -~~~~~~~~~~~~~~~~~~ - -* `PIO Unified Debugger `__ - - - "1-click" solution, zero configuration - - Support for 100+ embedded boards - - Multiple architectures and development platforms - - Windows, MacOS, Linux (+ARMv6-8) - - Built-in into `PlatformIO IDE for Atom `__ and `PlatformIO IDE for VScode `__ - - Integration with `Eclipse `__ and `Sublime Text `__ - -* Filter `PIO Unit Testing `__ - tests using a new ``test_filter`` option in `"platformio.ini" (Project Configuration File) `__ - or `platformio test --filter `__ command - (`issue #934 `_) -* Custom ``test_transport`` for `PIO Unit Testing `__ Engine -* Configure Serial Port Monitor in `"platformio.ini" (Project Configuration File) `__ - (`issue #787 `_) -* New `monitor `__ - target which allows to launch Serial Monitor automatically after successful - "build" or "upload" operations - (`issue #788 `_) -* Project generator for `VIM `__ -* Multi-line support for the different options in `"platformio.ini" (Project Configuration File) `__, - such as: ``build_flags``, ``build_unflags``, etc. - (`issue #889 `_) -* Handle dynamic ``SRC_FILTER`` environment variable from - `library.json extra script `__ -* Notify about multiple installations of PIO Core - (`issue #961 `_) -* Improved auto-detecting of mbed-enabled media disks -* Automatically update Git-submodules for development platforms and libraries - that were installed from repository -* Add support for ``.*cc`` extension - (`issue #939 `_) -* Handle ``env_default`` in `"platformio.ini" (Project Configuration File) `__ - when re-initializing a project - (`issue #950 `_) -* Use root directory for PIO Home when path contains non-ascii characters - (`issue #951 `_, - `issue #952 `_) -* Don't warn about known ``boards_dir`` option - (`pull #949 `_) -* Escape non-valid file name characters when installing a new package (library) - (`issue #985 `_) -* Fixed infinite dependency installing when repository consists of multiple - libraries - (`issue #935 `_) -* Fixed linter error "unity.h does not exist" for Unit Testing - (`issue #947 `_) -* Fixed issue when `Library Dependency Finder (LDF) `__ - does not handle custom ``src_dir`` - (`issue #942 `_) -* Fixed cloning a package (library) from a private Git repository with - custom user name and SSH port - (`issue #925 `_) - -3.3.1 (2017-05-27) -~~~~~~~~~~~~~~~~~~ - -* Hotfix for recently updated Python Requests package (2.16.0) - -3.3.0 (2017-03-27) -~~~~~~~~~~~~~~~~~~ - -* PlatformIO Library Registry statistics with new - `pio lib stats `__ command - - - Recently updated and added libraries - - Recent and popular keywords - - Featured libraries (today, week, month) - -* List built-in libraries based on development platforms with a new - `pio lib builtin `__ command -* Show detailed info about a library using `pio lib show `__ - command - (`issue #430 `_) -* List supported frameworks, SDKs with a new - `pio platform frameworks `__ command -* Visual Studio Code extension for PlatformIO - (`issue #619 `_) -* Added new options ``--no-reset``, ``--monitor-rts`` and ``--monitor-dtr`` - to `pio test `__ - command (allows to avoid automatic board's auto-reset when gathering test results) -* Added support for templated methods in ``*.ino to *.cpp`` converter - (`pull #858 `_) -* Package version as "Repository URL" in manifest of development version - (``"version": "https://github.com/user/repo.git"``) -* Produce less noisy output when ``-s/--silent`` options are used for - `platformio init `__ - and `platformio run `__ - commands - (`issue #850 `_) -* Use C++11 by default for CLion IDE based projects - (`pull #873 `_) -* Escape project path when Glob matching is used -* Do not overwrite project configuration variables when system environment - variables are set -* Handle dependencies when installing non-registry package/library (VCS, archive, local folder) - (`issue #913 `_) -* Fixed package installing with VCS branch for Python 2.7.3 - (`issue #885 `_) - -3.2.1 (2016-12-07) -~~~~~~~~~~~~~~~~~~ - -* Changed default `LDF Mode `__ - from ``chain+`` to ``chain`` - -3.2.0 (2016-12-07) -~~~~~~~~~~~~~~~~~~ - -* `PIO Remote™ `__. - **Your devices are always with you!** - - + Over-The-Air (OTA) Device Manager - + OTA Serial Port Monitor - + OTA Firmware Updates - + Continuous Deployment - + Continuous Delivery - -* Integration with `Cloud IDEs `__ - - + Cloud9 - + Codeanywhere - + Eclipse Che - -* `PIO Account `__ - and `PLATFORMIO_AUTH_TOKEN `__ - environment variable for CI systems - (`issue #808 `_, - `issue #467 `_) -* Inject system environment variables to configuration settings in - `"platformio.ini" (Project Configuration File) `__ - (`issue #792 `_) -* Custom boards per project with ``boards_dir`` option in - `"platformio.ini" (Project Configuration File) `__ - (`issue #515 `_) -* Unix shell-style wildcards for `upload_port `_ - (`issue #839 `_) -* Refactored `Library Dependency Finder (LDF) `__ - C/C++ Preprocessor for conditional syntax (``#ifdef``, ``#if``, ``#else``, - ``#elif``, ``#define``, etc.) - (`issue #837 `_) -* Added new `LDF Modes `__: - ``chain+`` and ``deep+`` and set ``chain+`` as default -* Added global ``lib_extra_dirs`` option to ``[platformio]`` section for - `"platformio.ini" (Project Configuration File) `__ - (`issue #842 `_) -* Enabled caching by default for API requests and Library Manager (see `enable_cache `__ setting) -* Native integration with VIM/Neovim using `neomake-platformio `__ plugin -* Changed a default exit combination for Device Monitor from ``Ctrl+]`` to ``Ctrl+C`` -* Improved detecting of ARM mbed media disk for uploading -* Improved Project Generator for CLion IDE when source folder contains nested items -* Improved handling of library dependencies specified in ``library.json`` manifest - (`issue #814 `_) -* Improved `Library Dependency Finder (LDF) `__ - for circular dependencies -* Show vendor version of a package for `platformio platform show `__ command - (`issue #838 `_) -* Fixed unable to include SSH user in ``lib_deps`` repository url - (`issue #830 `_) -* Fixed merging of ".gitignore" files when re-initialize project - (`issue #848 `_) -* Fixed issue with ``PATH`` auto-configuring for upload tools -* Fixed ``99-platformio-udev.rules`` checker for Linux OS - -3.1.0 (2016-09-19) -~~~~~~~~~~~~~~~~~~ - -* New! Dynamic variables/templates for `"platformio.ini" (Project Configuration File) `__ - (`issue #705 `_) -* Summary about processed environments - (`issue #777 `_) -* Implemented LocalCache system for API and improved a work in off-line mode -* Improved Project Generator when custom ``--project-option`` is passed to - `platformio init `__ - command -* Deprecated ``lib_force`` option, please use `lib_deps `__ instead -* Return valid exit code from ``plaformio test`` command -* Fixed Project Generator for CLion IDE using Windows OS - (`issue #785 `_) -* Fixed SSL Server-Name-Indication for Python < 2.7.9 - (`issue #774 `_) - -3.0.1 (2016-09-08) -~~~~~~~~~~~~~~~~~~ - -* Disabled temporary SSL for PlatformIO services - (`issue #772 `_) - -3.0.0 (2016-09-07) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Plus `__ - - + Local and Embedded `Unit Testing `__ - (`issue #408 `_, - `issue #519 `_) - -* Decentralized Development Platforms - - + Development platform manifest "platform.json" and - `open source development platforms `__ - + `Semantic Versioning `__ for platform commands, - development platforms and dependent packages - + Custom package repositories - + External embedded board configuration files, isolated build scripts - (`issue #479 `_) - + Embedded Board compatibility with more than one development platform - (`issue #456 `_) - -* Library Manager 3.0 - - + Project dependencies per build environment using `lib_deps `__ option - (`issue #413 `_) - + `Semantic Versioning `__ for library commands and - dependencies - (`issue #410 `_) - + Multiple library storages: Project's Local, PlatformIO's Global or Custom - (`issue #475 `_) - + Install library by name - (`issue #414 `_) - + Depend on a library using VCS URL (GitHub, Git, ARM mbed code registry, Hg, SVN) - (`issue #498 `_) - + Strict search for library dependencies - (`issue #588 `_) - + Allowed ``library.json`` to specify sources other than PlatformIO's Repository - (`issue #461 `_) - + Search libraries by headers/includes with ``platformio lib search --header`` option - -* New Intelligent Library Build System - - + `Library Dependency Finder `__ - that interprets C/C++ Preprocessor conditional macros with deep search behavior - + Check library compatibility with project environment before building - (`issue #415 `_) - + Control Library Dependency Finder for compatibility using - `lib_compat_mode `__ - option - + Custom library storages/directories with - `lib_extra_dirs `__ option - (`issue #537 `_) - + Handle extra build flags, source filters and build script from - `library.json `__ - (`issue #289 `_) - + Allowed to disable library archiving (``*.ar``) - (`issue #719 `_) - + Show detailed build information about dependent libraries - (`issue #617 `_) - + Support for the 3rd party manifests (Arduino IDE "library.properties" - and ARM mbed "module.json") - -* Removed ``enable_prompts`` setting. Now, all PlatformIO CLI is non-blocking! -* Switched to SSL PlatformIO API -* Renamed ``platformio serialports`` command to ``platformio device`` -* Build System: Attach custom Before/Pre and After/Post actions for targets - (`issue #542 `_) -* Allowed passing custom project configuration options to ``platformio ci`` - and ``platformio init`` commands using ``-O, --project-option``. -* Print human-readable information when processing environments without - ``-v, --verbose`` option - (`issue #721 `_) -* Improved INO to CPP converter - (`issue #659 `_, - `issue #765 `_) -* Added ``license`` field to `library.json `__ - (`issue #522 `_) -* Warn about unknown options in project configuration file ``platformio.ini`` - (`issue #740 `_) -* Fixed wrong line number for INO file when ``#warning`` directive is used - (`issue #742 `_) -* Stopped supporting Python 2.6 +See `PlatformIO Core 3.0 history `__. PlatformIO Core 2 ----------------- -2.11.2 (2016-08-02) -~~~~~~~~~~~~~~~~~~~ - -* Improved support for `Microchip PIC32 `__ development platform and ChipKIT boards - (`issue #438 `_) -* Added support for Pinoccio Scout board - (`issue #52 `_) -* Added support for `Teensy USB Features `__ - (HID, SERIAL_HID, DISK, DISK_SDFLASH, MIDI, etc.) - (`issue #722 `_) -* Switched to built-in GCC LwIP library for Espressif development platform -* Added support for local ``--echo`` for Serial Port Monitor - (`issue #733 `_) -* Updated ``udev`` rules for the new STM32F407DISCOVERY boards - (`issue #731 `_) -* Implemented firmware merging with base firmware for Nordic nRF51 development platform - (`issue #500 `_, - `issue #533 `_) -* Fixed Project Generator for ESP8266 and ARM mbed based projects - (resolves incorrect linter errors) -* Fixed broken LD Script for Element14 chipKIT Pi board - (`issue #725 `_) -* Fixed firmware uploading to Atmel SAMD21-XPRO board using ARM mbed framework - (`issue #732 `_) - -2.11.1 (2016-07-12) -~~~~~~~~~~~~~~~~~~~ - -* Added support for Arduino M0, M0 Pro and Tian boards - (`issue #472 `_) -* Added support for Microchip chipKIT Lenny board -* Updated Microchip PIC32 Arduino framework to v1.2.1 -* Documented `uploading of EEPROM data `__ - (from EEMEM directive) -* Added ``Rebuild C/C++ Project Index`` target to CLion and Eclipse IDEs -* Improved project generator for `CLion IDE `__ -* Added ``udev`` rules for OpenOCD CMSIS-DAP adapters - (`issue #718 `_) -* Auto-remove project cache when PlatformIO is upgraded -* Keep user changes for ``.gitignore`` file when re-generate/update project data -* Ignore ``[platformio]`` section from custom project configuration file when - `platformio ci --project-conf `__ - command is used -* Fixed missed ``--boot`` flag for the firmware uploader for ATSAM3X8E - Cortex-M3 MCU based boards (Arduino Due, etc) - (`issue #710 `_) -* Fixed missing trailing ``\`` for the source files list when generate project - for `Qt Creator IDE `__ - (`issue #711 `_) -* Split source files to ``HEADERS`` and ``SOURCES`` when generate project - for `Qt Creator IDE `__ - (`issue #713 `_) - -2.11.0 (2016-06-28) -~~~~~~~~~~~~~~~~~~~ - -* New ESP8266-based boards: Generic ESP8285 Module, Phoenix 1.0 & 2.0, WifInfo -* Added support for Arduino M0 Pro board - (`issue #472 `_) -* Added support for Arduino MKR1000 board - (`issue #620 `_) -* Added support for Adafruit Feather M0, SparkFun SAMD21 and SparkFun SAMD21 - Mini Breakout boards - (`issue #520 `_) -* Updated Arduino ESP8266 core for Espressif platform to 2.3.0 -* Better removing unnecessary flags using ``build_unflags`` option - (`issue #698 `_) -* Fixed issue with ``platformio init --ide`` command for Python 2.6 - -2.10.3 (2016-06-15) -~~~~~~~~~~~~~~~~~~~ - -* Fixed issue with ``platformio init --ide`` command - -2.10.2 (2016-06-15) -~~~~~~~~~~~~~~~~~~~ - -* Added support for ST Nucleo L031K6 board to ARM mbed framework -* Process ``build_unflags`` option for ARM mbed framework -* Updated Intel ARC32 Arduino framework to v1.0.6 - (`issue #695 `_) -* Improved a check of program size before uploading to the board -* Fixed issue with ARM mbed framework ``-u _printf_float`` and - ``-u _scanf_float`` when parsing ``$LINKFLAGS`` -* Fixed issue with ARM mbed framework and extra includes for the custom boards, - such as Seeeduino Arch Pro - -2.10.1 (2016-06-13) -~~~~~~~~~~~~~~~~~~~ - -* Re-submit a package to PyPI - -2.10.0 (2016-06-13) -~~~~~~~~~~~~~~~~~~~ - -* Added support for `emonPi `__, - the OpenEnergyMonitor system - (`issue #687 `_) -* Added support for `SPL `__ - framework for STM32F0 boards - (`issue #683 `_) -* Added support for `Arduboy DevKit `__, the game system - the size of a credit card -* Updated ARM mbed framework package to v121 -* Check program size before uploading to the board - (`issue #689 `_) -* Improved firmware uploading to Arduino Leonardo based boards - (`issue #691 `_) -* Fixed issue with ``-L relative/path`` when parsing ``build_flags`` - (`issue #688 `_) - -2.9.4 (2016-06-04) -~~~~~~~~~~~~~~~~~~ - -* Show ``udev`` warning only for the Linux OS while uploading firmware - -2.9.3 (2016-06-03) -~~~~~~~~~~~~~~~~~~ - -* Added support for `Arduboy `__, the game system - the size of a credit card -* Updated `99-platformio-udev.rules `__ for Linux OS -* Refactored firmware uploading to the embedded boards with SAM-BA bootloader - -2.9.2 (2016-06-02) -~~~~~~~~~~~~~~~~~~ - -* Simplified `Continuous Integration with AppVeyor `__ - (`issue #671 `_) -* Automatically add source directory to ``CPPPATH`` of Build System -* Added support for Silicon Labs SLSTK3401A (Pearl Gecko) and - MultiTech mDot F411 ARM mbed based boards -* Added support for MightyCore ATmega8535 board - (`issue #585 `_) -* Added ``stlink`` as the default uploader for STM32 Discovery boards - (`issue #665 `_) -* Use HTTP mirror for Package Manager in a case with SSL errors - (`issue #645 `_) -* Improved firmware uploading to Arduino Leonardo/Due based boards -* Fixed bug with ``env_default`` when ``pio run -e`` is used -* Fixed issue with ``src_filter`` option for Windows OS - (`issue #652 `_) -* Fixed configuration data for TI LaunchPads based on msp430fr4133 and - msp430fr6989 MCUs - (`issue #676 `_) -* Fixed issue with ARM mbed framework and multiple definition errors - on FRDM-KL46Z board - (`issue #641 `_) -* Fixed issue with ARM mbed framework when abstract class breaks compile - for LPC1768 - (`issue #666 `_) - -2.9.1 (2016-04-30) -~~~~~~~~~~~~~~~~~~ - -* Handle prototype pointers while converting ``*.ino`` to ``.cpp`` - (`issue #639 `_) - -2.9.0 (2016-04-28) -~~~~~~~~~~~~~~~~~~ - -* Project generator for `CodeBlocks IDE `__ - (`issue #600 `_) -* New `Lattice iCE40 FPGA `__ - development platform with support for Lattice iCEstick FPGA Evaluation - Kit and BQ IceZUM Alhambra FPGA - (`issue #480 `_) -* New `Intel ARC 32-bit `_ - development platform with support for Arduino/Genuino 101 board - (`issue #535 `_) -* New `Microchip PIC32 `__ - development platform with support for 20+ different PIC32 based boards - (`issue #438 `_) -* New RTOS and build Framework named `Simba `__ - (`issue #412 `_) -* New boards for `ARM mbed `__ - framework: ST Nucleo F410RB, ST Nucleo L073RZ and BBC micro:bit -* Added support for Arduino.Org boards: Arduino Leonardo ETH, Arduino Yun Mini, - Arduino Industrial 101 and Linino One - (`issue #472 `_) -* Added support for Generic ATTiny boards: ATTiny13, ATTiny24, ATTiny25, - ATTiny45 and ATTiny85 - (`issue #636 `_) -* Added support for MightyCore boards: ATmega1284, ATmega644, ATmega324, - ATmega164, ATmega32, ATmega16 and ATmega8535 - (`issue #585 `_) -* Added support for `TI MSP430 `__ - boards: TI LaunchPad w/ msp430fr4133 and TI LaunchPad w/ msp430fr6989 -* Updated Arduino core for Espressif platform to 2.2.0 - (`issue #627 `_) -* Updated native SDK for ESP8266 to 1.5 - (`issue #366 `_) -* PlatformIO Library Registry in JSON format! Implemented - ``--json-output`` and ``--page`` options for - `platformio lib search `__ - command - (`issue #604 `_) -* Allowed to specify default environments `env_default `__ - which should be processed by default with ``platformio run`` command - (`issue #576 `_) -* Allowed to unflag(remove) base/initial flags using - `build_unflags `__ - option - (`issue #559 `_) -* Allowed multiple VID/PID pairs when detecting serial ports - (`issue #632 `_) -* Automatically add ``-DUSB_MANUFACTURER`` with vendor's name - (`issue #631 `_) -* Automatically reboot Teensy board after upload when Teensy Loader GUI is used - (`issue #609 `_) -* Refactored source code converter from ``*.ino`` to ``*.cpp`` - (`issue #610 `_) -* Forced ``-std=gnu++11`` for Atmel SAM development platform - (`issue #601 `_) -* Don't check OS type for ARM mbed-enabled boards and ST STM32 development - platform before uploading to disk - (`issue #596 `_) -* Fixed broken compilation for Atmel SAMD based boards except Arduino Due - (`issue #598 `_) -* Fixed firmware uploading using serial port with spaces in the path -* Fixed cache system when project's root directory is used as ``src_dir`` - (`issue #635 `_) - -2.8.6 (2016-03-22) -~~~~~~~~~~~~~~~~~~ - -* Launched `PlatformIO Community Forums `_ - (`issue #530 `_) -* Added support for ARM mbed-enabled board Seed Arch Max (STM32F407VET6) - (`issue #572 `_) -* Improved DNS lookup for PlatformIO API -* Updated Arduino Wiring-based framework to the latest version for - Atmel AVR/SAM development platforms -* Updated "Teensy Loader CLI" and fixed uploading of large .hex files - (`issue #568 `_) -* Updated the support for Sanguino Boards - (`issue #586 `_) -* Better handling of used boards when re-initialize/update project -* Improved support for non-Unicode user profiles for Windows OS -* Disabled progress bar for download operations when prompts are disabled -* Fixed multiple definition errors for ST STM32 development platform and - ARM mbed framework - (`issue #571 `_) -* Fixed invalid board parameters (reset method and baudrate) for a few - ESP8266 based boards -* Fixed "KeyError: 'content-length'" in PlatformIO Download Manager - (`issue #591 `_) - - -2.8.5 (2016-03-07) -~~~~~~~~~~~~~~~~~~ - -* Project generator for `NetBeans IDE `__ - (`issue #541 `_) -* Created package for Homebrew Mac OS X Package Manager: ``brew install - platformio`` - (`issue #395 `_) -* Updated Arduino core for Espressif platform to 2.1.0 - (`issue #544 `_) -* Added support for the ESP8266 ESP-07 board to - `Espressif `__ - (`issue #527 `_) -* Improved handling of String-based ``CPPDEFINES`` passed to extra ``build_flags`` - (`issue #526 `_) -* Generate appropriate project for CLion IDE and CVS - (`issue #523 `_) -* Use ``src_dir`` directory from `Project Configuration File platformio.ini `__ - when initializing project otherwise create base ``src`` directory - (`issue #536 `_) -* Fixed issue with incorrect handling of user's build flags where the base flags - were passed after user's flags to GCC compiler - (`issue #528 `_) -* Fixed issue with Project Generator when optional build flags were passed using - system environment variables: `PLATFORMIO_BUILD_FLAGS `__ - or `PLATFORMIO_BUILD_SRC_FLAGS `__ -* Fixed invalid detecting of compiler type - (`issue #550 `_) -* Fixed issue with updating package which was deleted manually by user - (`issue #555 `_) -* Fixed incorrect parsing of GCC ``-include`` flag - (`issue #552 `_) - -2.8.4 (2016-02-17) -~~~~~~~~~~~~~~~~~~ - -* Added support for the new ESP8266-based boards (ESPDuino, ESP-WROOM-02, - ESPresso Lite 1.0 & 2.0, SparkFun ESP8266 Thing Dev, ThaiEasyElec ESPino) to - `Espressif `__ - development platform -* Added ``board_f_flash`` option to `Project Configuration File platformio.ini `__ - which allows to specify `custom flash chip frequency `_ - for Espressif development platform - (`issue #501 `_) -* Added ``board_flash_mode`` option to `Project Configuration File platformio.ini `__ - which allows to specify `custom flash chip mode `_ - for Espressif development platform -* Handle new environment variables - `PLATFORMIO_UPLOAD_PORT `_ - and `PLATFORMIO_UPLOAD_FLAGS `_ - (`issue #518 `_) -* Fixed issue with ``CPPDEFINES`` which contain space and break PlatformIO - IDE Linter - (`IDE issue #34 `_) -* Fixed unable to link C++ standard library to Espressif platform build - (`issue #503 `_) -* Fixed issue with pointer (``char* myfunc()``) while converting from ``*.ino`` - to ``*.cpp`` - (`issue #506 `_) - -2.8.3 (2016-02-02) -~~~~~~~~~~~~~~~~~~ - -* Better integration of PlatformIO Builder with PlatformIO IDE Linter -* Fixed issue with removing temporary file while converting ``*.ino`` to - ``*.cpp`` -* Fixed missing dependency (mbed framework) for Atmel SAM development platform - (`issue #487 `_) - -2.8.2 (2016-01-29) -~~~~~~~~~~~~~~~~~~ - -* Corrected RAM size for NXP LPC1768 based boards - (`issue #484 `_) -* Exclude only ``test`` and ``tests`` folders from build process -* Reverted ``-Wl,-whole-archive`` hook for ST STM32 and mbed - -2.8.1 (2016-01-29) -~~~~~~~~~~~~~~~~~~ - -* Fixed a bug with Project Initialization in PlatformIO IDE - -2.8.0 (2016-01-29) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO IDE `_ for - Atom - (`issue #470 `_) -* Added ``pio`` command line alias for ``platformio`` command - (`issue #447 `_) -* Added SPL-Framework support for Nucleo F401RE board - (`issue #453 `_) -* Added ``upload_resetmethod`` option to `Project Configuration File platformio.ini `__ - which allows to specify `custom upload reset method `_ - for Espressif development platform - (`issue #444 `_) -* Allowed to force output of color ANSI-codes or to disable progress bar even - if the output is a ``pipe`` (not a ``tty``) using `Environment variables `__ - (`issue #465 `_) -* Set 1Mb SPIFFS for Espressif boards by default - (`issue #458 `_) -* Exclude ``test*`` folder by default from build process -* Generate project for IDEs with information about installed libraries -* Fixed builder for mbed framework and ST STM32 platform - - -2.7.1 (2016-01-06) -~~~~~~~~~~~~~~~~~~ - -* Initial support for Arduino Zero board - (`issue #356 `_) -* Added support for completions to Atom text editor using ``.clang_complete`` -* Generate default targets for `supported IDE `__ - (CLion, Eclipse IDE, Emacs, Sublime Text, VIM): Build, - Clean, Upload, Upload SPIFFS image, Upload using Programmer, Update installed - platforms and libraries - (`issue #427 `_) -* Updated Teensy Arduino Framework to 1.27 - (`issue #434 `_) -* Fixed uploading of EEPROM data using ``uploadeep`` target for Atmel AVR - development platform -* Fixed project generator for CLion IDE - (`issue #422 `_) -* Fixed package ``shasum`` validation on Mac OS X 10.11.2 - (`issue #429 `_) -* Fixed CMakeLists.txt ``add_executable`` has only one source file - (`issue #421 `_) - -2.7.0 (2015-12-30) -~~~~~~~~~~~~~~~~~~ - -**Happy New Year!** - -* Moved SCons to PlatformIO packages. PlatformIO does not require SCons to be - installed in your system. Significantly simplified installation process of - PlatformIO. ``pip install platformio`` rocks! -* Implemented uploading files to file system of ESP8266 SPIFFS (including OTA) - (`issue #382 `_) -* Added support for the new Adafruit boards Bluefruit Micro and Feather - (`issue #403 `_) -* Added support for RFDuino - (`issue #319 `_) -* Project generator for `Emacs `__ - text editor - (`pull #404 `_) -* Updated Arduino framework for Atmel AVR development platform to 1.6.7 -* Documented `firmware uploading for Atmel AVR development platform using - Programmers `_: - AVR ISP, AVRISP mkII, USBtinyISP, USBasp, Parallel Programmer and Arduino as ISP -* Fixed issue with current Python interpreter for Python-based tools - (`issue #417 `_) - -2.6.3 (2015-12-21) -~~~~~~~~~~~~~~~~~~ - -* Restored support for Espressif ESP8266 ESP-01 1MB board (ready for OTA) -* Fixed invalid ROM size for ESP8266-based boards - (`issue #396 `_) - -2.6.2 (2015-12-21) -~~~~~~~~~~~~~~~~~~ - -* Removed ``SCons`` from requirements list. PlatformIO will try to install it - automatically, otherwise users need to install it manually -* Fixed ``ChunkedEncodingError`` when SF connection is broken - (`issue #356 `_) - -2.6.1 (2015-12-18) -~~~~~~~~~~~~~~~~~~ - -* Added support for the new ESP8266-based boards (SparkFun ESP8266 Thing, - NodeMCU 0.9 & 1.0, Olimex MOD-WIFI-ESP8266(-DEV), Adafruit HUZZAH ESP8266, - ESPino, SweetPea ESP-210, WeMos D1, WeMos D1 mini) to - `Espressif `__ - development platform -* Created public `platformio-pkg-ldscripts `_ - repository for LD scripts. Moved common configuration for ESP8266 MCU to - ``esp8266.flash.common.ld`` - (`issue #379 `_) -* Improved documentation for `Espressif `__ - development platform: OTA update, custom Flash Size, Upload Speed and CPU - frequency -* Fixed reset method for Espressif NodeMCU (ESP-12E Module) - (`issue #380 `_) -* Fixed issue with code builder when build path contains spaces - (`issue #387 `_) -* Fixed project generator for Eclipse IDE and "duplicate path entries found - in project path" - (`issue #383 `_) - - -2.6.0 (2015-12-15) -~~~~~~~~~~~~~~~~~~ - -* Install only required packages depending on build environment - (`issue #308 `_) -* Added support for Raspberry Pi `WiringPi `__ - framework - (`issue #372 `_) -* Implemented Over The Air (OTA) upgrades for `Espressif `__ - development platform. - (`issue #365 `_) -* Updated `CMSIS framework `__ - and added CMSIS support for Nucleo F401RE board - (`issue #373 `_) -* Added support for Espressif ESP8266 ESP-01-1MB board (ready for OTA) -* Handle ``upload_flags`` option in `platformio.ini `__ - (`issue #368 `_) -* Improved PlatformIO installation on the Mac OS X El Capitan - -2.5.0 (2015-12-08) -~~~~~~~~~~~~~~~~~~ - -* Improved code builder for parallel builds (up to 4 times faster than before) -* Generate `.travis.yml `__ - CI and `.gitignore` files for embedded projects by default - (`issue #354 `_) -* Removed prompt with "auto-uploading" from `platformio init `__ - command and added ``--enable-auto-uploading`` option - (`issue #352 `_) -* Fixed incorrect behaviour of `platformio serialports monitor `__ - in pair with PySerial 3.0 - -2.4.1 (2015-12-01) -~~~~~~~~~~~~~~~~~~ - -* Restored ``PLATFORMIO`` macros with the current version - -2.4.0 (2015-12-01) -~~~~~~~~~~~~~~~~~~ - -* Added support for the new boards: Atmel ATSAMR21-XPRO, Atmel SAML21-XPRO-B, - Atmel SAMD21-XPRO, ST 32F469IDISCOVERY, ST 32L476GDISCOVERY, ST Nucleo F031K6, - ST Nucleo F042K6, ST Nucleo F303K8 and ST Nucleo L476RG -* Updated Arduino core for Espressif platform to 2.0.0 - (`issue #345 `_) -* Added to FAQ explanation of `Can not compile a library that compiles without issue - with Arduino IDE `_ - (`issue #331 `_) -* Fixed ESP-12E flash size - (`pull #333 `_) -* Fixed configuration for LowPowerLab MoteinoMEGA board - (`issue #335 `_) -* Fixed "LockFailed: failed to create appstate.json.lock" error for Windows -* Fixed relative include path for preprocessor using ``build_flags`` - (`issue #271 `_) - -2.3.5 (2015-11-18) -~~~~~~~~~~~~~~~~~~ - -* Added support of `libOpenCM3 `_ - framework for Nucleo F103RB board - (`issue #309 `_) -* Added support for Espressif ESP8266 ESP-12E board (NodeMCU) - (`issue #310 `_) -* Added support for pySerial 3.0 - (`issue #307 `_) -* Updated Arduino AVR/SAM frameworks to 1.6.6 - (`issue #321 `_) -* Upload firmware using external programmer via `platformio run --target program `__ - target - (`issue #311 `_) -* Fixed handling of upload port when ``board`` option is not specified in - `platformio.ini `__ - (`issue #313 `_) -* Fixed firmware uploading for `nordicrf51 `__ - development platform - (`issue #316 `_) -* Fixed installation on Mac OS X El Capitan - (`issue #312 `_) -* Fixed project generator for CLion IDE under Windows OS with invalid path to - executable - (`issue #326 `_) -* Fixed empty list with serial ports on Mac OS X - (`isge #294 `_) -* Fixed compilation error ``TWI_Disable not declared`` for Arduino Due board - (`issue #329 `_) - -2.3.4 (2015-10-13) -~~~~~~~~~~~~~~~~~~ - -* Full support of `CLion IDE `_ - including code auto-completion - (`issue #132 `_) -* PlatformIO `command completion in Terminal `_ for ``bash`` and ``zsh`` -* Added support for ubIQio Ardhat board - (`pull #302 `_) -* Install SCons automatically and avoid ``error: option --single-version-externally-managed not recognized`` - (`issue #279 `_) -* Use Teensy CLI Loader for upload of .hex files on Mac OS X - (`issue #306 `_) -* Fixed missing `framework-mbed `_ - package for `teensy `_ - platform - (`issue #305 `_) - -2.3.3 (2015-10-02) -~~~~~~~~~~~~~~~~~~ - -* Added support for LightBlue Bean board - (`pull #292 `_) -* Added support for ST Nucleo F446RE board - (`pull #293 `_) -* Fixed broken lock file for "appstate" storage - (`issue #288 `_) -* Fixed ESP8266 compile errors about RAM size when adding 1 library - (`issue #296 `_) - -2.3.2 (2015-09-10) -~~~~~~~~~~~~~~~~~~ - -* Allowed to use ST-Link uploader for mbed-based projects -* Explained how to use ``lib`` directory from the PlatformIO based project in - ``readme.txt`` which will be automatically generated using - `platformio init `__ - command - (`issue #273 `_) -* Found solution for "pip/scons error: option --single-version-externally-managed not - recognized" when install PlatformIO using ``pip`` package manager - (`issue #279 `_) -* Fixed firmware uploading to Arduino Leonardo board using Mac OS - (`issue #287 `_) -* Fixed `SConsNotInstalled` error for Linux Debian-based distributives - -2.3.1 (2015-09-06) -~~~~~~~~~~~~~~~~~~ - -* Fixed critical issue when `platformio init --ide `__ command hangs PlatformIO - (`issue #283 `_) - -2.3.0 (2015-09-05) -~~~~~~~~~~~~~~~~~~ - -* Added - `native `__, - `linux_arm `__, - `linux_i686 `__, - `linux_x86_64 `__, - `windows_x86 `__ - development platforms - (`issue #263 `_) -* Added `PlatformIO Demo `_ - page to documentation -* Simplified `installation `__ - process of PlatformIO - (`issue #274 `_) -* Significantly improved `Project Generator `__ which allows to integrate with `the most popular - IDE `__ -* Added short ``-h`` help option for PlatformIO and sub-commands -* Updated `mbed `__ - framework -* Updated ``tool-teensy`` package for `Teensy `__ - platform - (`issue #268 `_) -* Added FAQ answer when `Program "platformio" not found in PATH `_ - (`issue #272 `_) -* Generate "readme.txt" for project "lib" directory - (`issue #273 `_) -* Use toolchain's includes pattern ``include*`` for Project Generator - (`issue #277 `_) -* Added support for Adafruit Gemma board to - `atmelavr `__ - platform - (`pull #256 `_) -* Fixed includes list for Windows OS when generating project for `Eclipse IDE `__ - (`issue #270 `_) -* Fixed ``AttributeError: 'module' object has no attribute 'packages'`` - (`issue #252 `_) - -2.2.2 (2015-07-30) -~~~~~~~~~~~~~~~~~~ - -* Integration with `Atom IDE `__ -* Support for off-line/unpublished/private libraries - (`issue #260 `_) -* Disable project auto-clean while building/uploading firmware using - `platformio run --disable-auto-clean `_ option - (`issue #255 `_) -* Show internal errors from "Miniterm" using `platformio serialports monitor `__ command - (`issue #257 `_) -* Fixed `platformio serialports monitor --help `__ information with HEX char for hotkeys - (`issue #253 `_) -* Handle "OSError: [Errno 13] Permission denied" for PlatformIO installer script - (`issue #254 `_) - -2.2.1 (2015-07-17) -~~~~~~~~~~~~~~~~~~ - -* Project generator for `CLion IDE `__ - (`issue #132 `_) -* Updated ``tool-bossac`` package to 1.5 version for `atmelsam `__ platform - (`issue #251 `_) -* Updated ``sdk-esp8266`` package for `espressif `__ platform -* Fixed incorrect arguments handling for `platformio serialports monitor `_ command - (`issue #248 `_) - -2.2.0 (2015-07-01) -~~~~~~~~~~~~~~~~~~ - -* Allowed to exclude/include source files from build process using - `src_filter `__ - (`issue #240 `_) -* Launch own extra script before firmware building/uploading processes - (`issue #239 `_) -* Specify own path to the linker script (ld) using - `build_flags `__ - option - (`issue #233 `_) -* Specify library compatibility with the all platforms/frameworks - using ``*`` symbol in - `library.json `__ -* Added support for new embedded boards: *ST 32L0538DISCOVERY and Delta DFCM-NNN40* - to `Framework mbed `__ -* Updated packages for - `Framework Arduino (AVR, SAM, Espressif and Teensy cores `__, - `Framework mbed `__, - `Espressif ESP8266 SDK `__ - (`issue #246 `_) -* Fixed ``stk500v2_command(): command failed`` - (`issue #238 `_) -* Fixed IDE project generator when board is specified - (`issue #242 `_) -* Fixed relative path for includes when generating project for IDE - (`issue #243 `_) -* Fixed ESP8266 native SDK exception - (`issue #245 `_) - -2.1.2 (2015-06-21) -~~~~~~~~~~~~~~~~~~ - -* Fixed broken link to SCons installer - -2.1.1 (2015-06-09) -~~~~~~~~~~~~~~~~~~ - -* Automatically detect upload port using VID:PID board settings - (`issue #231 `_) -* Improved detection of build changes -* Avoided ``LibInstallDependencyError`` when more than 1 library is found - (`issue #229 `_) - -2.1.0 (2015-06-03) -~~~~~~~~~~~~~~~~~~ - -* Added Silicon Labs EFM32 `siliconlabsefm32 `_ - development platform - (`issue #226 `_) -* Integrate PlatformIO with `Circle CI `_ and - `Shippable CI `_ -* Described in documentation how to `create/register own board `_ for PlatformIO -* Disabled "nano.specs" for ARM-based platforms - (`issue #219 `_) -* Fixed "ConnectionError" when PlatformIO SF Storage is off-line -* Fixed resolving of C/C++ std libs by Eclipse IDE - (`issue #220 `_) -* Fixed firmware uploading using USB programmer (USBasp) for - `atmelavr `_ - platform - (`issue #221 `_) - -2.0.2 (2015-05-27) -~~~~~~~~~~~~~~~~~~ - -* Fixed libraries order for "Library Dependency Finder" under Linux OS - -2.0.1 (2015-05-27) -~~~~~~~~~~~~~~~~~~ - -* Handle new environment variable - `PLATFORMIO_BUILD_FLAGS `_ -* Pass to API requests information about Continuous Integration system. This - information will be used by PlatformIO-API. -* Use ``include`` directories from toolchain when initialising project for IDE - (`issue #210 `_) -* Added support for new WildFire boards from - `Wicked Device `_ to - `atmelavr `__ - platform -* Updated `Arduino Framework `__ to - 1.6.4 version (`issue #212 `_) -* Handle Atmel AVR Symbols when initialising project for IDE - (`issue #216 `_) -* Fixed bug with converting ``*.ino`` to ``*.cpp`` -* Fixed failing with ``platformio init --ide eclipse`` without boards - (`issue #217 `_) - -2.0.0 (2015-05-22) -~~~~~~~~~~~~~~~~~~ - -*Made in* `Paradise `_ - -* PlatformIO as `Continuous Integration `_ - (CI) tool for embedded projects - (`issue #108 `_) -* Initialise PlatformIO project for the specified IDE - (`issue #151 `_) -* PlatformIO CLI 2.0: "platform" related commands have been - moved to ``platformio platforms`` subcommand - (`issue #158 `_) -* Created `PlatformIO gitter.im `_ room - (`issue #174 `_) -* Global ``-f, --force`` option which will force to accept any - confirmation prompts - (`issue #152 `_) -* Run project with `platformio run --project-dir `_ option without changing the current working - directory - (`issue #192 `_) -* Control verbosity of `platformio run `_ command via ``-v/--verbose`` option -* Add library dependencies for build environment using - `lib_install `_ - option in ``platformio.ini`` - (`issue #134 `_) -* Specify libraries which are compatible with build environment using - `lib_use `_ - option in ``platformio.ini`` - (`issue #148 `_) -* Add more boards to PlatformIO project with - `platformio init --board `__ - command - (`issue #167 `_) -* Choose which library to update - (`issue #168 `_) -* Specify `platformio init --env-prefix `__ when initialise/update project - (`issue #182 `_) -* Added new Armstrap boards - (`issue #204 `_) -* Updated SDK for `espressif `__ - development platform to v1.1 - (`issue #179 `_) -* Disabled automatic updates by default for platforms, packages and libraries - (`issue #171 `_) -* Fixed bug with creating copies of source files - (`issue #177 `_) +See `PlatformIO Core 2.0 history `__. PlatformIO Core 1 ----------------- -1.5.0 (2015-05-15) -~~~~~~~~~~~~~~~~~~ - -* Added support of `Framework mbed `_ - for Teensy 3.1 - (`issue #183 `_) -* Added GDB as alternative uploader to `ststm32 `__ platform - (`issue #175 `_) -* Added `examples `__ - with preconfigured IDE projects - (`issue #154 `_) -* Fixed firmware uploading under Linux OS for Arduino Leonardo board - (`issue #178 `_) -* Fixed invalid "mbed" firmware for Nucleo F411RE - (`issue #185 `_) -* Fixed parsing of includes for PlatformIO Library Dependency Finder - (`issue #189 `_) -* Fixed handling symbolic links within source code directory - (`issue #190 `_) -* Fixed cancelling any previous definition of name, either built in or provided - with a ``-D`` option - (`issue #191 `_) - -1.4.0 (2015-04-11) -~~~~~~~~~~~~~~~~~~ - -* Added `espressif `_ - development platform with ESP01 board -* Integrated PlatformIO with AppVeyor Windows based Continuous Integration system - (`issue #149 `_) -* Added support for Teensy LC board to - `teensy `__ - platform -* Added support for new Arduino based boards by *SparkFun, BQ, LightUp, - LowPowerLab, Quirkbot, RedBearLab, TinyCircuits* to - `atmelavr `__ - platform -* Upgraded `Arduino Framework `__ to - 1.6.3 version (`issue #156 `_) -* Upgraded `Energia Framework `__ to - 0101E0015 version (`issue #146 `_) -* Upgraded `Arduino Framework with Teensy Core `_ - to 1.22 version - (`issue #162 `_, - `issue #170 `_) -* Fixed exceptions with PlatformIO auto-updates when Internet connection isn't - active - - -1.3.0 (2015-03-27) -~~~~~~~~~~~~~~~~~~ - -* Moved PlatformIO source code and repositories from `Ivan Kravets `_ - account to `PlatformIO Organisation `_ - (`issue #138 `_) -* Added support for new Arduino based boards by *SparkFun, RepRap, Sanguino* to - `atmelavr `__ - platform - (`issue #127 `_, - `issue #131 `_) -* Added integration instructions for `Visual Studio `_ - and `Sublime Text `_ IDEs -* Improved handling of multi-file ``*.ino/pde`` sketches - (`issue #130 `_) -* Fixed wrong insertion of function prototypes converting ``*.ino/pde`` - (`issue #137 `_, - `issue #140 `_) - - - -1.2.0 (2015-03-20) -~~~~~~~~~~~~~~~~~~ - -* Added full support of `mbed `__ - framework including libraries: *RTOS, Ethernet, DSP, FAT, USB*. -* Added `freescalekinetis `_ - development platform with Freescale Kinetis Freedom boards -* Added `nordicnrf51 `_ - development platform with supported boards from *JKSoft, Nordic, RedBearLab, - Switch Science* -* Added `nxplpc `_ - development platform with supported boards from *CQ Publishing, Embedded - Artists, NGX Technologies, NXP, Outrageous Circuits, SeeedStudio, - Solder Splash Labs, Switch Science, u-blox* -* Added support for *ST Nucleo* boards to - `ststm32 `__ - development platform -* Created new `Frameworks `__ - page in documentation and added to `PlatformIO Web Site `_ - (`issue #115 `_) -* Introduced online `Embedded Boards Explorer `_ -* Automatically append define ``-DPLATFORMIO=%version%`` to - builder (`issue #105 `_) -* Renamed ``stm32`` development platform to - `ststm32 `__ -* Renamed ``opencm3`` framework to - `libopencm3 `__ -* Fixed uploading for `atmelsam `__ - development platform -* Fixed re-arranging the ``*.ino/pde`` files when converting to ``*.cpp`` - (`issue #100 `_) - -1.1.0 (2015-03-05) -~~~~~~~~~~~~~~~~~~ - -* Implemented ``PLATFORMIO_*`` environment variables - (`issue #102 `_) -* Added support for *SainSmart* boards to - `atmelsam `__ - development platform -* Added - `Project Configuration `__ - option named `envs_dir `__ -* Disabled "prompts" automatically for *Continuous Integration* systems - (`issue #103 `_) -* Fixed firmware uploading for - `atmelavr `__ - boards which work within ``usbtiny`` protocol -* Fixed uploading for *Digispark* board (`issue #106 `_) - -1.0.1 (2015-02-27) -~~~~~~~~~~~~~~~~~~ - -**PlatformIO 1.0 - recommended for production** - -* Changed development status from ``beta`` to ``Production/Stable`` -* Added support for *ARM*-based credit-card sized computers: - `Raspberry Pi `_, - `BeagleBone `_ and `CubieBoard `_ -* Added `atmelsam `__ - development platform with supported boards: *Arduino Due and Digistump DigiX* - (`issue #71 `_) -* Added `ststm32 `__ - development platform with supported boards: *Discovery kit for STM32L151/152, - STM32F303xx, STM32F407/417 lines* and `libOpenCM3 Framework `_ - (`issue #73 `_) -* Added `teensy `_ - development platform with supported boards: *Teensy 2.x & 3.x* - (`issue #72 `_) -* Added new *Arduino* boards to - `atmelavr `__ - platform: *Arduino NG, Arduino BT, Arduino Esplora, Arduino Ethernet, - Arduino Robot Control, Arduino Robot Motor and Arduino Yun* -* Added support for *Adafruit* boards to - `atmelavr `__ - platform: *Adafruit Flora and Adafruit Trinkets* - (`issue #65 `_) -* Added support for *Digispark* boards to - `atmelavr `__ - platform: *Digispark USB Development Board and Digispark Pro* - (`issue #47 `_) -* Covered code with tests (`issue #2 `_) -* Refactored *Library Dependency Finder* (issues - `#48 `_, - `#50 `_, - `#55 `_) -* Added `src_dir `__ - option to ``[platformio]`` section of - `platformio.ini `__ - which allows to redefine location to project's source directory - (`issue #83 `_) -* Added ``--json-output`` option to - `platformio boards `__ - and `platformio search `__ - commands which allows to return the output in `JSON `_ format - (`issue #42 `_) -* Allowed to ignore some libs from *Library Dependency Finder* via - `lib_ignore `_ option -* Improved `platformio run `__ - command: asynchronous output for build process, timing and detailed - information about environment configuration - (`issue #74 `_) -* Output compiled size and static memory usage with - `platformio run `__ - command (`issue #59 `_) -* Updated `framework-arduino` AVR & SAM to 1.6 stable version -* Fixed an issue with the libraries that are git repositories - (`issue #49 `_) -* Fixed handling of assembly files - (`issue #58 `_) -* Fixed compiling error if space is in user's folder - (`issue #56 `_) -* Fixed `AttributeError: 'module' object has no attribute 'disable_warnings'` - when a version of `requests` package is less then 2.4.0 -* Fixed bug with invalid process's "return code" when PlatformIO has internal - error (`issue #81 `_) -* Several bug fixes, increased stability and performance improvements +See `PlatformIO Core 1.0 history `__. PlatformIO Core Preview ----------------------- -0.10.2 (2015-01-06) -~~~~~~~~~~~~~~~~~~~ - -* Fixed an issue with ``--json-output`` - (`issue #42 `_) -* Fixed an exception during - `platformio upgrade `__ - under Windows OS (`issue #45 `_) - -0.10.1 (2015-01-02) -~~~~~~~~~~~~~~~~~~~ - -* Added ``--json-output`` option to - `platformio list `__, - `platformio serialports list `__ and - `platformio lib list `__ - commands which allows to return the output in `JSON `_ format - (`issue #42 `_) -* Fixed missing auto-uploading by default after `platformio init `__ - command - -0.10.0 (2015-01-01) -~~~~~~~~~~~~~~~~~~~ - -**Happy New Year!** - -* Implemented `platformio boards `_ - command (`issue #11 `_) -* Added support of *Engduino* boards for - `atmelavr `__ - platform (`issue #38 `_) -* Added ``--board`` option to `platformio init `__ - command which allows to initialise project with the specified embedded boards - (`issue #21 `_) -* Added `example with uploading firmware `_ - via USB programmer (USBasp) for - `atmelavr `_ - *MCUs* (`issue #35 `_) -* Automatic detection of port on `platformio serialports monitor `_ - (`issue #37 `_) -* Allowed auto-installation of platforms when prompts are disabled (`issue #43 `_) -* Fixed urllib3's *SSL* warning under Python <= 2.7.2 (`issue #39 `_) -* Fixed bug with *Arduino USB* boards (`issue #40 `_) - -0.9.2 (2014-12-10) -~~~~~~~~~~~~~~~~~~ - -* Replaced "dark blue" by "cyan" colour for the texts (`issue #33 `_) -* Added new setting ``enable_prompts`` and allowed to disable all *PlatformIO* prompts (useful for cloud compilers) - (`issue #34 `_) -* Fixed compilation bug on *Windows* with installed *MSVC* (`issue #18 `_) - -0.9.1 (2014-12-05) -~~~~~~~~~~~~~~~~~~ - -* Ask user to install platform (when it hasn't been installed yet) within - `platformio run `__ - and `platformio show `_ commands -* Improved main `documentation `_ -* Fixed "*OSError: [Errno 2] No such file or directory*" within - `platformio run `__ - command when PlatformIO isn't installed properly -* Fixed example for Eclipse IDE with Tiva board - (`issue #32 `_) -* Upgraded Eclipse Project Examples - to latest *Luna* and *PlatformIO* releases - -0.9.0 (2014-12-01) -~~~~~~~~~~~~~~~~~~ - -* Implemented `platformio settings `_ command -* Improved `platformio init `_ command. - Added new option ``--project-dir`` where you can specify another path to - directory where new project will be initialized (`issue #31 `_) -* Added *Migration Manager* which simplifies process with upgrading to a - major release -* Added *Telemetry Service* which should help us make *PlatformIO* better -* Implemented *PlatformIO AppState Manager* which allow to have multiple - ``.platformio`` states. -* Refactored *Package Manager* -* Download Manager: fixed SHA1 verification within *Cygwin Environment* - (`issue #26 `_) -* Fixed bug with code builder and built-in Arduino libraries - (`issue #28 `_) - -0.8.0 (2014-10-19) -~~~~~~~~~~~~~~~~~~ - -* Avoided trademark issues in `library.json `_ - with the new fields: `frameworks `_, - `platforms `_ - and `dependencies `_ - (`issue #17 `_) -* Switched logic from "Library Name" to "Library Registry ID" for all - `platformio lib `_ - commands (install, uninstall, update and etc.) -* Renamed ``author`` field to `authors `_ - and allowed to setup multiple authors per library in `library.json `_ -* Added option to specify "maintainer" status in `authors `_ field -* New filters/options for `platformio lib search `_ - command: ``--framework`` and ``--platform`` - -0.7.1 (2014-10-06) -~~~~~~~~~~~~~~~~~~ - -* Fixed bug with order for includes in conversation from INO/PDE to CPP -* Automatic detection of port on upload (`issue #15 `_) -* Fixed lib update crashing when no libs are installed (`issue #19 `_) - - -0.7.0 (2014-09-24) -~~~~~~~~~~~~~~~~~~ - -* Implemented new `[platformio] `_ - section for Configuration File with `home_dir `_ - option (`issue #14 `_) -* Implemented *Library Manager* (`issue #6 `_) - -0.6.0 (2014-08-09) -~~~~~~~~~~~~~~~~~~ - -* Implemented `platformio serialports monitor `_ (`issue #10 `_) -* Fixed an issue ``ImportError: No module named platformio.util`` (`issue #9 `_) -* Fixed bug with auto-conversation from Arduino \*.ino to \*.cpp - -0.5.0 (2014-08-04) -~~~~~~~~~~~~~~~~~~ - -* Improved nested lookups for libraries -* Disabled default warning flag "-Wall" -* Added auto-conversation from \*.ino to valid \*.cpp for Arduino/Energia - frameworks (`issue #7 `_) -* Added `Arduino example `_ - with external library (*Adafruit CC3000*) -* Implemented `platformio upgrade `_ - command and "auto-check" for the latest - version (`issue #8 `_) -* Fixed an issue with "auto-reset" for *Raspduino* board -* Fixed a bug with nested libs building - -0.4.0 (2014-07-31) -~~~~~~~~~~~~~~~~~~ - -* Implemented `platformio serialports `_ command -* Allowed to put special build flags only for ``src`` files via - `src_build_flags `_ - environment option -* Allowed to override some of settings via system environment variables - such as: ``PLATFORMIO_SRC_BUILD_FLAGS`` and ``PLATFORMIO_ENVS_DIR`` -* Added ``--upload-port`` option for `platformio run `__ command -* Implemented (especially for `SmartAnthill `_) - `platformio run -t uploadlazy `_ - target (no dependencies to framework libs, ELF and etc.) -* Allowed to skip default packages via `platformio install --skip-default-package `_ - option -* Added tools for *Raspberry Pi* platform -* Added support for *Microduino* and *Raspduino* boards in - `atmelavr `_ platform - -0.3.1 (2014-06-21) -~~~~~~~~~~~~~~~~~~ - -* Fixed auto-installer for Windows OS (bug with %PATH% custom installation) - - -0.3.0 (2014-06-21) -~~~~~~~~~~~~~~~~~~ - -* Allowed to pass multiple "SomePlatform" to install/uninstall commands -* Added "IDE Integration" section to README with Eclipse project examples -* Created auto installer script for *PlatformIO* (`issue #3 `_) -* Added "Super-Quick" way to Installation section (README) -* Implemented "build_flags" option for environments (`issue #4 `_) - - -0.2.0 (2014-06-15) -~~~~~~~~~~~~~~~~~~ - -* Resolved `issue #1 "Build referred libraries" `_ -* Renamed project's "libs" directory to "lib" -* Added `arduino-internal-library `_ example -* Changed to beta status - - -0.1.0 (2014-06-13) -~~~~~~~~~~~~~~~~~~ - -* Birth! First alpha release +See `PlatformIO Core Preview history `__. diff --git a/docs b/docs index c536bff835..6162e5d14a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c536bff8352fc0a26366d1d9ca73b8e60af8d205 +Subproject commit 6162e5d14a9b64bbc61390e641e38e336822fc10 diff --git a/examples b/examples index f0f4e0971b..84855946ea 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit f0f4e0971b0f9f7b4d77a47cb436f910bf3b4add +Subproject commit 84855946ea09b5e41ddbbae455f00e897060346d From 1aaa9b6707e3fc76254c6e9c26f1c412a39b3147 Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 26 Aug 2020 17:44:01 +0300 Subject: [PATCH 208/223] Update changelog with static analysis section --- HISTORY.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 53a933d377..37845a52e2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -70,6 +70,16 @@ PlatformIO Core 5 - Fixed an issue when running multiple test environments (`issue #3523 `_) - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) +* **Static Code Analysis** + + - Updated analysis tools: + + * ``Cppcheck v2.1`` with a new "soundy" analysis option and improved code parser + * ``PVS-Studio v7.08`` with a new file list analysis mode and extended list of diagnostic rules + + - Added Cppcheck package for ARM-based single-board computers (`issue #3559 `_) + - Fixed an issue with PIO Check when a defect with multiline error message is not reported in verbose mode (`issue #3631 `_) + * **Miscellaneous** - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) From 80c1774a192221bd66cc764dc930e05ed1faf330 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 14:08:26 +0300 Subject: [PATCH 209/223] Docs: PlatformIO Core 5.0: new commands, migration guide, other improvements --- HISTORY.rst | 47 +++++++++++++---------- docs | 2 +- platformio/app.py | 2 +- platformio/clients/account.py | 2 +- platformio/clients/http.py | 2 +- platformio/commands/access.py | 6 +-- platformio/commands/account.py | 10 ++--- platformio/commands/boards.py | 2 +- platformio/commands/check/command.py | 2 +- platformio/commands/ci.py | 2 +- platformio/commands/debug/command.py | 2 +- platformio/commands/device/command.py | 2 +- platformio/commands/home/command.py | 2 +- platformio/commands/lib/command.py | 2 +- platformio/commands/org.py | 33 ++++++++-------- platformio/commands/package.py | 2 +- platformio/commands/platform.py | 2 +- platformio/commands/project.py | 6 +-- platformio/commands/remote/client/base.py | 2 +- platformio/commands/remote/command.py | 7 ++-- platformio/commands/run/command.py | 2 +- platformio/commands/settings.py | 2 +- platformio/commands/team.py | 10 +++-- platformio/commands/test/command.py | 2 +- platformio/package/manager/core.py | 2 +- platformio/project/options.py | 6 +-- 26 files changed, 86 insertions(+), 75 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 37845a52e2..2d37cb6bc9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,45 +8,51 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +- `Migration guide from 4.x to 5.0 `__ + 5.0.0 (2020-??-??) ~~~~~~~~~~~~~~~~~~ - * Integration with the new **PlatformIO Trusted Registry** - Enterprise-grade package storage with high availability (multi replicas) - Secure, fast, and reliable global content delivery network (CDN) - - Universal support for all embedded packages: + - Universal support for all packages: * Libraries * Development platforms * Toolchains - - Built-in fine-grained access control (role based, teams, organizations) - - Command Line Interface: + - Built-in fine-grained access control (role-based, teams, organizations) + - New CLI commands: - * `pio package publish `__ – publish a personal or organization package - * `pio package unpublish `__ – remove a pushed package from the registry - * Grant package access to the team members or maintainers + * `pio package `__ – manage packages in the registry + * `pio access `__ – manage package access for users, teams, and maintainers -* Integration with the new `Account Management System `__ +* Integration with the new **Account Management System** - - Manage own organizations - - Manage organization teams - - Manage resource access + - `Manage organizations and owners of an organization `__ + - `Manage teams and team memberships `__ * New **Package Management System** - - Integrated PlatformIO Core with the new PlatformIO Trusted Registry + - Integrated PlatformIO Core with the new PlatformIO Registry - Strict dependency declaration using owner name (resolves name conflicts) (`issue #1824 `_) - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) + - Dropped support for "packageRepositories" section in "platform.json" manifest (please publish packages directly to the registry) + +* **Build System** -* **PlatformIO Build System** + - Upgraded build engine to the `SCons 4.0 - a next-generation software construction tool `__ + + * `Configuration files are Python scripts `__ – use the power of a real programming language to solve build problems + * Built-in reliable and automatic dependency analysis + * Improved support for parallel builds + * Ability to `share built files in a cache `__ to speed up multiple builds - - Upgraded to `SCons 4.0 - a next-generation software construction tool `__ - New `Custom Targets `__ - * Pre/Post processing based on a dependent sources (other target, source file, etc.) + * Pre/Post processing based on dependent sources (another target, source file, etc.) * Command launcher with own arguments * Launch command with custom options declared in `"platformio.ini" `__ * Python callback as a target (use the power of Python interpreter and PlatformIO Build API) @@ -55,12 +61,12 @@ PlatformIO Core 5 - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) - - Fixed an issue with ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) + - Fixed an issue with the ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) * **Project Management** - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required - - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using existing environment + - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using the existing environment - Dump build system data intended for IDE extensions/plugins using a new `pio project data `__ command - Do not generate ".travis.yml" for a new project, let the user have a choice @@ -75,17 +81,16 @@ PlatformIO Core 5 - Updated analysis tools: * ``Cppcheck v2.1`` with a new "soundy" analysis option and improved code parser - * ``PVS-Studio v7.08`` with a new file list analysis mode and extended list of diagnostic rules + * ``PVS-Studio v7.08`` with a new file list analysis mode and an extended list of diagnostic rules - Added Cppcheck package for ARM-based single-board computers (`issue #3559 `_) - - Fixed an issue with PIO Check when a defect with multiline error message is not reported in verbose mode (`issue #3631 `_) + - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 `_) * **Miscellaneous** - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) - - Do not escape compiler arguments in VSCode template on Windows - + - Do not escape compiler arguments in VSCode template on Windows. .. _release_notes_4: diff --git a/docs b/docs index 6162e5d14a..7def3c5008 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6162e5d14a9b64bbc61390e641e38e336822fc10 +Subproject commit 7def3c5008c99aae8f24984556e2a057195224e9 diff --git a/platformio/app.py b/platformio/app.py index 5990050065..0196fac41e 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -68,7 +68,7 @@ def projects_dir_validate(projects_dir): "value": False, }, "projects_dir": { - "description": "Default location for PlatformIO projects (PIO Home)", + "description": "Default location for PlatformIO projects (PlatformIO Home)", "value": get_default_projects_dir(), "validator": projects_dir_validate, }, diff --git a/platformio/clients/account.py b/platformio/clients/account.py index e2abde1798..1c4b6755fa 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -27,7 +27,7 @@ class AccountError(PlatformioException): class AccountNotAuthorized(AccountError): - MESSAGE = "You are not authorized! Please log in to PIO Account." + MESSAGE = "You are not authorized! Please log in to PlatformIO Account." class AccountAlreadyAuthorized(AccountError): diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 8e73295853..4d59bcaa62 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -45,7 +45,7 @@ class InternetIsOffline(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" "PlatformIO needs the Internet connection to" - " download dependent packages or to work with PIO Account." + " download dependent packages or to work with PlatformIO Account." ) diff --git a/platformio/commands/access.py b/platformio/commands/access.py index 6a59be7aae..8b65ba34a0 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -33,7 +33,7 @@ def validate_client(value): return value -@click.group("access", short_help="Manage Resource Access") +@click.group("access", short_help="Manage resource access") def cli(): pass @@ -75,7 +75,7 @@ def access_private(urn, urn_type): @click.argument("level", type=click.Choice(["admin", "maintainer", "guest"])) @click.argument( "client", - metavar="[ORGNAME:TEAMNAME|USERNAME]", + metavar="[|]", callback=lambda _, __, value: validate_client(value), ) @click.argument( @@ -108,7 +108,7 @@ def access_revoke(client, urn, urn_type): ) -@cli.command("list", short_help="List resources") +@cli.command("list", short_help="List published resources") @click.argument("owner", required=False) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") @click.option("--json-output", is_flag=True) diff --git a/platformio/commands/account.py b/platformio/commands/account.py index 3a1492ec23..88aab68b90 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -24,7 +24,7 @@ from platformio.clients.account import AccountClient, AccountNotAuthorized -@click.group("account", short_help="Manage PIO Account") +@click.group("account", short_help="Manage PlatformIO account") def cli(): pass @@ -60,7 +60,7 @@ def validate_password(value): return value -@cli.command("register", short_help="Create new PIO Account") +@cli.command("register", short_help="Create new PlatformIO Account") @click.option( "-u", "--username", @@ -90,7 +90,7 @@ def account_register(username, email, password, firstname, lastname): ) -@cli.command("login", short_help="Log in to PIO Account") +@cli.command("login", short_help="Log in to PlatformIO Account") @click.option("-u", "--username", prompt="Username or email") @click.option("-p", "--password", prompt=True, hide_input=True) def account_login(username, password): @@ -99,7 +99,7 @@ def account_login(username, password): return click.secho("Successfully logged in!", fg="green") -@cli.command("logout", short_help="Log out of PIO Account") +@cli.command("logout", short_help="Log out of PlatformIO Account") def account_logout(): client = AccountClient() client.logout() @@ -195,7 +195,7 @@ def account_destroy(): return click.secho("User account has been destroyed.", fg="green",) -@cli.command("show", short_help="PIO Account information") +@cli.command("show", short_help="PlatformIO Account information") @click.option("--offline", is_flag=True) @click.option("--json-output", is_flag=True) def account_show(offline, json_output): diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 962ab504d6..4170b32ff7 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -22,7 +22,7 @@ from platformio.package.manager.platform import PlatformPackageManager -@click.command("boards", short_help="Embedded Board Explorer") +@click.command("boards", short_help="Embedded board explorer") @click.argument("query", required=False) @click.option("--installed", is_flag=True) @click.option("--json-output", is_flag=True) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index a5c4e1e7f2..8f9a6dcae2 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -31,7 +31,7 @@ from platformio.project.helpers import find_project_dir_above, get_project_dir -@click.command("check", short_help="Run a static analysis tool on code") +@click.command("check", short_help="Static code analysis") @click.option("-e", "--environment", multiple=True) @click.option( "-d", diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index f68b2bb763..e72ddf76fb 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -44,7 +44,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument raise click.BadParameter("Found invalid path: %s" % invalid_path) -@click.command("ci", short_help="Continuous Integration") +@click.command("ci", short_help="Continuous integration") @click.argument("src", nargs=-1, callback=validate_path) @click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY") @click.option("--exclude", multiple=True) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 98115cbfbf..fc83405c0e 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -33,7 +33,7 @@ @click.command( "debug", context_settings=dict(ignore_unknown_options=True), - short_help="PIO Unified Debugger", + short_help="Unified debugger", ) @click.option( "-d", diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index 463116f92f..a66cb9961a 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -26,7 +26,7 @@ from platformio.project.exception import NotPlatformIOProjectError -@click.group(short_help="Monitor device or list existing") +@click.group(short_help="Device manager & serial/socket monitor") def cli(): pass diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index dd733bb601..6cb26ed955 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -25,7 +25,7 @@ from platformio.package.manager.core import get_core_package_dir, inject_contrib_pysite -@click.command("home", short_help="PIO Home") +@click.command("home", short_help="UI to manage PlatformIO") @click.option("--port", type=int, default=8008, help="HTTP port, default=8008") @click.option( "--host", diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 96d39814b2..d871c91729 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -50,7 +50,7 @@ def get_project_global_lib_dir(): return ProjectConfig.get_instance().get_optional_dir("globallib") -@click.group(short_help="Library Manager") +@click.group(short_help="Library manager") @click.option( "-d", "--storage-dir", diff --git a/platformio/commands/org.py b/platformio/commands/org.py index a7e0f1e974..ac13d13f74 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -23,7 +23,7 @@ from platformio.commands.account import validate_email, validate_username -@click.group("org", short_help="Manage Organizations") +@click.group("org", short_help="Manage organizations") def cli(): pass @@ -44,11 +44,11 @@ def org_create(orgname, email, displayname): client = AccountClient() client.create_org(orgname, email, displayname) return click.secho( - "The organization %s has been successfully created." % orgname, fg="green", + "The organization `%s` has been successfully created." % orgname, fg="green", ) -@cli.command("list", short_help="List organizations") +@cli.command("list", short_help="List organizations and their members") @click.option("--json-output", is_flag=True) def org_list(json_output): client = AccountClient() @@ -56,7 +56,7 @@ def org_list(json_output): if json_output: return click.echo(json.dumps(orgs)) if not orgs: - return click.echo("You do not have any organizations") + return click.echo("You do not have any organization") for org in orgs: click.echo() click.secho(org.get("orgname"), fg="cyan") @@ -77,15 +77,17 @@ def org_list(json_output): @cli.command("update", short_help="Update organization") -@click.argument("orgname") +@click.argument("cur_orgname") @click.option( - "--new-orgname", callback=lambda _, __, value: validate_orgname(value), + "--orgname", + callback=lambda _, __, value: validate_orgname(value), + help="A new orgname", ) @click.option("--email") -@click.option("--displayname",) -def org_update(orgname, **kwargs): +@click.option("--displayname") +def org_update(cur_orgname, **kwargs): client = AccountClient() - org = client.get_org(orgname) + org = client.get_org(cur_orgname) del org["owners"] new_org = org.copy() if not any(kwargs.values()): @@ -101,9 +103,10 @@ def org_update(orgname, **kwargs): new_org.update( {key.replace("new_", ""): value for key, value in kwargs.items() if value} ) - client.update_org(orgname, new_org) + client.update_org(cur_orgname, new_org) return click.secho( - "The organization %s has been successfully updated." % orgname, fg="green", + "The organization `%s` has been successfully updated." % cur_orgname, + fg="green", ) @@ -112,13 +115,13 @@ def org_update(orgname, **kwargs): def account_destroy(orgname): client = AccountClient() click.confirm( - "Are you sure you want to delete the %s organization account?\n" + "Are you sure you want to delete the `%s` organization account?\n" "Warning! All linked data will be permanently removed and can not be restored." % orgname, abort=True, ) client.destroy_org(orgname) - return click.secho("Organization %s has been destroyed." % orgname, fg="green",) + return click.secho("Organization `%s` has been destroyed." % orgname, fg="green",) @cli.command("add", short_help="Add a new owner to organization") @@ -128,7 +131,7 @@ def org_add_owner(orgname, username): client = AccountClient() client.add_org_owner(orgname, username) return click.secho( - "The new owner %s has been successfully added to the %s organization." + "The new owner `%s` has been successfully added to the `%s` organization." % (username, orgname), fg="green", ) @@ -141,7 +144,7 @@ def org_remove_owner(orgname, username): client = AccountClient() client.remove_org_owner(orgname, username) return click.secho( - "The %s owner has been successfully removed from the %s organization." + "The `%s` owner has been successfully removed from the `%s` organization." % (username, orgname), fg="green", ) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 6ec78d3808..88f6c0d3ed 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -32,7 +32,7 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument return value -@click.group("package", short_help="Package Manager") +@click.group("package", short_help="Package manager") def cli(): pass diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 7725be392e..588e7ccc39 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -26,7 +26,7 @@ from platformio.platform.factory import PlatformFactory -@click.group(short_help="Platform Manager") +@click.group(short_help="Platform manager") def cli(): pass diff --git a/platformio/commands/project.py b/platformio/commands/project.py index bd37175a49..70660fa26a 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -30,7 +30,7 @@ from platformio.project.helpers import is_platformio_project, load_project_ide_data -@click.group(short_help="Project Manager") +@click.group(short_help="Project manager") def cli(): pass @@ -333,7 +333,7 @@ def init_test_readme(test_dir): with open(os.path.join(test_dir, "README"), "w") as fp: fp.write( """ -This directory is intended for PIO Unit Testing and project tests. +This directory is intended for PlatformIO Unit Testing and project tests. Unit Testing is a software testing method by which individual units of source code, sets of one or more MCU program modules together with associated @@ -341,7 +341,7 @@ def init_test_readme(test_dir): determine whether they are fit for use. Unit testing finds problems early in the development cycle. -More information about PIO Unit Testing: +More information about PlatformIO Unit Testing: - https://docs.platformio.org/page/plus/unit-testing.html """, ) diff --git a/platformio/commands/remote/client/base.py b/platformio/commands/remote/client/base.py index 806d7bda23..7ca7be3b1d 100644 --- a/platformio/commands/remote/client/base.py +++ b/platformio/commands/remote/client/base.py @@ -72,7 +72,7 @@ def _log_observer(self, event): def connect(self): self.log.info("Name: {name}", name=self.name) - self.log.info("Connecting to PIO Remote Cloud") + self.log.info("Connecting to PlatformIO Remote Development Cloud") # pylint: disable=protected-access proto, options = endpoints._parse(__pioremote_endpoint__) diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index 66c1069051..cafbbd1f30 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -33,14 +33,15 @@ from platformio.project.exception import NotPlatformIOProjectError -@click.group("remote", short_help="PIO Remote") +@click.group("remote", short_help="Remote development") @click.option("-a", "--agent", multiple=True) @click.pass_context def cli(ctx, agent): if PY2: raise exception.UserSideException( - "PIO Remote requires Python 3.5 or above. \nPlease install the latest " - "Python 3 and reinstall PlatformIO Core using installation script:\n" + "PlatformIO Remote Development requires Python 3.5 or above. \n" + "Please install the latest Python 3 and reinstall PlatformIO Core using " + "installation script:\n" "https://docs.platformio.org/page/core/installation.html" ) ctx.obj = agent diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index c21427236d..00e129af56 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -36,7 +36,7 @@ DEFAULT_JOB_NUMS = 1 -@click.command("run", short_help="Process project environments") +@click.command("run", short_help="Run project targets (build, upload, clean, etc.)") @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") diff --git a/platformio/commands/settings.py b/platformio/commands/settings.py index 7f03f81bdf..695d902034 100644 --- a/platformio/commands/settings.py +++ b/platformio/commands/settings.py @@ -27,7 +27,7 @@ def format_value(raw): return str(raw) -@click.group(short_help="Manage PlatformIO settings") +@click.group(short_help="Manage system settings") def cli(): pass diff --git a/platformio/commands/team.py b/platformio/commands/team.py index 5461cabd4e..7c1e863816 100644 --- a/platformio/commands/team.py +++ b/platformio/commands/team.py @@ -50,7 +50,7 @@ def validate_teamname(value): return value -@click.group("team", short_help="Manage Teams") +@click.group("team", short_help="Manage organization teams") def cli(): pass @@ -119,7 +119,9 @@ def team_list(orgname, json_output): callback=lambda _, __, value: validate_orgname_teamname(value), ) @click.option( - "--name", callback=lambda _, __, value: validate_teamname(value), + "--name", + callback=lambda _, __, value: validate_teamname(value), + help="A new team name", ) @click.option("--description",) def team_update(orgname_teamname, **kwargs): @@ -189,8 +191,8 @@ def team_add_member(orgname_teamname, username): metavar="ORGNAME:TEAMNAME", callback=lambda _, __, value: validate_orgname_teamname(value), ) -@click.argument("username",) -def org_remove_owner(orgname_teamname, username): +@click.argument("username") +def team_remove_owner(orgname_teamname, username): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() client.remove_team_member(orgname, teamname, username) diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index b57b1d5918..13104bb2f9 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -28,7 +28,7 @@ from platformio.project.config import ProjectConfig -@click.command("test", short_help="Unit Testing") +@click.command("test", short_help="Unit testing") @click.option("--environment", "-e", multiple=True, metavar="") @click.option( "--filter", diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 2d01b1559f..a11217e94a 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -26,7 +26,7 @@ def get_core_package_dir(name): if name not in __core_packages__: - raise exception.PlatformioException("Please upgrade PIO Core") + raise exception.PlatformioException("Please upgrade PlatformIO Core") pm = ToolPackageManager() spec = PackageSpec( owner="platformio", name=name, requirements=__core_packages__[name] diff --git a/platformio/project/options.py b/platformio/project/options.py index 3f0cf76c30..b5eaf337d2 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -245,7 +245,7 @@ def ConfigEnvOption(*args, **kwargs): group="directory", name="test_dir", description=( - "A location where PIO Unit Testing engine looks for " + "A location where PlatformIO Unit Testing engine looks for " "test source files" ), sysenvvar="PLATFORMIO_TEST_DIR", @@ -262,8 +262,8 @@ def ConfigEnvOption(*args, **kwargs): group="directory", name="shared_dir", description=( - "A location which PIO Remote uses to synchronize extra files " - "between remote machines" + "A location which PlatformIO Remote Development service uses to " + "synchronize extra files between remote machines" ), sysenvvar="PLATFORMIO_SHARED_DIR", default=os.path.join("$PROJECT_DIR", "shared"), From cdbb8379489675f2bb103972b8ecca9021ec060b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 18:45:52 +0300 Subject: [PATCH 210/223] Minor fixes --- HISTORY.rst | 2 +- docs | 2 +- platformio/telemetry.py | 5 ++++- platformio/util.py | 1 - 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2d37cb6bc9..78a67f267f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -31,7 +31,7 @@ PlatformIO Core 5 * Integration with the new **Account Management System** - - `Manage organizations and owners of an organization `__ + - `Manage organizations `__ - `Manage teams and team memberships `__ * New **Package Management System** diff --git a/docs b/docs index 7def3c5008..1a4a4bf127 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7def3c5008c99aae8f24984556e2a057195224e9 +Subproject commit 1a4a4bf127ec0651f7ca299e031379ea96734139 diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 4c5a67066b..68392f98c3 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -145,13 +145,16 @@ def _first_arg_from_list(args_, list_): cmd_path = args[:1] if args[0] in ( + "access", "account", "device", - "platform", + "org", "package", + "platform", "project", "settings", "system", + "team" ): cmd_path = args[:2] if args[0] == "lib" and len(args) > 1: diff --git a/platformio/util.py b/platformio/util.py index f950a36485..6da4708b83 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -29,7 +29,6 @@ from platformio import __version__, exception, proc from platformio.compat import PY2, WINDOWS from platformio.fs import cd, load_json # pylint: disable=unused-import -from platformio.package.version import pepver_to_semver # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import From 9f2c134e44903ffc8297f13ce49d7076bd9ae7af Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 21:24:48 +0300 Subject: [PATCH 211/223] Do not detach a new package even if it comes from external source --- platformio/package/manager/_install.py | 27 +++++++++++++------------- platformio/telemetry.py | 2 +- tests/commands/test_lib_complex.py | 17 ++++++++-------- tests/package/test_manager.py | 6 ++++++ 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 1bafcf8c32..1a83d65b4d 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -180,20 +180,19 @@ def _install_tmp_pkg(self, tmp_pkg): dst_pkg = PackageItem( os.path.join(self.package_dir, tmp_pkg.metadata.spec.name) ) - elif dst_pkg.metadata and dst_pkg.metadata.spec.external: - if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: - action = "detach-existing" - elif tmp_pkg.metadata.spec.external: - action = "detach-new" - elif dst_pkg.metadata and ( - dst_pkg.metadata.version != tmp_pkg.metadata.version - or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner - ): - action = ( - "detach-existing" - if tmp_pkg.metadata.version > dst_pkg.metadata.version - else "detach-new" - ) + elif dst_pkg.metadata: + if dst_pkg.metadata.spec.external: + if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: + action = "detach-existing" + elif ( + dst_pkg.metadata.version != tmp_pkg.metadata.version + or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner + ): + action = ( + "detach-existing" + if tmp_pkg.metadata.version > dst_pkg.metadata.version + else "detach-new" + ) def _cleanup_dir(path): if os.path.isdir(path): diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 68392f98c3..3fbcb74f5d 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -154,7 +154,7 @@ def _first_arg_from_list(args_, list_): "project", "settings", "system", - "team" + "team", ): cmd_path = args[:2] if args[0] == "lib" and len(args) > 1: diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index dfb853f472..442edee35f 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -98,9 +98,10 @@ def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ "ArduinoJson", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "SomeLib", "OneWire", - "ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8", + "ESP32WebServer", ] assert set(items1) >= set(items2) @@ -122,11 +123,11 @@ def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_c validate_cliresult(result) items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] items2 = [ - "PJON@src-1204e8bbd80de05e54e171b3a07bcc3f", + "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377", - "PubSubClient@src-98ec699a461a31615982e5adaaefadda", + "platformio-libmirror", + "PubSubClient", ] assert set(items1) >= set(items2) @@ -293,13 +294,13 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "AsyncMqttClient", "AsyncTCP", - "ESP32WebServer@src-a1a3c75631882b35702e71966ea694e8", + "ESP32WebServer", "ESPAsyncTCP", "NeoPixelBus", - "PJON@src-1204e8bbd80de05e54e171b3a07bcc3f", + "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d", - "platformio-libmirror@src-b7e674cad84244c61b436fcea8f78377", - "PubSubClient@src-98ec699a461a31615982e5adaaefadda", + "platformio-libmirror", + "PubSubClient", "SomeLib", ] assert set(items1) == set(items2) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index f5939f156e..299b0020ca 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -186,6 +186,12 @@ def test_install_from_url(isolated_pio_core, tmpdir_factory): pkg = lm.install_from_url("file://%s" % src_dir, spec) assert str(pkg.metadata.version) == "5.2.7" + # check package folder names + lm.memcache_reset() + assert ["local-lib-dir", "manifest-lib-name", "wifilib"] == [ + os.path.basename(pkg.path) for pkg in lm.get_installed() + ] + def test_install_from_registry(isolated_pio_core, tmpdir_factory): # Libraries From 5dee0a31e6f605e0ad1362256c9cf1f8d4ba5c5f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 21:40:17 +0300 Subject: [PATCH 212/223] Do not test for package owner if resource is external --- platformio/package/manager/base.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index cf359d1302..6cb609a3c8 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -244,10 +244,6 @@ def test_pkg_spec(pkg, spec): if spec.id and spec.id != pkg.metadata.spec.id: return False - # "owner" mismatch - if spec.owner and not ci_strings_are_equal(spec.owner, pkg.metadata.spec.owner): - return False - # external "URL" mismatch if spec.external: # local folder mismatch @@ -259,6 +255,12 @@ def test_pkg_spec(pkg, spec): if spec.url != pkg.metadata.spec.url: return False + # "owner" mismatch + elif spec.owner and not ci_strings_are_equal( + spec.owner, pkg.metadata.spec.owner + ): + return False + # "name" mismatch elif not spec.id and not ci_strings_are_equal(spec.name, pkg.metadata.name): return False From be487019f561b545cd36f6e2a5c3c5e6c130d6a3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 21:54:47 +0300 Subject: [PATCH 213/223] Fix a broken handling multi-configuration project // Resolve #3615 --- platformio/commands/project.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 70660fa26a..f861b5b199 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -180,7 +180,10 @@ def project_init( ) if ide: - config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) + with fs.cd(project_dir): + config = ProjectConfig.get_instance( + os.path.join(project_dir, "platformio.ini") + ) config.validate() pg = ProjectGenerator( config, environment or get_best_envname(config, board), ide From 7a49a741355e84b1e2685b204b2959c01c27d86a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Aug 2020 21:55:55 +0300 Subject: [PATCH 214/223] Bump version to 5.0.0b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 82133bd490..f5d915ff15 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "0b2") +VERSION = (5, 0, "0b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 2edd7ae649c6f234db7ecf8306cb187e5b6188eb Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 31 Aug 2020 15:40:25 +0300 Subject: [PATCH 215/223] Update PVS-Studio to the latest v7.09 --- HISTORY.rst | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 78a67f267f..6148e40f27 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -81,7 +81,7 @@ PlatformIO Core 5 - Updated analysis tools: * ``Cppcheck v2.1`` with a new "soundy" analysis option and improved code parser - * ``PVS-Studio v7.08`` with a new file list analysis mode and an extended list of diagnostic rules + * ``PVS-Studio v7.09`` with a new file list analysis mode and an extended list of analysis diagnostics - Added Cppcheck package for ARM-based single-board computers (`issue #3559 `_) - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 `_) diff --git a/platformio/__init__.py b/platformio/__init__.py index f5d915ff15..e940a36427 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -53,7 +53,7 @@ "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", "tool-cppcheck": "~1.210.0", "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.8.0", + "tool-pvs-studio": "~7.9.0", } __check_internet_hosts__ = [ From 5cc21511ad2a9a8de560f0bb8e32bf45b3f2fd8c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Sep 2020 16:07:16 +0300 Subject: [PATCH 216/223] Show owner name for packages --- platformio/commands/lib/command.py | 7 ++--- platformio/commands/lib/helpers.py | 5 +++- platformio/commands/platform.py | 8 +++--- platformio/package/manager/_legacy.py | 2 ++ platformio/project/config.py | 6 +++++ tests/commands/test_lib.py | 37 ++++++++++++++++++++++----- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index d871c91729..23a5b46e00 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -450,12 +450,13 @@ def lib_show(library, json_output): if json_output: return click.echo(dump_json_to_unicode(lib)) - click.secho(lib["name"], fg="cyan") - click.echo("=" * len(lib["name"])) - click.secho("#ID: %d" % lib["id"], bold=True) + title = "{ownername}/{name}".format(**lib) + click.secho(title, fg="cyan") + click.echo("=" * len(title)) click.echo(lib["description"]) click.echo() + click.secho("ID: %d" % lib["id"]) click.echo( "Version: %s, released %s" % ( diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py index a5b0e260d8..7a156e0f0a 100644 --- a/platformio/commands/lib/helpers.py +++ b/platformio/commands/lib/helpers.py @@ -87,5 +87,8 @@ def save_project_libdeps(project_dir, specs, environments=None, action="add"): pass if action == "add": lib_deps.extend(spec.as_dependency() for spec in specs) - config.set("env:" + env, "lib_deps", lib_deps) + if lib_deps: + config.set("env:" + env, "lib_deps", lib_deps) + elif config.has_option("env:" + env, "lib_deps"): + config.remove_option("env:" + env, "lib_deps") config.save() diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 588e7ccc39..054a7a1217 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -140,6 +140,7 @@ def _get_registry_platform_data( # pylint: disable=unused-argument return None data = dict( + ownername=_data.get("ownername"), name=_data["name"], title=_data["title"], description=_data["description"], @@ -242,12 +243,11 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches if json_output: return click.echo(dump_json_to_unicode(data)) + dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"] click.echo( - "{name} ~ {title}".format( - name=click.style(data["name"], fg="cyan"), title=data["title"] - ) + "{dep} ~ {title}".format(dep=click.style(dep, fg="cyan"), title=data["title"]) ) - click.echo("=" * (3 + len(data["name"] + data["title"]))) + click.echo("=" * (3 + len(dep + data["title"]))) click.echo(data["description"]) click.echo() if "version" in data: diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py index 95f628d010..5c35ebeb12 100644 --- a/platformio/package/manager/_legacy.py +++ b/platformio/package/manager/_legacy.py @@ -53,6 +53,8 @@ def legacy_load_manifest(self, pkg): if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external: manifest["__src_url"] = pkg.metadata.spec.url manifest["version"] = str(pkg.metadata.version) + if pkg.metadata and pkg.metadata.spec.owner: + manifest["ownername"] = pkg.metadata.spec.owner return manifest def legacy_get_installed(self): diff --git a/platformio/project/config.py b/platformio/project/config.py index 786f080aab..2d841b396e 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -358,6 +358,12 @@ def validate(self, envs=None, silent=False): click.secho("Warning! %s" % warning, fg="yellow") return True + def remove_option(self, section, option): + return self._parser.remove_option(section, option) + + def remove_section(self, section): + return self._parser.remove_section(section) + class ProjectConfigDirsMixin(object): def _get_core_dir(self, exists=False): diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 332161e1f0..b077f63bea 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -44,16 +44,21 @@ def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_fa ArduinoJson @ 5.10.1 """ ) - result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "install", "64"]) + result = clirunner.invoke( + cmd_lib, + ["-d", str(project_dir), "install", "64", "knolleary/PubSubClient@~2.7"], + ) validate_cliresult(result) aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson") config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) assert config.get("env:one", "lib_deps") == [ - "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"] + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", ] assert config.get("env:two", "lib_deps") == [ "CustomLib", "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", ] # ensure "build" version without NPM spec @@ -68,6 +73,7 @@ def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_fa config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) assert config.get("env:one", "lib_deps") == [ "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], ] @@ -85,23 +91,41 @@ def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_fa ) validate_cliresult(result) config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) - assert len(config.get("env:one", "lib_deps")) == 3 - assert config.get("env:one", "lib_deps")[2] == ( + assert len(config.get("env:one", "lib_deps")) == 4 + assert config.get("env:one", "lib_deps")[3] == ( "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3" ) # test uninstalling + # from all envs result = clirunner.invoke( cmd_lib, ["-d", str(project_dir), "uninstall", "ArduinoJson"] ) validate_cliresult(result) + # from "one" env + result = clirunner.invoke( + cmd_lib, + [ + "-d", + str(project_dir), + "-e", + "one", + "uninstall", + "knolleary/PubSubClient@~2.7", + ], + ) + validate_cliresult(result) config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) assert len(config.get("env:one", "lib_deps")) == 2 - assert len(config.get("env:two", "lib_deps")) == 1 + assert len(config.get("env:two", "lib_deps")) == 2 assert config.get("env:one", "lib_deps") == [ "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", ] + assert config.get("env:two", "lib_deps") == [ + "CustomLib", + "knolleary/PubSubClient@~2.7", + ] # test list result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"]) @@ -122,7 +146,8 @@ def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_fa item for item in data["one"] if item["name"] == "AsyncMqttClient-esphome" ) ame_vcs = VCSClientFactory.new(ame_lib["__pkg_dir"], ame_lib["__src_url"]) - assert data["two"] == [] + assert len(data["two"]) == 1 + assert data["two"][0]["name"] == "PubSubClient" assert "__pkg_dir" in data["one"][0] assert ( ame_lib["__src_url"] From 44c2b65372804866fa1946a4b6fb784d86c32cb6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Sep 2020 17:31:32 +0300 Subject: [PATCH 217/223] Show ignored project environments only in the verbose mode // Resolve #3641 --- HISTORY.rst | 1 + platformio/commands/run/command.py | 6 ++++-- tests/commands/test_lib.py | 34 ++++++++++++++++++------------ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6148e40f27..286890f965 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -90,6 +90,7 @@ PlatformIO Core 5 - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) + - Show ignored project environments only in the verbose mode (`issue #3641 `_) - Do not escape compiler arguments in VSCode template on Windows. .. _release_notes_4: diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 00e129af56..db4b412175 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -147,7 +147,7 @@ def cli( command_failed = any(r.get("succeeded") is False for r in results) if not is_test_running and (command_failed or not silent) and len(results) > 1: - print_processing_summary(results) + print_processing_summary(results, verbose) if command_failed: raise exception.ReturnErrorCode(1) @@ -220,7 +220,7 @@ def print_processing_footer(result): ) -def print_processing_summary(results): +def print_processing_summary(results, verbose=False): tabular_data = [] succeeded_nums = 0 failed_nums = 0 @@ -232,6 +232,8 @@ def print_processing_summary(results): failed_nums += 1 status_str = click.style("FAILED", fg="red") elif result.get("succeeded") is None: + if not verbose: + continue status_str = "IGNORED" else: succeeded_nums += 1 diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index b077f63bea..b25418419d 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -51,15 +51,19 @@ def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_fa validate_cliresult(result) aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson") config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) - assert config.get("env:one", "lib_deps") == [ - "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], - "knolleary/PubSubClient@~2.7", - ] - assert config.get("env:two", "lib_deps") == [ - "CustomLib", - "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], - "knolleary/PubSubClient@~2.7", - ] + assert sorted(config.get("env:one", "lib_deps")) == sorted( + [ + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + ] + ) + assert sorted(config.get("env:two", "lib_deps")) == sorted( + [ + "CustomLib", + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + ] + ) # ensure "build" version without NPM spec result = clirunner.invoke( @@ -71,11 +75,13 @@ def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_fa PackageType.LIBRARY, "mbed-sam-grove", "LinkedList" ) config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) - assert config.get("env:one", "lib_deps") == [ - "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], - "knolleary/PubSubClient@~2.7", - "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], - ] + assert sorted(config.get("env:one", "lib_deps")) == sorted( + [ + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], + ] + ) # check external package via Git repo result = clirunner.invoke( From 6e5198f3739f536935916f700bd09a9f96ad0278 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Sep 2020 18:49:00 +0300 Subject: [PATCH 218/223] Minor improvements --- platformio/builder/main.py | 2 +- platformio/package/exception.py | 2 +- platformio/package/manager/_install.py | 2 +- platformio/package/manager/_uninstall.py | 8 +++++--- tests/commands/test_platform.py | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e73f686927..1547cf99aa 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -160,7 +160,7 @@ env.SConscriptChdir(0) env.SConsignFile( - join("$BUILD_DIR", ".sconsign%d%d.db" % (sys.version_info[0], sys.version_info[1])) + join("$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1])) ) for item in env.GetExtraScripts("pre"): diff --git a/platformio/package/exception.py b/platformio/package/exception.py index 0f34592f7e..5d63649e8d 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -55,7 +55,7 @@ class MissingPackageManifestError(ManifestException): class UnknownPackageError(UserSideException): MESSAGE = ( - "Could not find a package with '{0}' requirements for your system '%s'" + "Could not find the package with '{0}' requirements for your system '%s'" % util.get_systype() ) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index 1a83d65b4d..9d82d6fefd 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -106,7 +106,7 @@ def _install( # pylint: disable=too-many-arguments if not silent: self.print_message( - "{name} @ {version} has been successfully installed!".format( + "{name} @ {version} has been installed!".format( **pkg.metadata.as_dict() ), fg="green", diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py index 322eced6dc..68f7a30015 100644 --- a/platformio/package/manager/_uninstall.py +++ b/platformio/package/manager/_uninstall.py @@ -37,9 +37,8 @@ def _uninstall(self, spec, silent=False, skip_dependencies=False): if not silent: self.print_message( - "Removing %s @ %s: \t" + "Removing %s @ %s" % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), - nl=False, ) # firstly, remove dependencies @@ -68,7 +67,10 @@ def _uninstall(self, spec, silent=False, skip_dependencies=False): self.memcache_reset() if not silent: - click.echo("[%s]" % click.style("OK", fg="green")) + self.print_message( + "{name} @ {version} has been removed!".format(**pkg.metadata.as_dict()), + fg="green", + ) return pkg diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 39afbeb5e0..cfb7fe3131 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -119,7 +119,7 @@ def test_update_check(clirunner, validate_cliresult, isolated_pio_core): def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cli_platform.platform_update) validate_cliresult(result) - assert "Removing atmelavr @ 2.0.0:" in result.output + assert "Removing atmelavr @ 2.0.0" in result.output assert "Platform Manager: Installing platformio/atmelavr @" in result.output assert len(isolated_pio_core.join("packages").listdir()) == 2 From c8ea64edabcc0408ac7a5978f08eaf73fa54afe1 Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Wed, 2 Sep 2020 18:13:20 +0200 Subject: [PATCH 219/223] Fix link to FAQ sections (#3642) * Fix link to FAQ sections Use consistently the same host and url and fix one unmatched anchor. * Update HISTORY.rst Co-authored-by: Ivan Kravets --- platformio/exception.py | 8 ++++---- platformio/maintenance.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/platformio/exception.py b/platformio/exception.py index 8ae549bc3d..ef1d3bab5a 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -55,8 +55,8 @@ class InvalidUdevRules(PlatformioException): class MissedUdevRules(InvalidUdevRules): MESSAGE = ( - "Warning! Please install `99-platformio-udev.rules`. \nMode details: " - "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules" + "Warning! Please install `99-platformio-udev.rules`. \nMore details: " + "https://docs.platformio.org/page/faq.html#platformio-udev-rules" ) @@ -64,8 +64,8 @@ class OutdatedUdevRules(InvalidUdevRules): MESSAGE = ( "Warning! Your `{0}` are outdated. Please update or reinstall them." - "\n Mode details: https://docs.platformio.org" - "/en/latest/faq.html#platformio-udev-rules" + "\nMore details: " + "https://docs.platformio.org/page/faq.html#platformio-udev-rules" ) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 1900db4930..e038bcc097 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -148,7 +148,7 @@ def after_upgrade(ctx): click.secho("Please remove multiple PIO Cores from a system:", fg="yellow") click.secho( "https://docs.platformio.org/page/faq.html" - "#multiple-pio-cores-in-a-system", + "#multiple-platformio-cores-in-a-system", fg="cyan", ) click.secho("*" * terminal_width, fg="yellow") From fe4112a2a3c4b07c0cd0506d43c8c7dc0ff7d538 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Sep 2020 20:36:56 +0300 Subject: [PATCH 220/223] Follow SemVer complaint version constraints when checking library updates // Resolve #1281 --- HISTORY.rst | 3 ++- platformio/commands/lib/command.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 286890f965..fe9e137fd4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -37,8 +37,9 @@ PlatformIO Core 5 * New **Package Management System** - Integrated PlatformIO Core with the new PlatformIO Registry - - Strict dependency declaration using owner name (resolves name conflicts) (`issue #1824 `_) + - Support for owner-based dependency declaration (resolves name conflicts) (`issue #1824 `_) - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) + - Follow SemVer complaint version constraints when checking library updates `issue #1281 `_) - Dropped support for "packageRepositories" section in "platform.json" manifest (please publish packages directly to the registry) * **Build System** diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 23a5b46e00..543e439ccf 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -254,8 +254,9 @@ def lib_update( # pylint: disable=too-many-arguments for storage_dir in storage_dirs: if not json_output: print_storage_header(storage_dirs, storage_dir) + lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, []) lm = LibraryPackageManager(storage_dir) - _libraries = libraries or lm.get_installed() + _libraries = libraries or lib_deps or lm.get_installed() if only_check and json_output: result = [] @@ -286,9 +287,13 @@ def lib_update( # pylint: disable=too-many-arguments to_spec = ( None if isinstance(library, PackageItem) else PackageSpec(library) ) - lm.update( - library, to_spec=to_spec, only_check=only_check, silent=silent - ) + try: + lm.update( + library, to_spec=to_spec, only_check=only_check, silent=silent + ) + except UnknownPackageError as e: + if library not in lib_deps: + raise e if json_output: return click.echo( From 083edc4c76cc0e54689258743927e8e6a3d8d4cf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 2 Sep 2020 20:52:11 +0300 Subject: [PATCH 221/223] Refactor to os.path --- platformio/builder/tools/piolib.py | 115 +++++++++++++++-------------- 1 file changed, 61 insertions(+), 54 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 35d1462e1a..f6b9824be6 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -23,7 +23,6 @@ import os import re import sys -from os.path import basename, commonprefix, isdir, isfile, join, realpath, sep import click import SCons.Scanner # pylint: disable=import-error @@ -49,7 +48,7 @@ class LibBuilderFactory(object): @staticmethod def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): clsname = "UnknownLibBuilder" - if isfile(join(path, "library.json")): + if os.path.isfile(os.path.join(path, "library.json")): clsname = "PlatformIOLibBuilder" else: used_frameworks = LibBuilderFactory.get_used_frameworks(env, path) @@ -66,12 +65,12 @@ def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): @staticmethod def get_used_frameworks(env, path): if any( - isfile(join(path, fname)) + os.path.isfile(os.path.join(path, fname)) for fname in ("library.properties", "keywords.txt") ): return ["arduino"] - if isfile(join(path, "module.json")): + if os.path.isfile(os.path.join(path, "module.json")): return ["mbed"] include_re = re.compile( @@ -87,7 +86,7 @@ def get_used_frameworks(env, path): fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT ): continue - with io.open(join(root, fname), errors="ignore") as fp: + with io.open(os.path.join(root, fname), errors="ignore") as fp: content = fp.read() if not content: continue @@ -114,7 +113,7 @@ class LibBuilderBase(object): def __init__(self, env, path, manifest=None, verbose=False): self.env = env.Clone() self.envorigin = env.Clone() - self.path = realpath(env.subst(path)) + self.path = os.path.realpath(env.subst(path)) self.verbose = verbose try: @@ -148,11 +147,11 @@ def __contains__(self, path): p2 = p2.lower() if p1 == p2: return True - return commonprefix((p1 + sep, p2)) == p1 + sep + return os.path.commonprefix((p1 + os.path.sep, p2)) == p1 + os.path.sep @property def name(self): - return self._manifest.get("name", basename(self.path)) + return self._manifest.get("name", os.path.basename(self.path)) @property def version(self): @@ -173,13 +172,19 @@ def src_filter(self): @property def include_dir(self): - if not all(isdir(join(self.path, d)) for d in ("include", "src")): + if not all( + os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src") + ): return None - return join(self.path, "include") + return os.path.join(self.path, "include") @property def src_dir(self): - return join(self.path, "src") if isdir(join(self.path, "src")) else self.path + return ( + os.path.join(self.path, "src") + if os.path.isdir(os.path.join(self.path, "src")) + else self.path + ) def get_include_dirs(self): items = [] @@ -192,7 +197,9 @@ def get_include_dirs(self): @property def build_dir(self): lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3] - return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) + return os.path.join( + "$BUILD_DIR", "lib%s" % lib_hash, os.path.basename(self.path) + ) @property def build_flags(self): @@ -271,7 +278,7 @@ def process_extra_options(self): if self.extra_script: self.env.SConscriptChdir(1) self.env.SConscript( - realpath(self.extra_script), + os.path.realpath(self.extra_script), exports={"env": self.env, "pio_lib_builder": self}, ) self.env.ProcessUnFlags(self.build_unflags) @@ -297,14 +304,14 @@ def process_dependencies(self): def get_search_files(self): items = [ - join(self.src_dir, item) + os.path.join(self.src_dir, item) for item in self.env.MatchSourceFiles(self.src_dir, self.src_filter) ] include_dir = self.include_dir if include_dir: items.extend( [ - join(include_dir, item) + os.path.join(include_dir, item) for item in self.env.MatchSourceFiles(include_dir) ] ) @@ -373,7 +380,7 @@ def _get_found_includes( # pylint: disable=too-many-branches continue _f_part = _h_path[: _h_path.rindex(".")] for ext in piotool.SRC_C_EXT + piotool.SRC_CXX_EXT: - if not isfile("%s.%s" % (_f_part, ext)): + if not os.path.isfile("%s.%s" % (_f_part, ext)): continue _c_path = self.env.File("%s.%s" % (_f_part, ext)) if _c_path not in result: @@ -467,23 +474,23 @@ class UnknownLibBuilder(LibBuilderBase): class ArduinoLibBuilder(LibBuilderBase): def load_manifest(self): - manifest_path = join(self.path, "library.properties") - if not isfile(manifest_path): + manifest_path = os.path.join(self.path, "library.properties") + if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) - if isdir(join(self.path, "src")): + if os.path.isdir(os.path.join(self.path, "src")): return include_dirs - if isdir(join(self.path, "utility")): - include_dirs.append(join(self.path, "utility")) + if os.path.isdir(os.path.join(self.path, "utility")): + include_dirs.append(os.path.join(self.path, "utility")) return include_dirs @property def src_filter(self): - src_dir = join(self.path, "src") - if isdir(src_dir): + src_dir = os.path.join(self.path, "src") + if os.path.isdir(src_dir): # pylint: disable=no-member src_filter = LibBuilderBase.src_filter.fget(self) for root, _, files in os.walk(src_dir, followlinks=True): @@ -495,20 +502,20 @@ def src_filter(self): if not found: continue rel_path = root.replace(src_dir, "") - if rel_path.startswith(sep): - rel_path = rel_path[1:] + sep + if rel_path.startswith(os.path.sep): + rel_path = rel_path[1:] + os.path.sep src_filter.append("-<%s*.[aA][sS][mM]>" % rel_path) return src_filter src_filter = [] - is_utility = isdir(join(self.path, "utility")) + is_utility = os.path.isdir(os.path.join(self.path, "utility")) for ext in piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT: # arduino ide ignores files with .asm or .ASM extensions if ext.lower() == "asm": continue src_filter.append("+<*.%s>" % ext) if is_utility: - src_filter.append("+" % (sep, ext)) + src_filter.append("+" % (os.path.sep, ext)) return src_filter @property @@ -541,21 +548,21 @@ def is_platforms_compatible(self, platforms): class MbedLibBuilder(LibBuilderBase): def load_manifest(self): - manifest_path = join(self.path, "module.json") - if not isfile(manifest_path): + manifest_path = os.path.join(self.path, "module.json") + if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() @property def include_dir(self): - if isdir(join(self.path, "include")): - return join(self.path, "include") + if os.path.isdir(os.path.join(self.path, "include")): + return os.path.join(self.path, "include") return None @property def src_dir(self): - if isdir(join(self.path, "source")): - return join(self.path, "source") + if os.path.isdir(os.path.join(self.path, "source")): + return os.path.join(self.path, "source") return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member def get_include_dirs(self): @@ -565,13 +572,13 @@ def get_include_dirs(self): # library with module.json for p in self._manifest.get("extraIncludes", []): - include_dirs.append(join(self.path, p)) + include_dirs.append(os.path.join(self.path, p)) # old mbed library without manifest, add to CPPPATH all folders if not self._manifest: for root, _, __ in os.walk(self.path): part = root.replace(self.path, "").lower() - if any(s in part for s in ("%s." % sep, "test", "example")): + if any(s in part for s in ("%s." % os.path.sep, "test", "example")): continue if root not in include_dirs: include_dirs.append(root) @@ -587,7 +594,7 @@ def process_extra_options(self): def _process_mbed_lib_confs(self): mbed_lib_paths = [ - join(root, "mbed_lib.json") + os.path.join(root, "mbed_lib.json") for root, _, files in os.walk(self.path) if "mbed_lib.json" in files ] @@ -596,8 +603,8 @@ def _process_mbed_lib_confs(self): mbed_config_path = None for p in self.env.get("CPPPATH"): - mbed_config_path = join(self.env.subst(p), "mbed_config.h") - if isfile(mbed_config_path): + mbed_config_path = os.path.join(self.env.subst(p), "mbed_config.h") + if os.path.isfile(mbed_config_path): break mbed_config_path = None if not mbed_config_path: @@ -689,26 +696,26 @@ def _mbed_conf_append_macros(self, mbed_config_path, macros): class PlatformIOLibBuilder(LibBuilderBase): def load_manifest(self): - manifest_path = join(self.path, "library.json") - if not isfile(manifest_path): + manifest_path = os.path.join(self.path, "library.json") + if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() def _has_arduino_manifest(self): - return isfile(join(self.path, "library.properties")) + return os.path.isfile(os.path.join(self.path, "library.properties")) @property def include_dir(self): if "includeDir" in self._manifest.get("build", {}): with fs.cd(self.path): - return realpath(self._manifest.get("build").get("includeDir")) + return os.path.realpath(self._manifest.get("build").get("includeDir")) return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): with fs.cd(self.path): - return realpath(self._manifest.get("build").get("srcDir")) + return os.path.realpath(self._manifest.get("build").get("srcDir")) return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member @property @@ -786,10 +793,10 @@ def get_include_dirs(self): if ( "build" not in self._manifest and self._has_arduino_manifest() - and not isdir(join(self.path, "src")) - and isdir(join(self.path, "utility")) + and not os.path.isdir(os.path.join(self.path, "src")) + and os.path.isdir(os.path.join(self.path, "utility")) ): - include_dirs.append(join(self.path, "utility")) + include_dirs.append(os.path.join(self.path, "utility")) for path in self.env.get("CPPPATH", []): if path not in self.envorigin.get("CPPPATH", []): @@ -808,7 +815,7 @@ def __init__(self, env, *args, **kwargs): @property def include_dir(self): include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") - return include_dir if isdir(include_dir) else None + return include_dir if os.path.isdir(include_dir) else None @property def src_dir(self): @@ -817,7 +824,7 @@ def src_dir(self): def get_include_dirs(self): include_dirs = [] project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") - if isdir(project_include_dir): + if os.path.isdir(project_include_dir): include_dirs.append(project_include_dir) for include_dir in LibBuilderBase.get_include_dirs(self): if include_dir not in include_dirs: @@ -831,7 +838,7 @@ def get_search_files(self): if "__test" in COMMAND_LINE_TARGETS: items.extend( [ - join("$PROJECT_TEST_DIR", item) + os.path.join("$PROJECT_TEST_DIR", item) for item in self.env.MatchSourceFiles( "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) @@ -884,7 +891,7 @@ def _is_builtin(spec): did_install = False lm = LibraryPackageManager( - self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) + self.env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) ) for spec in not_found_specs: try: @@ -975,12 +982,12 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches found_incompat = False for storage_dir in env.GetLibSourceDirs(): - storage_dir = realpath(storage_dir) - if not isdir(storage_dir): + storage_dir = os.path.realpath(storage_dir) + if not os.path.isdir(storage_dir): continue for item in sorted(os.listdir(storage_dir)): - lib_dir = join(storage_dir, item) - if item == "__cores__" or not isdir(lib_dir): + lib_dir = os.path.join(storage_dir, item) + if item == "__cores__" or not os.path.isdir(lib_dir): continue try: lb = LibBuilderFactory.new(env, lib_dir) From fec4569ada6c753ffe55c7a8b4449042cd0ffc3b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 3 Sep 2020 14:37:24 +0300 Subject: [PATCH 222/223] Docs: Update docs with new owner-based dependency form --- HISTORY.rst | 4 ++-- docs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fe9e137fd4..d262283306 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -81,8 +81,8 @@ PlatformIO Core 5 - Updated analysis tools: - * ``Cppcheck v2.1`` with a new "soundy" analysis option and improved code parser - * ``PVS-Studio v7.09`` with a new file list analysis mode and an extended list of analysis diagnostics + * `Cppcheck `__ v2.1 with a new "soundy" analysis option and improved code parser + * `PVS-Studio `__ v7.09 with a new file list analysis mode and an extended list of analysis diagnostics - Added Cppcheck package for ARM-based single-board computers (`issue #3559 `_) - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 `_) diff --git a/docs b/docs index 1a4a4bf127..31718e5365 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1a4a4bf127ec0651f7ca299e031379ea96734139 +Subproject commit 31718e5365e11839db35ff920f71067c9c1e092a From cf4b835b0c757a8c6d7d68b3606162857829e8b5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 3 Sep 2020 14:42:59 +0300 Subject: [PATCH 223/223] Bump version to 5.0.0 --- HISTORY.rst | 2 +- README.rst | 8 ++++---- docs | 2 +- platformio/__init__.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d262283306..c28e2f3967 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,7 +10,7 @@ PlatformIO Core 5 - `Migration guide from 4.x to 5.0 `__ -5.0.0 (2020-??-??) +5.0.0 (2020-09-03) ~~~~~~~~~~~~~~~~~~ * Integration with the new **PlatformIO Trusted Registry** diff --git a/README.rst b/README.rst index c4ab3d5f53..bdd2858b7f 100644 --- a/README.rst +++ b/README.rst @@ -66,10 +66,10 @@ Instruments Professional ------------ -* `PIO Check `_ -* `PIO Remote `_ -* `PIO Unified Debugger `_ -* `PIO Unit Testing `_ +* `Debugging `_ +* `Unit Testing `_ +* `Static Code Analysis `_ +* `Remote Development `_ Registry -------- diff --git a/docs b/docs index 31718e5365..03a83c996f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 31718e5365e11839db35ff920f71067c9c1e092a +Subproject commit 03a83c996f0c209ce0faaa2bcc285447a7780500 diff --git a/platformio/__init__.py b/platformio/__init__.py index e940a36427..74707d9cd2 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, "0b3") +VERSION = (5, 0, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -47,7 +47,7 @@ __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.2.3", + "contrib-piohome": "~3.3.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0",