diff --git a/HISTORY.rst b/HISTORY.rst index c28e2f3967..c1668e51a3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,11 +8,23 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** -- `Migration guide from 4.x to 5.0 `__ +5.0.1 (2020-09-10) +~~~~~~~~~~~~~~~~~~ + +- Added support for "owner" requirement when declaring ``dependencies`` using `library.json `__ +- Fixed an issue when using a custom git/ssh package with `platform_packages `__ option (`issue #3624 `_) +- Fixed an issue with "ImportError: cannot import name '_get_backend' from 'cryptography.hazmat.backends'" when using `Remote Development `__ on RaspberryPi device (`issue #3652 `_) +- Fixed an issue when `pio package unpublish `__ command crashes (`issue #3660 `_) +- Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 `_) +- Fixed an issue with incorrect value for C++ language standard in IDE projects when an in-progress language standard is used (`issue #3653 `_) +- Fixed an issue with "Invalid simple block (semantic_version)" from library dependency that refs to an external source (repository, ZIP/Tar archives) (`issue #3658 `_) +- Fixed an issue when can not remove update or remove external dev-platform using PlatformIO Home (`issue #3663 `_) 5.0.0 (2020-09-03) ~~~~~~~~~~~~~~~~~~ +Please check `Migration guide from 4.x to 5.0 `__. + * Integration with the new **PlatformIO Trusted Registry** - Enterprise-grade package storage with high availability (multi replicas) diff --git a/docs b/docs index 03a83c996f..9bbb02295a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 03a83c996f0c209ce0faaa2bcc285447a7780500 +Subproject commit 9bbb02295a7f5ce325f1ed90a8b549fb81ce3857 diff --git a/platformio/__init__.py b/platformio/__init__.py index 74707d9cd2..145ad0cff4 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 0, 0) +VERSION = (5, 0, 1) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/clients/account.py b/platformio/clients/account.py index 1c4b6755fa..cae863ada7 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -80,7 +80,9 @@ def login(self, username, password): ) data = self.fetch_json_data( - "post", "/v1/login", data={"username": username, "password": password}, + "post", + "/v1/login", + data={"username": username, "password": password}, ) app.set_state_item("account", data) return data @@ -108,7 +110,9 @@ def logout(self): self.delete_local_session() try: self.fetch_json_data( - "post", "/v1/logout", data={"refresh_token": refresh_token}, + "post", + "/v1/logout", + data={"refresh_token": refresh_token}, ) except AccountError: pass @@ -153,15 +157,26 @@ def auth_token(self, password, regenerate): ).get("auth_token") def forgot_password(self, username): - return self.fetch_json_data("post", "/v1/forgot", data={"username": username},) + return self.fetch_json_data( + "post", + "/v1/forgot", + data={"username": username}, + ) def get_profile(self): - return self.send_auth_request("get", "/v1/profile",) + return self.send_auth_request( + "get", + "/v1/profile", + ) def update_profile(self, profile, current_password): profile["current_password"] = current_password self.delete_local_state("summary") - response = self.send_auth_request("put", "/v1/profile", data=profile,) + response = self.send_auth_request( + "put", + "/v1/profile", + data=profile, + ) return response def get_account_info(self, offline=False): @@ -178,7 +193,10 @@ def get_account_info(self, offline=False): "username": account.get("username"), } } - result = self.send_auth_request("get", "/v1/summary",) + result = self.send_auth_request( + "get", + "/v1/summary", + ) account["summary"] = dict( profile=result.get("profile"), packages=result.get("packages"), @@ -203,7 +221,10 @@ def get_org(self, orgname): return self.send_auth_request("get", "/v1/orgs/%s" % orgname) def list_orgs(self): - return self.send_auth_request("get", "/v1/orgs",) + return self.send_auth_request( + "get", + "/v1/orgs", + ) def update_org(self, orgname, data): return self.send_auth_request( @@ -211,19 +232,29 @@ def update_org(self, orgname, data): ) def destroy_org(self, orgname): - return self.send_auth_request("delete", "/v1/orgs/%s" % orgname,) + return self.send_auth_request( + "delete", + "/v1/orgs/%s" % orgname, + ) def add_org_owner(self, orgname, username): return self.send_auth_request( - "post", "/v1/orgs/%s/owners" % orgname, data={"username": username}, + "post", + "/v1/orgs/%s/owners" % orgname, + data={"username": username}, ) def list_org_owners(self, orgname): - return self.send_auth_request("get", "/v1/orgs/%s/owners" % orgname,) + return self.send_auth_request( + "get", + "/v1/orgs/%s/owners" % orgname, + ) def remove_org_owner(self, orgname, username): return self.send_auth_request( - "delete", "/v1/orgs/%s/owners" % orgname, data={"username": username}, + "delete", + "/v1/orgs/%s/owners" % orgname, + data={"username": username}, ) def create_team(self, orgname, teamname, description): @@ -235,16 +266,21 @@ def create_team(self, orgname, teamname, description): def destroy_team(self, orgname, teamname): return self.send_auth_request( - "delete", "/v1/orgs/%s/teams/%s" % (orgname, teamname), + "delete", + "/v1/orgs/%s/teams/%s" % (orgname, teamname), ) def get_team(self, orgname, teamname): return self.send_auth_request( - "get", "/v1/orgs/%s/teams/%s" % (orgname, teamname), + "get", + "/v1/orgs/%s/teams/%s" % (orgname, teamname), ) def list_teams(self, orgname): - return self.send_auth_request("get", "/v1/orgs/%s/teams" % orgname,) + return self.send_auth_request( + "get", + "/v1/orgs/%s/teams" % orgname, + ) def update_team(self, orgname, teamname, data): return self.send_auth_request( diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index c8fbeeeadf..f8afdfab1b 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -70,12 +70,16 @@ def unpublish_package( # pylint: disable=redefined-builtin if version: path += "/" + version return self.send_auth_request( - "delete", path, params={"undo": 1 if undo else 0}, + "delete", + path, + params={"undo": 1 if undo else 0}, ) def update_resource(self, urn, private): return self.send_auth_request( - "put", "/v3/resources/%s" % urn, data={"private": int(private)}, + "put", + "/v3/resources/%s" % urn, + data={"private": int(private)}, ) def grant_access_for_resource(self, urn, client, level): @@ -87,7 +91,9 @@ def grant_access_for_resource(self, urn, client, level): def revoke_access_from_resource(self, urn, client): return self.send_auth_request( - "delete", "/v3/resources/%s/access" % urn, data={"client": client}, + "delete", + "/v3/resources/%s/access" % urn, + data={"client": client}, ) def list_resources(self, owner): diff --git a/platformio/commands/access.py b/platformio/commands/access.py index 8b65ba34a0..d9fb39700f 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -47,27 +47,31 @@ def validate_urn(value): @cli.command("public", short_help="Make resource public") @click.argument( - "urn", callback=lambda _, __, value: validate_urn(value), + "urn", + callback=lambda _, __, value: validate_urn(value), ) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_public(urn, urn_type): client = RegistryClient() client.update_resource(urn=urn, private=0) return click.secho( - "The resource %s has been successfully updated." % urn, fg="green", + "The resource %s has been successfully updated." % urn, + fg="green", ) @cli.command("private", short_help="Make resource private") @click.argument( - "urn", callback=lambda _, __, value: validate_urn(value), + "urn", + callback=lambda _, __, value: validate_urn(value), ) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_private(urn, urn_type): client = RegistryClient() client.update_resource(urn=urn, private=1) return click.secho( - "The resource %s has been successfully updated." % urn, fg="green", + "The resource %s has been successfully updated." % urn, + fg="green", ) @@ -79,14 +83,16 @@ def access_private(urn, urn_type): callback=lambda _, __, value: validate_client(value), ) @click.argument( - "urn", callback=lambda _, __, value: validate_urn(value), + "urn", + callback=lambda _, __, value: validate_urn(value), ) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_grant(level, client, urn, urn_type): reg_client = RegistryClient() reg_client.grant_access_for_resource(urn=urn, client=client, level=level) return click.secho( - "Access for resource %s has been granted for %s" % (urn, client), fg="green", + "Access for resource %s has been granted for %s" % (urn, client), + fg="green", ) @@ -97,14 +103,16 @@ def access_grant(level, client, urn, urn_type): callback=lambda _, __, value: validate_client(value), ) @click.argument( - "urn", callback=lambda _, __, value: validate_urn(value), + "urn", + callback=lambda _, __, value: validate_urn(value), ) @click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") def access_revoke(client, urn, urn_type): reg_client = RegistryClient() reg_client.revoke_access_from_resource(urn=urn, client=client) return click.secho( - "Access for resource %s has been revoked for %s" % (urn, client), fg="green", + "Access for resource %s has been revoked for %s" % (urn, client), + fg="green", ) diff --git a/platformio/commands/account.py b/platformio/commands/account.py index 88aab68b90..41af292297 100644 --- a/platformio/commands/account.py +++ b/platformio/commands/account.py @@ -192,7 +192,10 @@ def account_destroy(): client.logout() except AccountNotAuthorized: pass - return click.secho("User account has been destroyed.", fg="green",) + return click.secho( + "User account has been destroyed.", + fg="green", + ) @cli.command("show", short_help="PlatformIO Account information") diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index a66cb9961a..2e254742c7 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -203,7 +203,9 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches kwargs["port"] = ports[0]["port"] elif "platform" in project_options and "board" in project_options: board_hwids = device_helpers.get_board_hwids( - kwargs["project_dir"], platform, project_options["board"], + kwargs["project_dir"], + platform, + project_options["board"], ) for item in ports: for hwid in board_hwids: diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 543e439ccf..d49e6cb6b1 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -22,11 +22,7 @@ from platformio import exception, fs, util from platformio.commands import PlatformioCLI -from platformio.commands.lib.helpers import ( - get_builtin_libs, - is_builtin_lib, - save_project_libdeps, -) +from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps from platformio.compat import dump_json_to_unicode from platformio.package.exception import NotGlobalLibDir, UnknownPackageError from platformio.package.manager.library import LibraryPackageManager @@ -164,15 +160,8 @@ def lib_install( # pylint: disable=too-many-arguments,unused-argument } elif storage_dir in storage_libdeps: - builtin_lib_storages = None for library in storage_libdeps[storage_dir]: - try: - lm.install(library, silent=silent, force=force) - except UnknownPackageError as e: - if builtin_lib_storages is None: - builtin_lib_storages = get_builtin_libs() - if not silent or not is_builtin_lib(builtin_lib_storages, library): - click.secho("Warning! %s" % e, fg="yellow") + lm.install(library, silent=silent, force=force) if save and installed_pkgs: _save_deps(ctx, installed_pkgs) diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py index 7a156e0f0a..732604a3f9 100644 --- a/platformio/commands/lib/helpers.py +++ b/platformio/commands/lib/helpers.py @@ -45,10 +45,11 @@ def get_builtin_libs(storage_names=None): return items -def is_builtin_lib(storages, name): - for storage in storages or []: - if any(lib.get("name") == name for lib in storage["items"]): - return True +def is_builtin_lib(name, storages=None): + for storage in storages or get_builtin_libs(): + for lib in storage["items"]: + if lib.get("name") == name: + return True return False diff --git a/platformio/commands/org.py b/platformio/commands/org.py index ac13d13f74..7f003cb0e0 100644 --- a/platformio/commands/org.py +++ b/platformio/commands/org.py @@ -34,17 +34,21 @@ def validate_orgname(value): @cli.command("create", short_help="Create a new organization") @click.argument( - "orgname", callback=lambda _, __, value: validate_orgname(value), + "orgname", + callback=lambda _, __, value: validate_orgname(value), ) @click.option( "--email", callback=lambda _, __, value: validate_email(value) if value else value ) -@click.option("--displayname",) +@click.option( + "--displayname", +) def org_create(orgname, email, displayname): client = AccountClient() client.create_org(orgname, email, displayname) return click.secho( - "The organization `%s` has been successfully created." % orgname, fg="green", + "The organization `%s` has been successfully created." % orgname, + fg="green", ) @@ -121,12 +125,19 @@ def account_destroy(orgname): abort=True, ) client.destroy_org(orgname) - return click.secho("Organization `%s` has been destroyed." % orgname, fg="green",) + return click.secho( + "Organization `%s` has been destroyed." % orgname, + fg="green", + ) @cli.command("add", short_help="Add a new owner to organization") -@click.argument("orgname",) -@click.argument("username",) +@click.argument( + "orgname", +) +@click.argument( + "username", +) def org_add_owner(orgname, username): client = AccountClient() client.add_org_owner(orgname, username) @@ -138,8 +149,12 @@ def org_add_owner(orgname, username): @cli.command("remove", short_help="Remove an owner from organization") -@click.argument("orgname",) -@click.argument("username",) +@click.argument( + "orgname", +) +@click.argument( + "username", +) def org_remove_owner(orgname, username): client = AccountClient() client.remove_org_owner(orgname, username) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 88f6c0d3ed..f62362ff57 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -107,7 +107,7 @@ def package_unpublish(package, type, undo): # pylint: disable=redefined-builtin type=type, name=spec.name, owner=spec.owner, - version=spec.requirements, + version=str(spec.requirements), undo=undo, ) click.secho(response.get("message"), fg="green") diff --git a/platformio/commands/remote/factory/client.py b/platformio/commands/remote/factory/client.py index 2b47ab01a6..712449d527 100644 --- a/platformio/commands/remote/factory/client.py +++ b/platformio/commands/remote/factory/client.py @@ -45,7 +45,10 @@ def clientConnectionMade(self, broker): return d d = self.login( - credentials.UsernamePassword(auth_token.encode(), get_host_id().encode(),), + credentials.UsernamePassword( + auth_token.encode(), + get_host_id().encode(), + ), client=self.remote_client, ) d.addCallback(self.remote_client.cb_client_authorization_made) diff --git a/platformio/commands/team.py b/platformio/commands/team.py index 7c1e863816..6733a4201a 100644 --- a/platformio/commands/team.py +++ b/platformio/commands/team.py @@ -63,13 +63,16 @@ def cli(): value, teamname_validate=True ), ) -@click.option("--description",) +@click.option( + "--description", +) def team_create(orgname_teamname, description): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() client.create_team(orgname, teamname, description) return click.secho( - "The team %s has been successfully created." % teamname, fg="green", + "The team %s has been successfully created." % teamname, + fg="green", ) @@ -123,7 +126,9 @@ def team_list(orgname, json_output): callback=lambda _, __, value: validate_teamname(value), help="A new team name", ) -@click.option("--description",) +@click.option( + "--description", +) def team_update(orgname_teamname, **kwargs): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() @@ -142,7 +147,8 @@ def team_update(orgname_teamname, **kwargs): new_team.update({key: value for key, value in kwargs.items() if value}) client.update_team(orgname, teamname, new_team) return click.secho( - "The team %s has been successfully updated." % teamname, fg="green", + "The team %s has been successfully updated." % teamname, + fg="green", ) @@ -163,7 +169,8 @@ def team_destroy(orgname_teamname): client = AccountClient() client.destroy_team(orgname, teamname) return click.secho( - "The team %s has been successfully destroyed." % teamname, fg="green", + "The team %s has been successfully destroyed." % teamname, + fg="green", ) @@ -173,7 +180,9 @@ def team_destroy(orgname_teamname): metavar="ORGNAME:TEAMNAME", callback=lambda _, __, value: validate_orgname_teamname(value), ) -@click.argument("username",) +@click.argument( + "username", +) def team_add_member(orgname_teamname, username): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index a974ee4fe7..dc03b2e5c8 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -63,7 +63,7 @@ SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}") SET(CMAKE_CXX_FLAGS "{{ _normalize_path(to_unix_path(cxx_flags)) }}") SET(CMAKE_C_FLAGS "{{ _normalize_path(to_unix_path(cc_flags)) }}") -% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)") +% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") % cc_stds = STD_RE.findall(cc_flags) % cxx_stds = STD_RE.findall(cxx_flags) % if cc_stds: diff --git a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl index eefe1a5215..d8826fca80 100644 --- a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl +++ b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl @@ -1,5 +1,5 @@ % import re -% STD_RE = re.compile(r"(\-std=[a-z\+]+\d+)") +% STD_RE = re.compile(r"(\-std=[a-z\+]+\w+)") % cxx_stds = STD_RE.findall(cxx_flags) % cxx_std = cxx_stds[-1] if cxx_stds else "" % diff --git a/platformio/ide/tpls/emacs/.ccls.tpl b/platformio/ide/tpls/emacs/.ccls.tpl index 9aaf54658e..b6d7d55aa4 100644 --- a/platformio/ide/tpls/emacs/.ccls.tpl +++ b/platformio/ide/tpls/emacs/.ccls.tpl @@ -1,5 +1,5 @@ % import re -% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)") +% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") % cc_stds = STD_RE.findall(cc_flags) % cxx_stds = STD_RE.findall(cxx_flags) % diff --git a/platformio/ide/tpls/vim/.ccls.tpl b/platformio/ide/tpls/vim/.ccls.tpl index 9aaf54658e..b6d7d55aa4 100644 --- a/platformio/ide/tpls/vim/.ccls.tpl +++ b/platformio/ide/tpls/vim/.ccls.tpl @@ -1,5 +1,5 @@ % import re -% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)") +% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") % cc_stds = STD_RE.findall(cc_flags) % cxx_stds = STD_RE.findall(cxx_flags) % diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index e6dda8957d..e99957cb4d 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -6,6 +6,14 @@ % % systype = platform.system().lower() % +% cpp_standards_remap = { +% "0x": "11", +% "1y": "14", +% "1z": "17", +% "2a": "20", +% "2b": "23" +% } +% % def _escape(text): % return to_unix_path(text).replace('"', '\\"') % end @@ -68,7 +76,7 @@ % % cleaned_includes = filter_includes(includes, ["toolchain"]) % -% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)") +% STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") % cc_stds = STD_RE.findall(cc_flags) % cxx_stds = STD_RE.findall(cxx_flags) % cc_m_flags = split_args(cc_flags) @@ -115,7 +123,7 @@ "cStandard": "c{{ cc_stds[-1] }}", % end % if cxx_stds: - "cppStandard": "c++{{ cxx_stds[-1] }}", + "cppStandard": "c++{{ cpp_standards_remap.get(cxx_stds[-1], cxx_stds[-1]) }}", % end % if forced_includes: "forcedInclude": [ diff --git a/platformio/maintenance.py b/platformio/maintenance.py index e038bcc097..5fa66bd160 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -124,7 +124,9 @@ def _update_pkg_metadata(_): continue result = result[0] pkg.metadata.spec = PackageSpec( - id=result["id"], owner=result["owner"]["username"], name=result["name"], + id=result["id"], + owner=result["owner"]["username"], + name=result["name"], ) pkg.dump_meta() return True diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index 6cb609a3c8..dcfe03f0c6 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -91,7 +91,10 @@ def is_system_compatible(value): @staticmethod def ensure_dir_exists(path): if not os.path.isdir(path): - os.makedirs(path) + try: + os.makedirs(path) + except: # pylint: disable=bare-except + pass assert os.path.isdir(path) return path diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index a11217e94a..fd1cd856b3 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -16,11 +16,12 @@ import os import subprocess import sys +from datetime import date from platformio import __core_packages__, exception, fs, util from platformio.compat import PY2 from platformio.package.manager.tool import ToolPackageManager -from platformio.package.meta import PackageSpec +from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import get_pythonexe_path @@ -95,16 +96,8 @@ def build_contrib_pysite_deps(target_dir): if os.path.isdir(target_dir): fs.rmtree(target_dir) os.makedirs(target_dir) - with open(os.path.join(target_dir, "package.json"), "w") as fp: - json.dump( - dict( - name="contrib-pysite", - version="2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), - system=util.get_systype(), - ), - fp, - ) + # build dependencies pythonexe = get_pythonexe_path() for dep in get_contrib_pysite_deps(): subprocess.check_call( @@ -115,11 +108,36 @@ def build_contrib_pysite_deps(target_dir): "install", # "--no-cache-dir", "--no-compile", + "--no-binary", + ":all:", "-t", target_dir, dep, ] ) + + # build manifests + with open(os.path.join(target_dir, "package.json"), "w") as fp: + json.dump( + dict( + name="contrib-pysite", + version="2.%d%d.%s" + % ( + sys.version_info.major, + sys.version_info.minor, + date.today().strftime("%y%m%d"), + ), + system=util.get_systype(), + ), + fp, + ) + pm = ToolPackageManager() + pkg = PackageItem(target_dir) + pkg.metadata = pm.build_metadata( + target_dir, PackageSpec(owner="platformio", name="contrib-pysite") + ) + pkg.dump_meta() + return True @@ -130,7 +148,7 @@ def get_contrib_pysite_deps(): twisted_version = "19.10.0" if PY2 else "20.3.0" result = [ "twisted == %s" % twisted_version, - "autobahn == 20.4.3", + "autobahn == %s" % ("19.11.2" if PY2 else "20.4.3"), "json-rpc == 1.13.0", ] diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py index a0d1407f9c..2843e9babe 100644 --- a/platformio/package/manager/library.py +++ b/platformio/package/manager/library.py @@ -15,7 +15,10 @@ import json import os -from platformio.package.exception import MissingPackageManifestError +from platformio.package.exception import ( + MissingPackageManifestError, + UnknownPackageError, +) from platformio.package.manager.base import BasePackageManager from platformio.package.meta import PackageItem, PackageSpec, PackageType from platformio.project.helpers import get_project_global_lib_dir @@ -43,7 +46,10 @@ def find_pkg_root(self, path, spec): # automatically generate library manifest with open(os.path.join(root_dir, "library.json"), "w") as fp: json.dump( - dict(name=spec.name, version=self.generate_rand_version(),), + dict( + name=spec.name, + version=self.generate_rand_version(), + ), fp, indent=2, ) @@ -63,6 +69,33 @@ def find_library_root(path): return root return path + def _install( # pylint: disable=too-many-arguments + self, + spec, + search_filters=None, + silent=False, + skip_dependencies=False, + force=False, + ): + try: + return super(LibraryPackageManager, self)._install( + spec, + search_filters=search_filters, + silent=silent, + skip_dependencies=skip_dependencies, + force=force, + ) + except UnknownPackageError as e: + # pylint: disable=import-outside-toplevel + from platformio.commands.lib.helpers import is_builtin_lib + + spec = self.ensure_spec(spec) + if is_builtin_lib(spec.name): + self.print_message("Already installed, built-in library", fg="yellow") + return True + + raise e + def install_dependencies(self, pkg, silent=False): assert isinstance(pkg, PackageItem) manifest = self.load_manifest(pkg) @@ -79,9 +112,16 @@ def install_dependencies(self, pkg, silent=False): ) def _install_dependency(self, dependency, silent=False): - spec = PackageSpec( - name=dependency.get("name"), requirements=dependency.get("version") - ) + if set(["name", "version"]) <= set(dependency.keys()) and any( + c in dependency["version"] for c in (":", "/", "@") + ): + spec = PackageSpec("%s=%s" % (dependency["name"], dependency["version"])) + else: + spec = PackageSpec( + owner=dependency.get("owner"), + name=dependency.get("name"), + requirements=dependency.get("version"), + ) search_filters = { key: value for key, value in dependency.items() diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 8949f43e53..b492bb7957 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -388,7 +388,15 @@ def _parse_dependencies(raw): raw = [raw] if isinstance(raw, dict): - return [dict(name=name, version=version) for name, version in raw.items()] + result = [] + for name, version in raw.items(): + if "/" in name: + owner, name = name.split("/", 1) + result.append(dict(owner=owner, name=name, version=version)) + else: + result.append(dict(name=name, version=version)) + return result + if isinstance(raw, list): for i, dependency in enumerate(raw): if isinstance(dependency, dict): diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 39327f4a55..f293ba5a23 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -106,6 +106,7 @@ class RepositorySchema(StrictSchema): class DependencySchema(StrictSchema): + owner = fields.Str(validate=validate.Length(min=1, max=100)) name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) version = fields.Str(validate=validate.Length(min=1, max=100)) authors = StrictListField(fields.Str(validate=validate.Length(min=1, max=50))) @@ -242,7 +243,7 @@ def validate_license(self, value): raise ValidationError("Could not load SPDX licenses for validation") for item in spdx.get("licenses", []): if item.get("licenseId") == value: - return + return True raise ValidationError( "Invalid SPDX license identifier. See valid identifiers at " "https://spdx.org/licenses/" @@ -251,9 +252,5 @@ def validate_license(self, value): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.10" - spdx_data_url = ( - "https://raw.githubusercontent.com/spdx/license-list-data" - "/v%s/json/licenses.json" % version - ) + spdx_data_url = "https://dl.bintray.com/platformio/dl-misc/spdx-licenses-3.json" return json.loads(fetch_remote_content(spdx_data_url)) diff --git a/platformio/package/meta.py b/platformio/package/meta.py index 147a1faf2d..edc5d0ffb1 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -209,6 +209,7 @@ def _parse(self, raw): raw = raw.strip() parsers = ( + self._parse_local_file, self._parse_requirements, self._parse_custom_name, self._parse_id, @@ -227,10 +228,16 @@ def _parse(self, raw): # the leftover is a package name self.name = raw - def _parse_requirements(self, raw): - if "@" not in raw: + @staticmethod + def _parse_local_file(raw): + if raw.startswith("file://") or not any(c in raw for c in ("/", "\\")): return raw - if raw.startswith("file://") and os.path.exists(raw[7:]): + if os.path.exists(raw): + return "file://%s" % raw + return raw + + def _parse_requirements(self, raw): + if "@" not in raw or raw.startswith("file://"): return raw tokens = raw.rsplit("@", 1) if any(s in tokens[1] for s in (":", "/")): diff --git a/platformio/package/pack.py b/platformio/package/pack.py index c833a1eb72..d88957fc5a 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -51,7 +51,9 @@ def get_archive_name(name, version, system=None): r"[^\da-zA-Z\-\._\+]+", "", "{name}{system}-{version}.tar.gz".format( - name=name, system=("-" + system) if system else "", version=version, + name=name, + system=("-" + system) if system else "", + version=version, ), ) diff --git a/platformio/platform/base.py b/platformio/platform/base.py index b29a9d7b82..b214da5f54 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -94,7 +94,7 @@ def packages(self): name = item version = "*" if "@" in item: - name, version = item.split("@", 2) + name, version = item.split("@", 1) spec = self.pm.ensure_spec(name) options = {"version": version.strip(), "optional": False} if spec.owner: diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 3fbcb74f5d..6ca60b0f10 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -301,7 +301,11 @@ def on_command(): def on_exception(e): skip_conditions = [ isinstance(e, cls) - for cls in (IOError, exception.ReturnErrorCode, exception.UserSideException,) + for cls in ( + IOError, + exception.ReturnErrorCode, + exception.UserSideException, + ) ] if any(skip_conditions): return diff --git a/setup.py b/setup.py index 512d2757ab..6472eadbe2 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ [">=2.7", "!=3.0.*", "!=3.1.*", "!=3.2.*", "!=3.3.*", "!=3.4.*"] ), install_requires=install_requires, - packages=find_packages() + ["scripts"], + packages=find_packages(exclude=["tests.*", "tests"]) + ["scripts"], package_data={ "platformio": [ "ide/tpls/*/.*.tpl", diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py index fc64db41ec..f04af53613 100644 --- a/tests/commands/test_account_org_team.py +++ b/tests/commands/test_account_org_team.py @@ -100,14 +100,21 @@ def test_account_register( def test_account_login( - clirunner, validate_cliresult, isolated_pio_core, + clirunner, + validate_cliresult, + isolated_pio_core, ): - result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) + result = clirunner.invoke( + cmd_account, + ["login", "-u", username, "-p", password], + ) validate_cliresult(result) def test_account_summary( - clirunner, validate_cliresult, isolated_pio_core, + clirunner, + validate_cliresult, + isolated_pio_core, ): result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) validate_cliresult(result) @@ -160,13 +167,21 @@ def test_account_summary( def test_account_token(clirunner, validate_cliresult, isolated_pio_core): - result = clirunner.invoke(cmd_account, ["token", "--password", password,],) + result = clirunner.invoke( + cmd_account, + [ + "token", + "--password", + password, + ], + ) validate_cliresult(result) assert "Personal Authentication Token:" in result.output token = result.output.strip().split(": ")[-1] result = clirunner.invoke( - cmd_account, ["token", "--password", password, "--json-output"], + cmd_account, + ["token", "--password", password, "--json-output"], ) validate_cliresult(result) json_result = json.loads(result.output.strip()) @@ -177,7 +192,14 @@ def test_account_token(clirunner, validate_cliresult, isolated_pio_core): clirunner.invoke(cmd_account, ["logout"]) - result = clirunner.invoke(cmd_account, ["token", "--password", password,],) + result = clirunner.invoke( + cmd_account, + [ + "token", + "--password", + password, + ], + ) assert result.exit_code > 0 assert result.exception assert "You are not authorized! Please log in to PIO Account" in str( @@ -187,7 +209,8 @@ def test_account_token(clirunner, validate_cliresult, isolated_pio_core): os.environ["PLATFORMIO_AUTH_TOKEN"] = token result = clirunner.invoke( - cmd_account, ["token", "--password", password, "--json-output"], + cmd_account, + ["token", "--password", password, "--json-output"], ) validate_cliresult(result) json_result = json.loads(result.output.strip()) @@ -197,7 +220,10 @@ def test_account_token(clirunner, validate_cliresult, isolated_pio_core): os.environ.pop("PLATFORMIO_AUTH_TOKEN") - result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) + result = clirunner.invoke( + cmd_account, + ["login", "-u", username, "-p", password], + ) validate_cliresult(result) @@ -205,7 +231,13 @@ def test_account_change_password(clirunner, validate_cliresult, isolated_pio_cor new_password = "Testpassword123" result = clirunner.invoke( cmd_account, - ["password", "--old-password", password, "--new-password", new_password,], + [ + "password", + "--old-password", + password, + "--new-password", + new_password, + ], ) validate_cliresult(result) assert "Password successfully changed!" in result.output @@ -213,13 +245,20 @@ def test_account_change_password(clirunner, validate_cliresult, isolated_pio_cor clirunner.invoke(cmd_account, ["logout"]) result = clirunner.invoke( - cmd_account, ["login", "-u", username, "-p", new_password], + cmd_account, + ["login", "-u", username, "-p", new_password], ) validate_cliresult(result) result = clirunner.invoke( cmd_account, - ["password", "--old-password", new_password, "--new-password", password,], + [ + "password", + "--old-password", + new_password, + "--new-password", + password, + ], ) validate_cliresult(result) @@ -272,14 +311,20 @@ def test_account_update( link = link.replace("&", "&") session.get(link) - result = clirunner.invoke(cmd_account, ["show"],) + result = clirunner.invoke( + cmd_account, + ["show"], + ) assert result.exit_code > 0 assert result.exception assert "You are not authorized! Please log in to PIO Account" in str( result.exception ) - result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) + result = clirunner.invoke( + cmd_account, + ["login", "-u", username, "-p", password], + ) validate_cliresult(result) @@ -317,7 +362,8 @@ def test_account_update( def test_org_create(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( - cmd_org, ["create", "--email", email, "--displayname", display_name, orgname], + cmd_org, + ["create", "--email", email, "--displayname", display_name, orgname], ) validate_cliresult(result) @@ -405,13 +451,21 @@ def test_org_update(clirunner, validate_cliresult, isolated_pio_core): def test_team_create(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cmd_team, - ["create", "%s:%s" % (orgname, teamname), "--description", team_description,], + [ + "create", + "%s:%s" % (orgname, teamname), + "--description", + team_description, + ], ) validate_cliresult(result) def test_team_list(clirunner, validate_cliresult, isolated_pio_core): - result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + result = clirunner.invoke( + cmd_team, + ["list", "%s" % orgname, "--json-output"], + ) validate_cliresult(result) json_result = json.loads(result.output.strip()) for item in json_result: @@ -423,22 +477,30 @@ def test_team_list(clirunner, validate_cliresult, isolated_pio_core): def test_team_add_member(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( - cmd_team, ["add", "%s:%s" % (orgname, teamname), second_username], + cmd_team, + ["add", "%s:%s" % (orgname, teamname), second_username], ) validate_cliresult(result) - result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + result = clirunner.invoke( + cmd_team, + ["list", "%s" % orgname, "--json-output"], + ) validate_cliresult(result) assert second_username in result.output def test_team_remove(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( - cmd_team, ["remove", "%s:%s" % (orgname, teamname), second_username], + cmd_team, + ["remove", "%s:%s" % (orgname, teamname), second_username], ) validate_cliresult(result) - result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + result = clirunner.invoke( + cmd_team, + ["list", "%s" % orgname, "--json-output"], + ) validate_cliresult(result) assert second_username not in result.output @@ -459,7 +521,10 @@ def test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_ ) validate_cliresult(result) - result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + result = clirunner.invoke( + cmd_team, + ["list", "%s" % orgname, "--json-output"], + ) validate_cliresult(result) json_result = json.loads(result.output.strip()) for item in json_result: diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index fa33af6874..06c133fca1 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -446,7 +446,10 @@ def test_check_embedded_platform_all_tools(clirunner, validate_cliresult, tmpdir result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(result) defects = sum(count_defects(result.output)) - assert defects > 0, "Failed %s with %s" % (framework, tool,) + assert defects > 0, "Failed %s with %s" % ( + framework, + tool, + ) def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir): diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py index 442edee35f..0014801b9f 100644 --- a/tests/commands/test_lib_complex.py +++ b/tests/commands/test_lib_complex.py @@ -339,14 +339,17 @@ def test_lib_stats(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) validate_cliresult(result) - assert set( - [ - "dlweek", - "added", - "updated", - "topkeywords", - "dlmonth", - "dlday", - "lastkeywords", - ] - ) == set(json.loads(result.output).keys()) + assert ( + set( + [ + "dlweek", + "added", + "updated", + "topkeywords", + "dlmonth", + "dlday", + "lastkeywords", + ] + ) + == set(json.loads(result.output).keys()) + ) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index cfb7fe3131..704543adc9 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -60,7 +60,8 @@ def test_install_unknown_from_registry(clirunner): def test_install_core_3_dev_platform(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( - cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], + cli_platform.platform_install, + ["atmelavr@1.2.0", "--skip-default-package"], ) assert result.exit_code == 0 diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index e0a64a8cf0..a50b9fa7d4 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -77,7 +77,8 @@ def test_multiple_env_build(clirunner, validate_cliresult, tmpdir): ) result = clirunner.invoke( - cmd_test, ["-d", str(project_dir), "--without-testing", "--without-uploading"], + cmd_test, + ["-d", str(project_dir), "--without-testing", "--without-uploading"], ) validate_cliresult(result) @@ -127,7 +128,8 @@ def test_setup_teardown_are_compilable(clirunner, validate_cliresult, tmpdir): ) native_result = clirunner.invoke( - cmd_test, ["-d", str(project_dir), "-e", "native"], + cmd_test, + ["-d", str(project_dir), "-e", "native"], ) test_dir.join("unittest_transport.h").write( diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 299b0020ca..8b948a95e4 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -230,6 +230,41 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory): tm.install("owner/unknown-package-tool", silent=True) +def test_install_lib_depndencies(isolated_pio_core, tmpdir_factory): + tmp_dir = tmpdir_factory.mktemp("tmp") + + src_dir = tmp_dir.join("lib-with-deps").mkdir() + root_dir = src_dir.mkdir("root") + root_dir.mkdir("src").join("main.cpp").write("#include ") + root_dir.join("library.json").write( + """ +{ + "name": "lib-with-deps", + "version": "2.0.0", + "dependencies": [ + { + "owner": "bblanchon", + "name": "ArduinoJson", + "version": "^6.16.1" + }, + { + "name": "external-repo", + "version": "https://github.com/milesburton/Arduino-Temperature-Control-Library.git#4a0ccc1" + } + ] +} +""" + ) + + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) + lm.install("file://%s" % str(src_dir), silent=True) + installed = lm.get_installed() + assert len(installed) == 4 + assert set(["external-repo", "ArduinoJson", "lib-with-deps", "OneWire"]) == set( + p.metadata.name for p in installed + ) + + def test_install_force(isolated_pio_core, tmpdir_factory): lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) # install #64 ArduinoJson diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 426cbdf17a..4a0afc76ae 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -44,7 +44,7 @@ def test_library_json_parser(): "dependencies": { "deps1": "1.2.0", "deps2": "https://github.com/username/package.git", - "@owner/deps3": "^2.1.3" + "owner/deps3": "^2.1.3" }, "customField": "Custom Value" } @@ -65,9 +65,9 @@ def test_library_json_parser(): "homepage": "http://old.url.format", "build": {"flags": ["-DHELLO"]}, "dependencies": [ - {"name": "@owner/deps3", "version": "^2.1.3"}, {"name": "deps1", "version": "1.2.0"}, {"name": "deps2", "version": "https://github.com/username/package.git"}, + {"owner": "owner", "name": "deps3", "version": "^2.1.3"}, ], "customField": "Custom Value", }, @@ -83,7 +83,7 @@ def test_library_json_parser(): }, "dependencies": [ {"name": "deps1", "version": "1.0.0"}, - {"name": "@owner/deps2", "version": "1.0.0", "platforms": "*", "frameworks": "arduino, espidf"}, + {"owner": "owner", "name": "deps2", "version": "1.0.0", "platforms": "*", "frameworks": "arduino, espidf"}, {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} ] } @@ -98,13 +98,14 @@ def test_library_json_parser(): "export": {"exclude": ["audio_samples"]}, "platforms": ["atmelavr"], "dependencies": [ + {"name": "deps1", "version": "1.0.0"}, { - "name": "@owner/deps2", + "owner": "owner", + "name": "deps2", "version": "1.0.0", "platforms": ["*"], "frameworks": ["arduino", "espidf"], }, - {"name": "deps1", "version": "1.0.0"}, { "name": "deps3", "version": "1.0.0", @@ -115,16 +116,16 @@ def test_library_json_parser(): ) raw_data = parser.LibraryJsonManifestParser( - '{"dependencies": ["dep1", "dep2", "@owner/dep3"]}' + '{"dependencies": ["dep1", "dep2", "owner/dep3@1.2.3"]}' ).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( raw_data, { "dependencies": [ - {"name": "@owner/dep3"}, {"name": "dep1"}, {"name": "dep2"}, + {"name": "owner/dep3@1.2.3"}, ], }, ) diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py index 0629f274fc..32b3d56a90 100644 --- a/tests/package/test_meta.py +++ b/tests/package/test_meta.py @@ -90,12 +90,13 @@ def test_spec_local_urls(tmpdir_factory): assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( url="file:///tmp/some-lib/", name="some-lib" ) - assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec( - url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1" + # detached package + assert PackageSpec("file:///tmp/some-lib@src-67e1043a673d2") == PackageSpec( + url="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib" ) - # detached folder with "@" symbol + # detached folder without scheme pkg_dir = tmpdir_factory.mktemp("storage").join("detached@1.2.3").mkdir() - assert PackageSpec("file://%s" % str(pkg_dir)) == PackageSpec( + assert PackageSpec(str(pkg_dir)) == PackageSpec( name="detached", url="file://%s" % pkg_dir )