diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index c84a97d71d..e0aed6a973 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -1,6 +1,6 @@ name: Core -on: [push] +on: [push, pull_request] jobs: build: @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [2.7, 3.7] + python-version: [2.7, 3.7, 3.8] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 @@ -28,8 +28,9 @@ jobs: tox -e lint - name: Integration Tests env: - PLATFORMIO_TEST_ACCOUNT_LOGIN: ${{ secrets.PLATFORMIO_TEST_ACCOUNT_LOGIN }} - PLATFORMIO_TEST_ACCOUNT_PASSWORD: ${{ secrets.PLATFORMIO_TEST_ACCOUNT_PASSWORD }} + TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }} + TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }} + TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }} run: | tox -e testcore diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bfe2c116ca..39de401e6a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,6 +1,6 @@ name: Docs -on: [push] +on: [push, pull_request] jobs: build: diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index b5452909e2..e3bb201f3a 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -1,6 +1,6 @@ name: Examples -on: [push] +on: [push, pull_request] jobs: build: @@ -49,6 +49,7 @@ jobs: if: startsWith(matrix.os, 'windows') env: PLATFORMIO_CORE_DIR: C:/pio + PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap" run: | tox -e testexamples diff --git a/.pylintrc b/.pylintrc index 67ce4ef721..e21dfef9f5 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,3 +1,6 @@ +[REPORTS] +output-format=colorized + [MESSAGES CONTROL] disable= bad-continuation, @@ -12,4 +15,8 @@ disable= useless-object-inheritance, useless-import-alias, fixme, - bad-option-value + bad-option-value, + + ; PY2 Compat + super-with-arguments, + raise-missing-from diff --git a/HISTORY.rst b/HISTORY.rst index ab9119fe24..c28e2f3967 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,2028 +1,122 @@ Release Notes ============= -.. _release_notes_4: +.. _release_notes_5: -PlatformIO Core 4 +PlatformIO Core 5 ----------------- -4.3.4 (2020-05-23) -~~~~~~~~~~~~~~~~~~ - -* Added `PlatformIO CLI Shell Completion `__ for Fish, Zsh, Bash, and PowerShell (`issue #3435 `_) -* Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 `_) -* Fixed an issue on Windows when installing a library dependency from Git repository (`issue #2844 `_, `issue #3328 `_) - -4.3.3 (2020-04-28) -~~~~~~~~~~~~~~~~~~ - -* Fixed "UnicodeDecodeError: 'utf-8' codec can't decode byte" when non-Latin chars are used in project path (`issue #3481 `_) - -4.3.2 (2020-04-28) -~~~~~~~~~~~~~~~~~~ - -* New `Account Management System `__ (preview) -* Open source `PIO Remote `__ client -* Improved `PIO Check `__ with more accurate project processing -* Echo what is typed when ``send_on_enter`` `device monitor filter `__ is used (`issue #3452 `_) -* Fixed PIO Unit Testing for Zephyr RTOS -* Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 `_) -* Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 `_) -* Fixed an incorrect node path used for pattern matching when processing middleware nodes -* Fixed an issue with missing ``lib_extra_dirs`` option in SRC_LIST for CLion (`issue #3460 `_) - -4.3.1 (2020-03-20) -~~~~~~~~~~~~~~~~~~ +**A professional collaborative platform for embedded development** -* Fixed a SyntaxError "'return' with argument inside generator" for PIO Unified Debugger when Python 2.7 is used -* Fixed an issue when ``lib_archive = no`` was not honored in `"platformio.ini" `__ -* Fixed a TypeError "super(type, obj): obj must be an instance or subtype of type" when device monitor is used with a custom dev-platform filter (`issue #3431 `_) +- `Migration guide from 4.x to 5.0 `__ -4.3.0 (2020-03-19) +5.0.0 (2020-09-03) ~~~~~~~~~~~~~~~~~~ -* Initial support for an official `PlatformIO for CLion IDE `__ plugin: - - - Smart C and C++ editor - - Code refactoring - - On-the-fly code analysis - - "New PlatformIO Project" wizard - - Building, Uploading, Testing - - Integrated debugger (inline variable view, conditional breakpoints, expressions, watchpoints, peripheral registers, multi-thread support, etc.) - -* `Device Monitor 2.0 `__ - - - Added **PlatformIO Device Monitor Filter API** (dev-platforms can extend base device monitor with a custom functionality, such as exception decoding) (`pull #3383 `_) - - Configure project device monitor with `monitor_filters `__ option - - `Capture device monitor output to a file `__ with ``log2file`` filter (`issue #670 `_) - - Show a timestamp for each new line with ``time`` filter (`issue #981 `_) - - Send a text to device on ENTER with ``send_on_enter`` filter (`issue #926 `_) - - Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter - -* New standalone (1-script) `PlatformIO Core Installer `_ -* Initial support for `Renode `__ simulation framework (`issue #3401 `_) -* Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) -* Improved support for Arduino "library.properties" ``depends`` field -* Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server -* Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) -* Fixed an issue when Python 2 does not keep encoding when converting ".ino" (`issue #3393 `_) -* Fixed an issue when ``"libArchive": false`` in "library.json" does not work (`issue #3403 `_) -* Fixed an issue when not all commands in `compilation database "compile_commands.json" `__ use absolute paths (`pull #3415 `_) -* Fixed an issue when unknown transport is used for `PIO Unit Testing `__ engine (`issue #3422 `_) - -4.2.1 (2020-02-17) -~~~~~~~~~~~~~~~~~~ - -* Improved VSCode template with special ``forceInclude`` field for direct includes via ``-include`` flag (`issue #3379 `_) -* Improved support of PIO Home on card-sized PC (Raspberry Pi, etc.) (`issue #3313 `_) -* Froze "marshmallow" dependency to 2.X for Python 2 (`issue #3380 `_) -* Fixed "TypeError: unsupported operand type(s)" when system environment variable is used by project configuration parser (`issue #3377 `_) -* Fixed an issue when Library Dependency Finder (LDF) ignores custom "libLDFMode" and "libCompatMode" options in `library.json `__ -* Fixed an issue when generating of compilation database "compile_commands.json" does not work with Python 2.7 (`issue #3378 `_) - - -4.2.0 (2020-02-12) -~~~~~~~~~~~~~~~~~~ +* Integration with the new **PlatformIO Trusted Registry** -* `PlatformIO Home 3.1 `__: + - Enterprise-grade package storage with high availability (multi replicas) + - Secure, fast, and reliable global content delivery network (CDN) + - Universal support for all packages: - - Project Manager - - Project Configuration UI for `"platformio.ini" `__ + * Libraries + * Development platforms + * Toolchains -* `PIO Check `__ – automated code analysis without hassle: + - Built-in fine-grained access control (role-based, teams, organizations) + - New CLI commands: - - Added support for `PVS-Studio `__ static code analyzer + * `pio package `__ – manage packages in the registry + * `pio access `__ – manage package access for users, teams, and maintainers -* Initial support for `Project Manager `_ CLI: +* Integration with the new **Account Management System** - - Show computed project configuration with a new `platformio project config `_ command or dump to JSON with ``platformio project config --json-output`` (`issue #3335 `_) - - Moved ``platformio init`` command to `platformio project init `_ + - `Manage organizations `__ + - `Manage teams and team memberships `__ -* Generate `compilation database "compile_commands.json" `__ (`issue #2990 `_) -* Control debug flags and optimization level with a new `debug_build_flags `__ option -* Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install `__ command (`issue #3345 `_) -* Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) -* Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 `_) -* Added support for Arduino's library.properties ``depends`` field (`issue #2781 `_) -* Autodetect monitor port for boards with specified HWIDs (`issue #3349 `_) -* Updated SCons tool to 3.1.2 -* Updated Unity tool to 2.5.0 -* Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 `_) -* Warn about broken library manifest when scanning dependencies (`issue #3268 `_) -* Do not overwrite custom items in VSCode's "extensions.json" (`issue #3374 `_) -* Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 `_) -* Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 `_) -* Fixed default PIO Unified Debugger configuration for `J-Link probe `__ -* Fixed an issue with LDF when header files not found if "libdeps_dir" is within a subdirectory of "lib_extra_dirs" (`issue #3311 `_) -* Fixed an issue "Import of non-existent variable 'projenv''" when development platform does not call "env.BuildProgram()" (`issue #3315 `_) -* Fixed an issue when invalid CLI command does not return non-zero exit code -* Fixed an issue when Project Inspector crashes when flash use > 100% (`issue #3368 `_) -* Fixed a "UnicodeDecodeError" when listing built-in libraries on macOS with Python 2.7 (`issue #3370 `_) -* Fixed an issue with improperly handled compiler flags with space symbols in VSCode template (`issue #3364 `_) -* Fixed an issue when no error is raised if referred parameter (interpolation) is missing in a project configuration file (`issue #3279 `_) +* New **Package Management System** + - Integrated PlatformIO Core with the new PlatformIO Registry + - Support for owner-based dependency declaration (resolves name conflicts) (`issue #1824 `_) + - Automatically save dependencies to `"platformio.ini" `__ when installing using PlatformIO CLI (`issue #2964 `_) + - Follow SemVer complaint version constraints when checking library updates `issue #1281 `_) + - Dropped support for "packageRepositories" section in "platform.json" manifest (please publish packages directly to the registry) -4.1.0 (2019-11-07) -~~~~~~~~~~~~~~~~~~ - -* `PIO Check `__ – automated code analysis without hassle: - - - Potential NULL pointer dereferences - - Possible indexing beyond array bounds - - Suspicious assignments - - Reads of potentially uninitialized objects - - Unused variables or functions - - Out of scope memory usage. - -* `PlatformIO Home 3.0 `__: - - - Project Inspection - - Static Code Analysis - - Firmware File Explorer - - Firmware Memory Inspection - - Firmware Sections & Symbols Viewer. - -* Added support for `Build Middlewares `__: configure custom build flags per specific file, skip any build nodes from a framework, replace build file with another on-the-fly, etc. -* Extend project environment configuration in "platformio.ini" with other sections using a new `extends `__ option (`issue #2953 `_) -* Generate ``.ccls`` LSP file for `Emacs `__ cross references, hierarchies, completion and semantic highlighting -* Added ``--no-ansi`` flag for `PIO Core `__ to disable ANSI control characters -* Added ``--shutdown-timeout`` option to `PIO Home Server `__ -* Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) -* Fixed default PIO Unified Debugger configuration for `J-Link probe `__ -* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 `_) -* Fixed an issue when installing a package using custom Git tag and submodules were not updated correctly (`issue #3060 `_) -* Fixed an issue with linking process when ``$LDSCRIPT`` contains a space in path -* Fixed security issue when extracting items from TAR archive (`issue #2995 `_) -* Fixed an issue with project generator when ``src_build_flags`` were not respected (`issue #3137 `_) -* Fixed an issue when booleans in "platformio.ini" are not parsed properly (`issue #3022 `_) -* Fixed an issue with invalid encoding when generating project for Visual Studio (`issue #3183 `_) -* Fixed an issue when Project Config Parser does not remove in-line comments when Python 3 is used (`issue #3213 `_) -* Fixed an issue with a GCC Linter for PlatformIO IDE for Atom (`issue #3218 `_) - -4.0.3 (2019-08-30) -~~~~~~~~~~~~~~~~~~ - -* Added support for multi-environment PlatformIO project for `CLion IDE `__ (`issue #2824 `_) -* Generate ``.ccls`` LSP file for `Vim `__ cross references, hierarchies, completion and semantic highlighting (`issue #2952 `_) -* Added support for `PLATFORMIO_DISABLE_COLOR `__ system environment variable which disables color ANSI-codes in a terminal output (`issue #2956 `_) -* Updated SCons tool to 3.1.1 -* Remove ProjectConfig cache when "platformio.ini" was modified outside -* Fixed an issue with PIO Unified Debugger on Windows OS when debug server is piped -* Fixed an issue when `--upload-port `__ CLI flag does not override declared `upload_port `__ option in `"platformio.ini" (Project Configuration File) `__ - -4.0.2 (2019-08-23) -~~~~~~~~~~~~~~~~~~ +* **Build System** -* Fixed an issue with a broken `LDF `__ when checking for framework compatibility (`issue #2940 `_) + - Upgraded build engine to the `SCons 4.0 - a next-generation software construction tool `__ -4.0.1 (2019-08-22) -~~~~~~~~~~~~~~~~~~ + * `Configuration files are Python scripts `__ – use the power of a real programming language to solve build problems + * Built-in reliable and automatic dependency analysis + * Improved support for parallel builds + * Ability to `share built files in a cache `__ to speed up multiple builds -* Print `debug tool `__ name for the active debugging session -* Do not shutdown PIO Home Server for "upgrade" operations (`issue #2784 `_) -* Improved computing of project check sum (structure, configuration) and avoid unnecessary rebuilding -* Improved printing of tabulated results -* Automatically normalize file system paths to UNIX-style for Project Generator (`issue #2857 `_) -* Ability to set "databaseFilename" for VSCode and C/C++ extension (`issue #2825 `_) -* Renamed "enable_ssl" setting to `strict_ssl `__ -* Fixed an issue with incorrect escaping of Windows slashes when using `PIO Unified Debugger `__ and "piped" openOCD -* Fixed an issue when "debug", "home", "run", and "test" commands were not shown in "platformio --help" CLI -* Fixed an issue with PIO Home's "No JSON object could be decoded" (`issue #2823 `_) -* Fixed an issue when `library.json `__ had priority over project configuration for `LDF `__ (`issue #2867 `_) + - New `Custom Targets `__ -4.0.0 (2019-07-10) -~~~~~~~~~~~~~~~~~~ + * Pre/Post processing based on dependent sources (another target, source file, etc.) + * Command launcher with own arguments + * Launch command with custom options declared in `"platformio.ini" `__ + * Python callback as a target (use the power of Python interpreter and PlatformIO Build API) + * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets `__ command (`issue #3544 `_) -`Migration Guide from 3.0 to 4.0 `__. + - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 `_) + - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) `__ for Arduino library when "library.property" has "depends" field (`issue #3607 `_) + - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 `_) + - Fixed an issue with the ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 `_) -* `PlatformIO Plus Goes Open Source `__ +* **Project Management** - - Built-in `PIO Unified Debugger `__ - - Built-in `PIO Unit Testing `__ + - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci `__, `src_filter `__, `check_patterns `__, `library.json > srcFilter `__). Python 3.5+ is required + - Added a new ``-e, --environment`` option to `pio project init `__ command that helps to update a PlatformIO project using the existing environment + - Dump build system data intended for IDE extensions/plugins using a new `pio project data `__ command + - Do not generate ".travis.yml" for a new project, let the user have a choice -* **Project Configuration** +* **Unit Testing** - - New project configuration parser with a strict options typing (`API `__) - - Unified workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 `_) - - Share common (global) options between project environments using `[env] `__ section (`issue #1643 `_) - - Include external configuration files with `extra_configs `__ option (`issue #1590 `_) - - Custom project ``***_dir`` options declared in `platformio `__ section have higher priority than `Environment variables `__ - - Added support for Unix shell-style wildcards for `monitor_port `__ option (`issue #2541 `_) - - Added new `monitor_flags `__ option which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) - - Added support for `PLATFORMIO_DEFAULT_ENVS `__ system environment variable (`issue #1967 `_) - - Added support for `shared_dir `__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a `PIO Remote `__ machine + - Updated PIO Unit Testing support for Mbed framework and added compatibility with Mbed OS 6 + - Fixed an issue when running multiple test environments (`issue #3523 `_) + - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 `_) -* **Library Management** +* **Static Code Analysis** - - Switched to workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` - - Save libraries passed to `platformio lib install `__ command into the project dependency list (`lib_deps `__) with a new ``--save`` flag (`issue #1028 `_) - - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - - Use isolated library dependency storage per project build environment (`issue #1696 `_) - - Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry (`issue #1654 `_) - - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 `_) - - Fixed an issue when `lib_compat_mode = strict `__ does not ignore libraries incompatible with a project framework - -* **Build System** + - Updated analysis tools: - - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` - - Switch between `Build Configurations `__ (``release`` and ``debug``) with a new project configuration option `build_type `__ - - Custom `platform_packages `__ per a build environment with an option to override default (`issue #1367 `_) - - Print platform package details, such as version, VSC source and commit (`issue #2155 `_) - - Control a number of parallel build jobs with a new `-j, --jobs `__ option - - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) - - Configure a shared folder for the derived files (objects, firmwares, ELFs) from a build system using `build_cache_dir `__ option (`issue #2674 `_) - - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) + * `Cppcheck `__ v2.1 with a new "soundy" analysis option and improved code parser + * `PVS-Studio `__ v7.09 with a new file list analysis mode and an extended list of analysis diagnostics -* **Infrastructure** + - Added Cppcheck package for ARM-based single-board computers (`issue #3559 `_) + - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 `_) - - Python 3 support (`issue #895 `_) - - Significantly speedup back-end for PIO Home. It works super fast now! - - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) - - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) - - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #143 `_, `issue #1342 `_, `issue #1959 `_, `issue #2100 `_) +* **Miscellaneous** -* **Integration** + - Display system-wide information using a new `pio system info `__ command (`issue #3521 `_) + - Remove unused data using a new `pio system prune `__ command (`issue #3522 `_) + - Show ignored project environments only in the verbose mode (`issue #3641 `_) + - Do not escape compiler arguments in VSCode template on Windows. - - Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file - - Fixed an issue with hardcoded C standard version when generating project for CLion IDE (`issue #2527 `_) - - Fixed an issue with Project Generator when an include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) - - Fixed an issue when generating invalid "Eclipse CDT Cross GCC Built-in Compiler Settings" if a custom `PLATFORMIO_CORE_DIR `__ is used (`issue #806 `_) +.. _release_notes_4: -* **Miscellaneous** +PlatformIO Core 4 +----------------- - - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead - - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) - - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 `_) +See `PlatformIO Core 4.0 history `__. PlatformIO Core 3 ----------------- -3.6.7 (2019-04-23) -~~~~~~~~~~~~~~~~~~ - -* `PIO Unified Debugger `__: improved debugging in ``debug_load_mode = modified`` and fixed an issue with useless project rebuilding -* Project Generator: fixed a VSCode C/C++'s "Cannot find" warning when CPPPATH folder does not exist -* Fixed an "IndexError: list index out of range" for Arduino sketch preprocessor - (`issue #2268 `_) -* Fixed an issue when invalid "env_default" in `"platformio.ini" (Project Configuration File) `__ results into unhandled errors - (`issue #2265 `_) - -3.6.6 (2019-03-29) -~~~~~~~~~~~~~~~~~~ - -* Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode - (`issue #2243 `_) -* Fixed an issue when PlatformIO Build System does not pick up "mbed_lib.json" files from libraries - (`issue #2164 `_) -* Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor) -* Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option -* Fixed an issue with incorrect order of project "include" and "src" paths in ``CPPPATH`` - (`issue #1914 `_) - -3.6.5 (2019-03-07) -~~~~~~~~~~~~~~~~~~ - -* Project Generator: added new targets for CLion IDE "BUILD_VERBOSE" and "MONITOR" (serial port monitor) - (`issue #359 `_) -* Fixed an issue with slow updating of PlatformIO Core packages on Windows -* Fixed an issue when `platformio ci `__ recompiles project if ``--keep-build-dir`` option is passed - (`issue #2109 `_) -* Fixed an issue when ``$PROJECT_HASH`` template was not expanded for the other directory ``***_dir`` options in `"platformio.ini" (Project Configuration File) `__ - (`issue #2170 `_) - -3.6.4 (2019-01-23) -~~~~~~~~~~~~~~~~~~ - -* Improved Project Generator for IDEs: - - - Use full path to PlatformIO CLI when generating a project - (`issue #1674 `_) - - CLion: Improved project portability using "${CMAKE_CURRENT_LIST_DIR}" instead of full path - - Eclipse: Provide language standard to a project C/C++ indexer - (`issue #1010 `_) - -* Fixed an issue with incorrect detecting of compatibility (LDF) between generic library and Arduino or ARM mbed frameworks -* Fixed "Runtime Error: Dictionary size changed during iteration" - (`issue #2003 `_) -* Fixed an error "Could not extract item..." when extracting TAR archive with symbolic items on Windows platform - (`issue #2015 `_) - -3.6.3 (2018-12-12) -~~~~~~~~~~~~~~~~~~ - -* Ignore ``*.asm`` and ``*.ASM`` files when building Arduino-based library (compatibility with Arduino builder) -* Fixed spurious project's "Problems" for `PlatformIO IDE for VSCode `__ when ARM mbed framework is used -* Fixed an issue with a broken headers list when generating ".clang_complete" for `Emacs `__ - (`issue #1960 `_) - -3.6.2 (2018-11-29) -~~~~~~~~~~~~~~~~~~ - -* Improved IntelliSense for `PlatformIO IDE for VSCode `__ via passing extra compiler information for C/C++ Code Parser (resolves issues with spurious project's "Problems") -* Fixed an issue with VSCode IntelliSense warning about the missed headers located in `include `__ folder -* Fixed incorrect wording when initializing/updating project -* Fixed an issue with incorrect order for library dependencies ``CPPPATH`` - (`issue #1914 `_) -* Fixed an issue when Library Dependency Finder (LDF) does not handle project `src_filter `__ - (`issue #1905 `_) -* Fixed an issue when Library Dependency Finder (LDF) finds spurious dependencies in ``chain+`` and ``deep+`` modes - (`issue #1930 `_) - -3.6.1 (2018-10-29) -~~~~~~~~~~~~~~~~~~ - -* Generate an `include `__ and `test `__ directories with a README file when initializing a new project -* Support in-line comments for multi-line value (``lib_deps``, ``build_flags``, etc) in `"platformio.ini" (Project Configuration File) `__ -* Added ``$PROJECT_HASH`` template variable for `build_dir `__. One of the use cases is setting a global storage for project artifacts using `PLATFORMIO_BUILD_DIR `__ system environment variable. For example, ``/tmp/pio-build/$PROJECT_HASH`` (Unix) or ``$[sysenv.TEMP}/pio-build/$PROJECT_HASH`` (Windows) -* Improved a loading speed of PIO Home "Recent News" -* Improved `PIO Unified Debugger `__ for "mbed" framework and fixed issue with missed local variables -* Introduced `"Release" and "Debug" Build Configurations `__ -* Build project in "Debug Mode" including debugging information with a new ``debug`` target using `platformio run `__ command or `targets `__ option in ``platformio.ini``. The last option allows avoiding project rebuilding between "Run/Debug" modes. - (`issue #1833 `_) -* Process ``build_unflags`` for the cloned environment when building a static library -* Report on outdated `99-platformio-udev.rules `__ - (`issue #1823 `_) -* Show a valid error when the Internet is off-line while initializing a new project - (`issue #1784 `_) -* Do not re-create ".gitignore" and ".travis.yml" files if they were removed from a project -* Fixed an issue when dynamic build flags were not handled correctly - (`issue #1799 `_) -* Fixed an issue when ``pio run -t monitor`` always uses the first ``monitor_port`` even with multiple environments - (`issue #1841 `_) -* Fixed an issue with broken includes when generating ``.clang_complete`` and space is used in a path - (`issue #1873 `_) -* Fixed an issue with incorrect handling of a custom package name when using `platformio lib install `__ or `platformio platform install `__ commands - -3.6.0 (2018-08-06) -~~~~~~~~~~~~~~~~~~ - -* `Program Memory Usage `_ - - - Print human-readable memory usage information after a build and before uploading - - Print detailed memory usage information with "sections" and "addresses" - in `verbose mode `__ - - Check maximum allowed "program" and "data" sizes before uploading/programming - (`issue #1412 `_) - -* `PIO Unit Testing `__: - - - Documented `Project Shared Code `__ - - Force building of project source code using `test_build_project_src `__ option - - Fixed missed ``UNIT_TEST`` macro for unit test components/libraries - -* Check package structure after unpacking and raise error when antivirus tool - blocks PlatformIO package manager - (`issue #1462 `_) -* Lock interprocess requests to PlatformIO Package Manager for - install/uninstall operations - (`issue #1594 `_) -* Fixed an issue with `PIO Remote `__ - when upload process depends on the source code of a project framework -* Fixed an issue when ``srcFilter`` field in `library.json `__ - breaks a library build - (`issue #1735 `_) - -3.5.4 (2018-07-03) -~~~~~~~~~~~~~~~~~~ - -* Improved removing of default build flags using `build_unflags `__ option - (`issue #1712 `_) -* Export ``LIBS``, ``LIBPATH``, and ``LINKFLAGS`` data from project dependent - libraries to the global build environment -* Don't export ``CPPPATH`` data of project dependent libraries to framework's - build environment - (`issue #1665 `_) -* Handle "architectures" data from "library.properties" manifest in - `lib_compat_mode = strict `__ -* Added workaround for Python SemVer package's `issue #61 `_ with caret range and pre-releases -* Replaced conflicted "env" pattern by "sysenv" for `"platformio.ini" Dynamic Variables" `__ - (`issue #1705 `_) -* Removed "date&time" when processing project with `platformio run `__ command - (`issue #1343 `_) -* Fixed issue with invalid LD script if path contains space -* Fixed preprocessor for Arduino sketch when function returns certain type - (`issue #1683 `_) -* Fixed issue when `platformio lib uninstall `__ - removes initial source code - (`issue #1023 `_) - -3.5.3 (2018-06-01) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home `__ - - interact with PlatformIO ecosystem using modern and cross-platform GUI: - - - "Recent News" block on "Welcome" page - - Direct import of development platform's example - -* Simplify configuration for `PIO Unit Testing `__: separate main program from a test build process, drop - requirement for ``#ifdef UNIT_TEST`` guard -* Override any option from board manifest in `"platformio.ini" (Project Configuration File) `__ - (`issue #1612 `_) -* Configure a custom path to SVD file using `debug_svd_path `__ - option -* Custom project `description `_ - which will be used by `PlatformIO Home `_ -* Updated Unity tool to 2.4.3 -* Improved support for Black Magic Probe in "uploader" mode -* Renamed "monitor_baud" option to "monitor_speed" -* Fixed issue when a custom `lib_dir `__ - was not handled correctly - (`issue #1473 `_) -* Fixed issue with useless project rebuilding for case insensitive file - systems (Windows) -* Fixed issue with ``build_unflags`` option when a macro contains value - (e.g., ``-DNAME=VALUE``) -* Fixed issue which did not allow to override runtime build environment using - extra POST script -* Fixed "RuntimeError: maximum recursion depth exceeded" for library manager - (`issue #1528 `_) - -3.5.2 (2018-03-13) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home `__ - - interact with PlatformIO ecosystem using modern and cross-platform GUI: - - - Multiple themes (Dark & Light) - - Ability to specify a name for new project - -* Control `PIO Unified Debugger `__ - and its firmware loading mode using - `debug_load_mode `__ option -* Added aliases (off, light, strict) for - `LDF Compatibility Mode `__ -* Search for a library using PIO Library Registry ID ``id:X`` (e.g. ``pio lib search id:13``) -* Show device system information (MCU, Frequency, RAM, Flash, Debugging tools) - in a build log -* Show all available upload protocols before firmware uploading in a build log -* Handle "os.mbed.com" URL as a Mercurial (hg) repository -* Improved support for old mbed libraries without manifest -* Fixed project generator for Qt Creator IDE - (`issue #1303 `_, - `issue #1323 `_) -* Mark project source and library directories for CLion IDE - (`issue #1359 `_, - `issue #1345 `_, - `issue #897 `_) -* Fixed issue with duplicated "include" records when generating data for IDE - (`issue #1301 `_) - -3.5.1 (2018-01-18) -~~~~~~~~~~~~~~~~~~ - -* New ``test_speed`` option to control a communication baudrate/speed between - `PIO Unit Testing `__ - engine and a target device - (`issue #1273 `_) -* Show full library version in "Library Dependency Graph" including VCS - information - (`issue #1274 `_) -* Configure a custom firmware/program name in build directory (`example `__) -* Renamed ``envs_dir`` option to ``build_dir`` - in `"platformio.ini" (Project Configuration File) `__ -* Refactored code without "arrow" dependency (resolve issue with "ImportError: - No module named backports.functools_lru_cache") -* Improved support of PIO Unified Debugger for Eclipse Oxygen -* Improved a work in off-line mode -* Fixed project generator for CLion and Qt Creator IDE - (`issue #1299 `_) -* Fixed PIO Unified Debugger for mbed framework -* Fixed library updates when a version is declared in VCS format (not SemVer) - -3.5.0 (2017-12-28) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Home `__ - - interact with PlatformIO ecosystem using modern and cross-platform GUI: - - - Library Manager: - - * Search for new libraries in PlatformIO Registry - * "1-click" library installation, per-project libraries, extra storages - * List installed libraries in multiple storages - * List built-in libraries (by frameworks) - * Updates for installed libraries - * Multiple examples, trending libraries, and more. - - - PlatformIO Projects - - PIO Account - - Development platforms, frameworks and board explorer - - Device Manager: serial, logical, and multicast DNS services - -* Integration with `Jenkins CI `_ -* New `include `__ - folder for project's header files - (`issue #1107 `_) -* Depend on development platform using VCS URL (Git, Mercurial and Subversion) - instead of a name in `"platformio.ini" (Project Configuration File) `__. - Drop support for ``*_stage`` dev/platform names (use VCS URL instead). -* Reinstall/redownload package with a new ``-f, --force`` option for - `platformio lib install `__ - and `platformio platform install `__ - commands - (`issue #778 `_) -* Handle missed dependencies and provide a solution based on PlatformIO Library - Registry - (`issue #781 `_) -* New setting `projects_dir `__ - that allows to override a default PIO Home Projects location - (`issue #1161 `_) - -* `Library Dependency Finder (LDF) `__: - - - Search for dependencies used in `PIO Unit Testing `__ - (`issue #953 `_) - - Parse library source file in pair with a header when they have the same name - (`issue #1175 `_) - - Handle library dependencies defined as VCS or SemVer in - `"platformio.ini" (Project Configuration File) `__ - (`issue #1155 `_) - - Added option to configure library `Compatible Mode `__ - using `library.json `__ - -* New options for `platformio device list `__ - command: - - - ``--serial`` list available serial ports (default) - - ``--logical`` list logical devices - - ``--mdns`` discover multicast DNS services - (`issue #463 `_) - -* Fixed platforms, packages, and libraries updating behind proxy - (`issue #1061 `_) -* Fixed missing toolchain include paths for project generator - (`issue #1154 `_) -* Fixed "Super-Quick (Mac / Linux)" installation in "get-platformio.py" script - (`issue #1017 `_) -* Fixed "get-platformio.py" script which hangs on Windows 10 - (`issue #1118 `_) -* Other bug fixes and performance improvements - -3.4.1 (2017-08-02) -~~~~~~~~~~~~~~~~~~ - -* Pre/Post extra scripting for advanced control of PIO Build System - (`issue #891 `_) -* New `lib_archive `_ - option to control library archiving and linking behavior - (`issue #993 `_) -* Add "inc" folder automatically to CPPPATH when "src" is available (works for project and library) - (`issue #1003 `_) -* Use a root of library when filtering source code using - `library.json `__ - and ``srcFilter`` field -* Added ``monitor_*`` options to white-list for `"platformio.ini" (Project Configuration File) `__ - (`issue #982 `_) -* Do not ask for board ID when initialize project for desktop platform -* Handle broken PIO Core state and create new one -* Fixed an issue with a custom transport for `PIO Unit Testing `__ - when multiple tests are present -* Fixed an issue when can not upload firmware to SAM-BA based board (Due) - -3.4.0 (2017-06-26) -~~~~~~~~~~~~~~~~~~ - -* `PIO Unified Debugger `__ - - - "1-click" solution, zero configuration - - Support for 100+ embedded boards - - Multiple architectures and development platforms - - Windows, MacOS, Linux (+ARMv6-8) - - Built-in into `PlatformIO IDE for Atom `__ and `PlatformIO IDE for VScode `__ - - Integration with `Eclipse `__ and `Sublime Text `__ - -* Filter `PIO Unit Testing `__ - tests using a new ``test_filter`` option in `"platformio.ini" (Project Configuration File) `__ - or `platformio test --filter `__ command - (`issue #934 `_) -* Custom ``test_transport`` for `PIO Unit Testing `__ Engine -* Configure Serial Port Monitor in `"platformio.ini" (Project Configuration File) `__ - (`issue #787 `_) -* New `monitor `__ - target which allows to launch Serial Monitor automatically after successful - "build" or "upload" operations - (`issue #788 `_) -* Project generator for `VIM `__ -* Multi-line support for the different options in `"platformio.ini" (Project Configuration File) `__, - such as: ``build_flags``, ``build_unflags``, etc. - (`issue #889 `_) -* Handle dynamic ``SRC_FILTER`` environment variable from - `library.json extra script `__ -* Notify about multiple installations of PIO Core - (`issue #961 `_) -* Improved auto-detecting of mbed-enabled media disks -* Automatically update Git-submodules for development platforms and libraries - that were installed from repository -* Add support for ``.*cc`` extension - (`issue #939 `_) -* Handle ``env_default`` in `"platformio.ini" (Project Configuration File) `__ - when re-initializing a project - (`issue #950 `_) -* Use root directory for PIO Home when path contains non-ascii characters - (`issue #951 `_, - `issue #952 `_) -* Don't warn about known ``boards_dir`` option - (`pull #949 `_) -* Escape non-valid file name characters when installing a new package (library) - (`issue #985 `_) -* Fixed infinite dependency installing when repository consists of multiple - libraries - (`issue #935 `_) -* Fixed linter error "unity.h does not exist" for Unit Testing - (`issue #947 `_) -* Fixed issue when `Library Dependency Finder (LDF) `__ - does not handle custom ``src_dir`` - (`issue #942 `_) -* Fixed cloning a package (library) from a private Git repository with - custom user name and SSH port - (`issue #925 `_) - -3.3.1 (2017-05-27) -~~~~~~~~~~~~~~~~~~ - -* Hotfix for recently updated Python Requests package (2.16.0) - -3.3.0 (2017-03-27) -~~~~~~~~~~~~~~~~~~ - -* PlatformIO Library Registry statistics with new - `pio lib stats `__ command - - - Recently updated and added libraries - - Recent and popular keywords - - Featured libraries (today, week, month) - -* List built-in libraries based on development platforms with a new - `pio lib builtin `__ command -* Show detailed info about a library using `pio lib show `__ - command - (`issue #430 `_) -* List supported frameworks, SDKs with a new - `pio platform frameworks `__ command -* Visual Studio Code extension for PlatformIO - (`issue #619 `_) -* Added new options ``--no-reset``, ``--monitor-rts`` and ``--monitor-dtr`` - to `pio test `__ - command (allows to avoid automatic board's auto-reset when gathering test results) -* Added support for templated methods in ``*.ino to *.cpp`` converter - (`pull #858 `_) -* Package version as "Repository URL" in manifest of development version - (``"version": "https://github.com/user/repo.git"``) -* Produce less noisy output when ``-s/--silent`` options are used for - `platformio init `__ - and `platformio run `__ - commands - (`issue #850 `_) -* Use C++11 by default for CLion IDE based projects - (`pull #873 `_) -* Escape project path when Glob matching is used -* Do not overwrite project configuration variables when system environment - variables are set -* Handle dependencies when installing non-registry package/library (VCS, archive, local folder) - (`issue #913 `_) -* Fixed package installing with VCS branch for Python 2.7.3 - (`issue #885 `_) - -3.2.1 (2016-12-07) -~~~~~~~~~~~~~~~~~~ - -* Changed default `LDF Mode `__ - from ``chain+`` to ``chain`` - -3.2.0 (2016-12-07) -~~~~~~~~~~~~~~~~~~ - -* `PIO Remote™ `__. - **Your devices are always with you!** - - + Over-The-Air (OTA) Device Manager - + OTA Serial Port Monitor - + OTA Firmware Updates - + Continuous Deployment - + Continuous Delivery - -* Integration with `Cloud IDEs `__ - - + Cloud9 - + Codeanywhere - + Eclipse Che - -* `PIO Account `__ - and `PLATFORMIO_AUTH_TOKEN `__ - environment variable for CI systems - (`issue #808 `_, - `issue #467 `_) -* Inject system environment variables to configuration settings in - `"platformio.ini" (Project Configuration File) `__ - (`issue #792 `_) -* Custom boards per project with ``boards_dir`` option in - `"platformio.ini" (Project Configuration File) `__ - (`issue #515 `_) -* Unix shell-style wildcards for `upload_port `_ - (`issue #839 `_) -* Refactored `Library Dependency Finder (LDF) `__ - C/C++ Preprocessor for conditional syntax (``#ifdef``, ``#if``, ``#else``, - ``#elif``, ``#define``, etc.) - (`issue #837 `_) -* Added new `LDF Modes `__: - ``chain+`` and ``deep+`` and set ``chain+`` as default -* Added global ``lib_extra_dirs`` option to ``[platformio]`` section for - `"platformio.ini" (Project Configuration File) `__ - (`issue #842 `_) -* Enabled caching by default for API requests and Library Manager (see `enable_cache `__ setting) -* Native integration with VIM/Neovim using `neomake-platformio `__ plugin -* Changed a default exit combination for Device Monitor from ``Ctrl+]`` to ``Ctrl+C`` -* Improved detecting of ARM mbed media disk for uploading -* Improved Project Generator for CLion IDE when source folder contains nested items -* Improved handling of library dependencies specified in ``library.json`` manifest - (`issue #814 `_) -* Improved `Library Dependency Finder (LDF) `__ - for circular dependencies -* Show vendor version of a package for `platformio platform show `__ command - (`issue #838 `_) -* Fixed unable to include SSH user in ``lib_deps`` repository url - (`issue #830 `_) -* Fixed merging of ".gitignore" files when re-initialize project - (`issue #848 `_) -* Fixed issue with ``PATH`` auto-configuring for upload tools -* Fixed ``99-platformio-udev.rules`` checker for Linux OS - -3.1.0 (2016-09-19) -~~~~~~~~~~~~~~~~~~ - -* New! Dynamic variables/templates for `"platformio.ini" (Project Configuration File) `__ - (`issue #705 `_) -* Summary about processed environments - (`issue #777 `_) -* Implemented LocalCache system for API and improved a work in off-line mode -* Improved Project Generator when custom ``--project-option`` is passed to - `platformio init `__ - command -* Deprecated ``lib_force`` option, please use `lib_deps `__ instead -* Return valid exit code from ``plaformio test`` command -* Fixed Project Generator for CLion IDE using Windows OS - (`issue #785 `_) -* Fixed SSL Server-Name-Indication for Python < 2.7.9 - (`issue #774 `_) - -3.0.1 (2016-09-08) -~~~~~~~~~~~~~~~~~~ - -* Disabled temporary SSL for PlatformIO services - (`issue #772 `_) - -3.0.0 (2016-09-07) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO Plus `__ - - + Local and Embedded `Unit Testing `__ - (`issue #408 `_, - `issue #519 `_) - -* Decentralized Development Platforms - - + Development platform manifest "platform.json" and - `open source development platforms `__ - + `Semantic Versioning `__ for platform commands, - development platforms and dependent packages - + Custom package repositories - + External embedded board configuration files, isolated build scripts - (`issue #479 `_) - + Embedded Board compatibility with more than one development platform - (`issue #456 `_) - -* Library Manager 3.0 - - + Project dependencies per build environment using `lib_deps `__ option - (`issue #413 `_) - + `Semantic Versioning `__ for library commands and - dependencies - (`issue #410 `_) - + Multiple library storages: Project's Local, PlatformIO's Global or Custom - (`issue #475 `_) - + Install library by name - (`issue #414 `_) - + Depend on a library using VCS URL (GitHub, Git, ARM mbed code registry, Hg, SVN) - (`issue #498 `_) - + Strict search for library dependencies - (`issue #588 `_) - + Allowed ``library.json`` to specify sources other than PlatformIO's Repository - (`issue #461 `_) - + Search libraries by headers/includes with ``platformio lib search --header`` option - -* New Intelligent Library Build System - - + `Library Dependency Finder `__ - that interprets C/C++ Preprocessor conditional macros with deep search behavior - + Check library compatibility with project environment before building - (`issue #415 `_) - + Control Library Dependency Finder for compatibility using - `lib_compat_mode `__ - option - + Custom library storages/directories with - `lib_extra_dirs `__ option - (`issue #537 `_) - + Handle extra build flags, source filters and build script from - `library.json `__ - (`issue #289 `_) - + Allowed to disable library archiving (``*.ar``) - (`issue #719 `_) - + Show detailed build information about dependent libraries - (`issue #617 `_) - + Support for the 3rd party manifests (Arduino IDE "library.properties" - and ARM mbed "module.json") - -* Removed ``enable_prompts`` setting. Now, all PlatformIO CLI is non-blocking! -* Switched to SSL PlatformIO API -* Renamed ``platformio serialports`` command to ``platformio device`` -* Build System: Attach custom Before/Pre and After/Post actions for targets - (`issue #542 `_) -* Allowed passing custom project configuration options to ``platformio ci`` - and ``platformio init`` commands using ``-O, --project-option``. -* Print human-readable information when processing environments without - ``-v, --verbose`` option - (`issue #721 `_) -* Improved INO to CPP converter - (`issue #659 `_, - `issue #765 `_) -* Added ``license`` field to `library.json `__ - (`issue #522 `_) -* Warn about unknown options in project configuration file ``platformio.ini`` - (`issue #740 `_) -* Fixed wrong line number for INO file when ``#warning`` directive is used - (`issue #742 `_) -* Stopped supporting Python 2.6 +See `PlatformIO Core 3.0 history `__. PlatformIO Core 2 ----------------- -2.11.2 (2016-08-02) -~~~~~~~~~~~~~~~~~~~ - -* Improved support for `Microchip PIC32 `__ development platform and ChipKIT boards - (`issue #438 `_) -* Added support for Pinoccio Scout board - (`issue #52 `_) -* Added support for `Teensy USB Features `__ - (HID, SERIAL_HID, DISK, DISK_SDFLASH, MIDI, etc.) - (`issue #722 `_) -* Switched to built-in GCC LwIP library for Espressif development platform -* Added support for local ``--echo`` for Serial Port Monitor - (`issue #733 `_) -* Updated ``udev`` rules for the new STM32F407DISCOVERY boards - (`issue #731 `_) -* Implemented firmware merging with base firmware for Nordic nRF51 development platform - (`issue #500 `_, - `issue #533 `_) -* Fixed Project Generator for ESP8266 and ARM mbed based projects - (resolves incorrect linter errors) -* Fixed broken LD Script for Element14 chipKIT Pi board - (`issue #725 `_) -* Fixed firmware uploading to Atmel SAMD21-XPRO board using ARM mbed framework - (`issue #732 `_) - -2.11.1 (2016-07-12) -~~~~~~~~~~~~~~~~~~~ - -* Added support for Arduino M0, M0 Pro and Tian boards - (`issue #472 `_) -* Added support for Microchip chipKIT Lenny board -* Updated Microchip PIC32 Arduino framework to v1.2.1 -* Documented `uploading of EEPROM data `__ - (from EEMEM directive) -* Added ``Rebuild C/C++ Project Index`` target to CLion and Eclipse IDEs -* Improved project generator for `CLion IDE `__ -* Added ``udev`` rules for OpenOCD CMSIS-DAP adapters - (`issue #718 `_) -* Auto-remove project cache when PlatformIO is upgraded -* Keep user changes for ``.gitignore`` file when re-generate/update project data -* Ignore ``[platformio]`` section from custom project configuration file when - `platformio ci --project-conf `__ - command is used -* Fixed missed ``--boot`` flag for the firmware uploader for ATSAM3X8E - Cortex-M3 MCU based boards (Arduino Due, etc) - (`issue #710 `_) -* Fixed missing trailing ``\`` for the source files list when generate project - for `Qt Creator IDE `__ - (`issue #711 `_) -* Split source files to ``HEADERS`` and ``SOURCES`` when generate project - for `Qt Creator IDE `__ - (`issue #713 `_) - -2.11.0 (2016-06-28) -~~~~~~~~~~~~~~~~~~~ - -* New ESP8266-based boards: Generic ESP8285 Module, Phoenix 1.0 & 2.0, WifInfo -* Added support for Arduino M0 Pro board - (`issue #472 `_) -* Added support for Arduino MKR1000 board - (`issue #620 `_) -* Added support for Adafruit Feather M0, SparkFun SAMD21 and SparkFun SAMD21 - Mini Breakout boards - (`issue #520 `_) -* Updated Arduino ESP8266 core for Espressif platform to 2.3.0 -* Better removing unnecessary flags using ``build_unflags`` option - (`issue #698 `_) -* Fixed issue with ``platformio init --ide`` command for Python 2.6 - -2.10.3 (2016-06-15) -~~~~~~~~~~~~~~~~~~~ - -* Fixed issue with ``platformio init --ide`` command - -2.10.2 (2016-06-15) -~~~~~~~~~~~~~~~~~~~ - -* Added support for ST Nucleo L031K6 board to ARM mbed framework -* Process ``build_unflags`` option for ARM mbed framework -* Updated Intel ARC32 Arduino framework to v1.0.6 - (`issue #695 `_) -* Improved a check of program size before uploading to the board -* Fixed issue with ARM mbed framework ``-u _printf_float`` and - ``-u _scanf_float`` when parsing ``$LINKFLAGS`` -* Fixed issue with ARM mbed framework and extra includes for the custom boards, - such as Seeeduino Arch Pro - -2.10.1 (2016-06-13) -~~~~~~~~~~~~~~~~~~~ - -* Re-submit a package to PyPI - -2.10.0 (2016-06-13) -~~~~~~~~~~~~~~~~~~~ - -* Added support for `emonPi `__, - the OpenEnergyMonitor system - (`issue #687 `_) -* Added support for `SPL `__ - framework for STM32F0 boards - (`issue #683 `_) -* Added support for `Arduboy DevKit `__, the game system - the size of a credit card -* Updated ARM mbed framework package to v121 -* Check program size before uploading to the board - (`issue #689 `_) -* Improved firmware uploading to Arduino Leonardo based boards - (`issue #691 `_) -* Fixed issue with ``-L relative/path`` when parsing ``build_flags`` - (`issue #688 `_) - -2.9.4 (2016-06-04) -~~~~~~~~~~~~~~~~~~ - -* Show ``udev`` warning only for the Linux OS while uploading firmware - -2.9.3 (2016-06-03) -~~~~~~~~~~~~~~~~~~ - -* Added support for `Arduboy `__, the game system - the size of a credit card -* Updated `99-platformio-udev.rules `__ for Linux OS -* Refactored firmware uploading to the embedded boards with SAM-BA bootloader - -2.9.2 (2016-06-02) -~~~~~~~~~~~~~~~~~~ - -* Simplified `Continuous Integration with AppVeyor `__ - (`issue #671 `_) -* Automatically add source directory to ``CPPPATH`` of Build System -* Added support for Silicon Labs SLSTK3401A (Pearl Gecko) and - MultiTech mDot F411 ARM mbed based boards -* Added support for MightyCore ATmega8535 board - (`issue #585 `_) -* Added ``stlink`` as the default uploader for STM32 Discovery boards - (`issue #665 `_) -* Use HTTP mirror for Package Manager in a case with SSL errors - (`issue #645 `_) -* Improved firmware uploading to Arduino Leonardo/Due based boards -* Fixed bug with ``env_default`` when ``pio run -e`` is used -* Fixed issue with ``src_filter`` option for Windows OS - (`issue #652 `_) -* Fixed configuration data for TI LaunchPads based on msp430fr4133 and - msp430fr6989 MCUs - (`issue #676 `_) -* Fixed issue with ARM mbed framework and multiple definition errors - on FRDM-KL46Z board - (`issue #641 `_) -* Fixed issue with ARM mbed framework when abstract class breaks compile - for LPC1768 - (`issue #666 `_) - -2.9.1 (2016-04-30) -~~~~~~~~~~~~~~~~~~ - -* Handle prototype pointers while converting ``*.ino`` to ``.cpp`` - (`issue #639 `_) - -2.9.0 (2016-04-28) -~~~~~~~~~~~~~~~~~~ - -* Project generator for `CodeBlocks IDE `__ - (`issue #600 `_) -* New `Lattice iCE40 FPGA `__ - development platform with support for Lattice iCEstick FPGA Evaluation - Kit and BQ IceZUM Alhambra FPGA - (`issue #480 `_) -* New `Intel ARC 32-bit `_ - development platform with support for Arduino/Genuino 101 board - (`issue #535 `_) -* New `Microchip PIC32 `__ - development platform with support for 20+ different PIC32 based boards - (`issue #438 `_) -* New RTOS and build Framework named `Simba `__ - (`issue #412 `_) -* New boards for `ARM mbed `__ - framework: ST Nucleo F410RB, ST Nucleo L073RZ and BBC micro:bit -* Added support for Arduino.Org boards: Arduino Leonardo ETH, Arduino Yun Mini, - Arduino Industrial 101 and Linino One - (`issue #472 `_) -* Added support for Generic ATTiny boards: ATTiny13, ATTiny24, ATTiny25, - ATTiny45 and ATTiny85 - (`issue #636 `_) -* Added support for MightyCore boards: ATmega1284, ATmega644, ATmega324, - ATmega164, ATmega32, ATmega16 and ATmega8535 - (`issue #585 `_) -* Added support for `TI MSP430 `__ - boards: TI LaunchPad w/ msp430fr4133 and TI LaunchPad w/ msp430fr6989 -* Updated Arduino core for Espressif platform to 2.2.0 - (`issue #627 `_) -* Updated native SDK for ESP8266 to 1.5 - (`issue #366 `_) -* PlatformIO Library Registry in JSON format! Implemented - ``--json-output`` and ``--page`` options for - `platformio lib search `__ - command - (`issue #604 `_) -* Allowed to specify default environments `env_default `__ - which should be processed by default with ``platformio run`` command - (`issue #576 `_) -* Allowed to unflag(remove) base/initial flags using - `build_unflags `__ - option - (`issue #559 `_) -* Allowed multiple VID/PID pairs when detecting serial ports - (`issue #632 `_) -* Automatically add ``-DUSB_MANUFACTURER`` with vendor's name - (`issue #631 `_) -* Automatically reboot Teensy board after upload when Teensy Loader GUI is used - (`issue #609 `_) -* Refactored source code converter from ``*.ino`` to ``*.cpp`` - (`issue #610 `_) -* Forced ``-std=gnu++11`` for Atmel SAM development platform - (`issue #601 `_) -* Don't check OS type for ARM mbed-enabled boards and ST STM32 development - platform before uploading to disk - (`issue #596 `_) -* Fixed broken compilation for Atmel SAMD based boards except Arduino Due - (`issue #598 `_) -* Fixed firmware uploading using serial port with spaces in the path -* Fixed cache system when project's root directory is used as ``src_dir`` - (`issue #635 `_) - -2.8.6 (2016-03-22) -~~~~~~~~~~~~~~~~~~ - -* Launched `PlatformIO Community Forums `_ - (`issue #530 `_) -* Added support for ARM mbed-enabled board Seed Arch Max (STM32F407VET6) - (`issue #572 `_) -* Improved DNS lookup for PlatformIO API -* Updated Arduino Wiring-based framework to the latest version for - Atmel AVR/SAM development platforms -* Updated "Teensy Loader CLI" and fixed uploading of large .hex files - (`issue #568 `_) -* Updated the support for Sanguino Boards - (`issue #586 `_) -* Better handling of used boards when re-initialize/update project -* Improved support for non-Unicode user profiles for Windows OS -* Disabled progress bar for download operations when prompts are disabled -* Fixed multiple definition errors for ST STM32 development platform and - ARM mbed framework - (`issue #571 `_) -* Fixed invalid board parameters (reset method and baudrate) for a few - ESP8266 based boards -* Fixed "KeyError: 'content-length'" in PlatformIO Download Manager - (`issue #591 `_) - - -2.8.5 (2016-03-07) -~~~~~~~~~~~~~~~~~~ - -* Project generator for `NetBeans IDE `__ - (`issue #541 `_) -* Created package for Homebrew Mac OS X Package Manager: ``brew install - platformio`` - (`issue #395 `_) -* Updated Arduino core for Espressif platform to 2.1.0 - (`issue #544 `_) -* Added support for the ESP8266 ESP-07 board to - `Espressif `__ - (`issue #527 `_) -* Improved handling of String-based ``CPPDEFINES`` passed to extra ``build_flags`` - (`issue #526 `_) -* Generate appropriate project for CLion IDE and CVS - (`issue #523 `_) -* Use ``src_dir`` directory from `Project Configuration File platformio.ini `__ - when initializing project otherwise create base ``src`` directory - (`issue #536 `_) -* Fixed issue with incorrect handling of user's build flags where the base flags - were passed after user's flags to GCC compiler - (`issue #528 `_) -* Fixed issue with Project Generator when optional build flags were passed using - system environment variables: `PLATFORMIO_BUILD_FLAGS `__ - or `PLATFORMIO_BUILD_SRC_FLAGS `__ -* Fixed invalid detecting of compiler type - (`issue #550 `_) -* Fixed issue with updating package which was deleted manually by user - (`issue #555 `_) -* Fixed incorrect parsing of GCC ``-include`` flag - (`issue #552 `_) - -2.8.4 (2016-02-17) -~~~~~~~~~~~~~~~~~~ - -* Added support for the new ESP8266-based boards (ESPDuino, ESP-WROOM-02, - ESPresso Lite 1.0 & 2.0, SparkFun ESP8266 Thing Dev, ThaiEasyElec ESPino) to - `Espressif `__ - development platform -* Added ``board_f_flash`` option to `Project Configuration File platformio.ini `__ - which allows to specify `custom flash chip frequency `_ - for Espressif development platform - (`issue #501 `_) -* Added ``board_flash_mode`` option to `Project Configuration File platformio.ini `__ - which allows to specify `custom flash chip mode `_ - for Espressif development platform -* Handle new environment variables - `PLATFORMIO_UPLOAD_PORT `_ - and `PLATFORMIO_UPLOAD_FLAGS `_ - (`issue #518 `_) -* Fixed issue with ``CPPDEFINES`` which contain space and break PlatformIO - IDE Linter - (`IDE issue #34 `_) -* Fixed unable to link C++ standard library to Espressif platform build - (`issue #503 `_) -* Fixed issue with pointer (``char* myfunc()``) while converting from ``*.ino`` - to ``*.cpp`` - (`issue #506 `_) - -2.8.3 (2016-02-02) -~~~~~~~~~~~~~~~~~~ - -* Better integration of PlatformIO Builder with PlatformIO IDE Linter -* Fixed issue with removing temporary file while converting ``*.ino`` to - ``*.cpp`` -* Fixed missing dependency (mbed framework) for Atmel SAM development platform - (`issue #487 `_) - -2.8.2 (2016-01-29) -~~~~~~~~~~~~~~~~~~ - -* Corrected RAM size for NXP LPC1768 based boards - (`issue #484 `_) -* Exclude only ``test`` and ``tests`` folders from build process -* Reverted ``-Wl,-whole-archive`` hook for ST STM32 and mbed - -2.8.1 (2016-01-29) -~~~~~~~~~~~~~~~~~~ - -* Fixed a bug with Project Initialization in PlatformIO IDE - -2.8.0 (2016-01-29) -~~~~~~~~~~~~~~~~~~ - -* `PlatformIO IDE `_ for - Atom - (`issue #470 `_) -* Added ``pio`` command line alias for ``platformio`` command - (`issue #447 `_) -* Added SPL-Framework support for Nucleo F401RE board - (`issue #453 `_) -* Added ``upload_resetmethod`` option to `Project Configuration File platformio.ini `__ - which allows to specify `custom upload reset method `_ - for Espressif development platform - (`issue #444 `_) -* Allowed to force output of color ANSI-codes or to disable progress bar even - if the output is a ``pipe`` (not a ``tty``) using `Environment variables `__ - (`issue #465 `_) -* Set 1Mb SPIFFS for Espressif boards by default - (`issue #458 `_) -* Exclude ``test*`` folder by default from build process -* Generate project for IDEs with information about installed libraries -* Fixed builder for mbed framework and ST STM32 platform - - -2.7.1 (2016-01-06) -~~~~~~~~~~~~~~~~~~ - -* Initial support for Arduino Zero board - (`issue #356 `_) -* Added support for completions to Atom text editor using ``.clang_complete`` -* Generate default targets for `supported IDE `__ - (CLion, Eclipse IDE, Emacs, Sublime Text, VIM): Build, - Clean, Upload, Upload SPIFFS image, Upload using Programmer, Update installed - platforms and libraries - (`issue #427 `_) -* Updated Teensy Arduino Framework to 1.27 - (`issue #434 `_) -* Fixed uploading of EEPROM data using ``uploadeep`` target for Atmel AVR - development platform -* Fixed project generator for CLion IDE - (`issue #422 `_) -* Fixed package ``shasum`` validation on Mac OS X 10.11.2 - (`issue #429 `_) -* Fixed CMakeLists.txt ``add_executable`` has only one source file - (`issue #421 `_) - -2.7.0 (2015-12-30) -~~~~~~~~~~~~~~~~~~ - -**Happy New Year!** - -* Moved SCons to PlatformIO packages. PlatformIO does not require SCons to be - installed in your system. Significantly simplified installation process of - PlatformIO. ``pip install platformio`` rocks! -* Implemented uploading files to file system of ESP8266 SPIFFS (including OTA) - (`issue #382 `_) -* Added support for the new Adafruit boards Bluefruit Micro and Feather - (`issue #403 `_) -* Added support for RFDuino - (`issue #319 `_) -* Project generator for `Emacs `__ - text editor - (`pull #404 `_) -* Updated Arduino framework for Atmel AVR development platform to 1.6.7 -* Documented `firmware uploading for Atmel AVR development platform using - Programmers `_: - AVR ISP, AVRISP mkII, USBtinyISP, USBasp, Parallel Programmer and Arduino as ISP -* Fixed issue with current Python interpreter for Python-based tools - (`issue #417 `_) - -2.6.3 (2015-12-21) -~~~~~~~~~~~~~~~~~~ - -* Restored support for Espressif ESP8266 ESP-01 1MB board (ready for OTA) -* Fixed invalid ROM size for ESP8266-based boards - (`issue #396 `_) - -2.6.2 (2015-12-21) -~~~~~~~~~~~~~~~~~~ - -* Removed ``SCons`` from requirements list. PlatformIO will try to install it - automatically, otherwise users need to install it manually -* Fixed ``ChunkedEncodingError`` when SF connection is broken - (`issue #356 `_) - -2.6.1 (2015-12-18) -~~~~~~~~~~~~~~~~~~ - -* Added support for the new ESP8266-based boards (SparkFun ESP8266 Thing, - NodeMCU 0.9 & 1.0, Olimex MOD-WIFI-ESP8266(-DEV), Adafruit HUZZAH ESP8266, - ESPino, SweetPea ESP-210, WeMos D1, WeMos D1 mini) to - `Espressif `__ - development platform -* Created public `platformio-pkg-ldscripts `_ - repository for LD scripts. Moved common configuration for ESP8266 MCU to - ``esp8266.flash.common.ld`` - (`issue #379 `_) -* Improved documentation for `Espressif `__ - development platform: OTA update, custom Flash Size, Upload Speed and CPU - frequency -* Fixed reset method for Espressif NodeMCU (ESP-12E Module) - (`issue #380 `_) -* Fixed issue with code builder when build path contains spaces - (`issue #387 `_) -* Fixed project generator for Eclipse IDE and "duplicate path entries found - in project path" - (`issue #383 `_) - - -2.6.0 (2015-12-15) -~~~~~~~~~~~~~~~~~~ - -* Install only required packages depending on build environment - (`issue #308 `_) -* Added support for Raspberry Pi `WiringPi `__ - framework - (`issue #372 `_) -* Implemented Over The Air (OTA) upgrades for `Espressif `__ - development platform. - (`issue #365 `_) -* Updated `CMSIS framework `__ - and added CMSIS support for Nucleo F401RE board - (`issue #373 `_) -* Added support for Espressif ESP8266 ESP-01-1MB board (ready for OTA) -* Handle ``upload_flags`` option in `platformio.ini `__ - (`issue #368 `_) -* Improved PlatformIO installation on the Mac OS X El Capitan - -2.5.0 (2015-12-08) -~~~~~~~~~~~~~~~~~~ - -* Improved code builder for parallel builds (up to 4 times faster than before) -* Generate `.travis.yml `__ - CI and `.gitignore` files for embedded projects by default - (`issue #354 `_) -* Removed prompt with "auto-uploading" from `platformio init `__ - command and added ``--enable-auto-uploading`` option - (`issue #352 `_) -* Fixed incorrect behaviour of `platformio serialports monitor `__ - in pair with PySerial 3.0 - -2.4.1 (2015-12-01) -~~~~~~~~~~~~~~~~~~ - -* Restored ``PLATFORMIO`` macros with the current version - -2.4.0 (2015-12-01) -~~~~~~~~~~~~~~~~~~ - -* Added support for the new boards: Atmel ATSAMR21-XPRO, Atmel SAML21-XPRO-B, - Atmel SAMD21-XPRO, ST 32F469IDISCOVERY, ST 32L476GDISCOVERY, ST Nucleo F031K6, - ST Nucleo F042K6, ST Nucleo F303K8 and ST Nucleo L476RG -* Updated Arduino core for Espressif platform to 2.0.0 - (`issue #345 `_) -* Added to FAQ explanation of `Can not compile a library that compiles without issue - with Arduino IDE `_ - (`issue #331 `_) -* Fixed ESP-12E flash size - (`pull #333 `_) -* Fixed configuration for LowPowerLab MoteinoMEGA board - (`issue #335 `_) -* Fixed "LockFailed: failed to create appstate.json.lock" error for Windows -* Fixed relative include path for preprocessor using ``build_flags`` - (`issue #271 `_) - -2.3.5 (2015-11-18) -~~~~~~~~~~~~~~~~~~ - -* Added support of `libOpenCM3 `_ - framework for Nucleo F103RB board - (`issue #309 `_) -* Added support for Espressif ESP8266 ESP-12E board (NodeMCU) - (`issue #310 `_) -* Added support for pySerial 3.0 - (`issue #307 `_) -* Updated Arduino AVR/SAM frameworks to 1.6.6 - (`issue #321 `_) -* Upload firmware using external programmer via `platformio run --target program `__ - target - (`issue #311 `_) -* Fixed handling of upload port when ``board`` option is not specified in - `platformio.ini `__ - (`issue #313 `_) -* Fixed firmware uploading for `nordicrf51 `__ - development platform - (`issue #316 `_) -* Fixed installation on Mac OS X El Capitan - (`issue #312 `_) -* Fixed project generator for CLion IDE under Windows OS with invalid path to - executable - (`issue #326 `_) -* Fixed empty list with serial ports on Mac OS X - (`isge #294 `_) -* Fixed compilation error ``TWI_Disable not declared`` for Arduino Due board - (`issue #329 `_) - -2.3.4 (2015-10-13) -~~~~~~~~~~~~~~~~~~ - -* Full support of `CLion IDE `_ - including code auto-completion - (`issue #132 `_) -* PlatformIO `command completion in Terminal `_ for ``bash`` and ``zsh`` -* Added support for ubIQio Ardhat board - (`pull #302 `_) -* Install SCons automatically and avoid ``error: option --single-version-externally-managed not recognized`` - (`issue #279 `_) -* Use Teensy CLI Loader for upload of .hex files on Mac OS X - (`issue #306 `_) -* Fixed missing `framework-mbed `_ - package for `teensy `_ - platform - (`issue #305 `_) - -2.3.3 (2015-10-02) -~~~~~~~~~~~~~~~~~~ - -* Added support for LightBlue Bean board - (`pull #292 `_) -* Added support for ST Nucleo F446RE board - (`pull #293 `_) -* Fixed broken lock file for "appstate" storage - (`issue #288 `_) -* Fixed ESP8266 compile errors about RAM size when adding 1 library - (`issue #296 `_) - -2.3.2 (2015-09-10) -~~~~~~~~~~~~~~~~~~ - -* Allowed to use ST-Link uploader for mbed-based projects -* Explained how to use ``lib`` directory from the PlatformIO based project in - ``readme.txt`` which will be automatically generated using - `platformio init `__ - command - (`issue #273 `_) -* Found solution for "pip/scons error: option --single-version-externally-managed not - recognized" when install PlatformIO using ``pip`` package manager - (`issue #279 `_) -* Fixed firmware uploading to Arduino Leonardo board using Mac OS - (`issue #287 `_) -* Fixed `SConsNotInstalled` error for Linux Debian-based distributives - -2.3.1 (2015-09-06) -~~~~~~~~~~~~~~~~~~ - -* Fixed critical issue when `platformio init --ide `__ command hangs PlatformIO - (`issue #283 `_) - -2.3.0 (2015-09-05) -~~~~~~~~~~~~~~~~~~ - -* Added - `native `__, - `linux_arm `__, - `linux_i686 `__, - `linux_x86_64 `__, - `windows_x86 `__ - development platforms - (`issue #263 `_) -* Added `PlatformIO Demo `_ - page to documentation -* Simplified `installation `__ - process of PlatformIO - (`issue #274 `_) -* Significantly improved `Project Generator `__ which allows to integrate with `the most popular - IDE `__ -* Added short ``-h`` help option for PlatformIO and sub-commands -* Updated `mbed `__ - framework -* Updated ``tool-teensy`` package for `Teensy `__ - platform - (`issue #268 `_) -* Added FAQ answer when `Program "platformio" not found in PATH `_ - (`issue #272 `_) -* Generate "readme.txt" for project "lib" directory - (`issue #273 `_) -* Use toolchain's includes pattern ``include*`` for Project Generator - (`issue #277 `_) -* Added support for Adafruit Gemma board to - `atmelavr `__ - platform - (`pull #256 `_) -* Fixed includes list for Windows OS when generating project for `Eclipse IDE `__ - (`issue #270 `_) -* Fixed ``AttributeError: 'module' object has no attribute 'packages'`` - (`issue #252 `_) - -2.2.2 (2015-07-30) -~~~~~~~~~~~~~~~~~~ - -* Integration with `Atom IDE `__ -* Support for off-line/unpublished/private libraries - (`issue #260 `_) -* Disable project auto-clean while building/uploading firmware using - `platformio run --disable-auto-clean `_ option - (`issue #255 `_) -* Show internal errors from "Miniterm" using `platformio serialports monitor `__ command - (`issue #257 `_) -* Fixed `platformio serialports monitor --help `__ information with HEX char for hotkeys - (`issue #253 `_) -* Handle "OSError: [Errno 13] Permission denied" for PlatformIO installer script - (`issue #254 `_) - -2.2.1 (2015-07-17) -~~~~~~~~~~~~~~~~~~ - -* Project generator for `CLion IDE `__ - (`issue #132 `_) -* Updated ``tool-bossac`` package to 1.5 version for `atmelsam `__ platform - (`issue #251 `_) -* Updated ``sdk-esp8266`` package for `espressif `__ platform -* Fixed incorrect arguments handling for `platformio serialports monitor `_ command - (`issue #248 `_) - -2.2.0 (2015-07-01) -~~~~~~~~~~~~~~~~~~ - -* Allowed to exclude/include source files from build process using - `src_filter `__ - (`issue #240 `_) -* Launch own extra script before firmware building/uploading processes - (`issue #239 `_) -* Specify own path to the linker script (ld) using - `build_flags `__ - option - (`issue #233 `_) -* Specify library compatibility with the all platforms/frameworks - using ``*`` symbol in - `library.json `__ -* Added support for new embedded boards: *ST 32L0538DISCOVERY and Delta DFCM-NNN40* - to `Framework mbed `__ -* Updated packages for - `Framework Arduino (AVR, SAM, Espressif and Teensy cores `__, - `Framework mbed `__, - `Espressif ESP8266 SDK `__ - (`issue #246 `_) -* Fixed ``stk500v2_command(): command failed`` - (`issue #238 `_) -* Fixed IDE project generator when board is specified - (`issue #242 `_) -* Fixed relative path for includes when generating project for IDE - (`issue #243 `_) -* Fixed ESP8266 native SDK exception - (`issue #245 `_) - -2.1.2 (2015-06-21) -~~~~~~~~~~~~~~~~~~ - -* Fixed broken link to SCons installer - -2.1.1 (2015-06-09) -~~~~~~~~~~~~~~~~~~ - -* Automatically detect upload port using VID:PID board settings - (`issue #231 `_) -* Improved detection of build changes -* Avoided ``LibInstallDependencyError`` when more than 1 library is found - (`issue #229 `_) - -2.1.0 (2015-06-03) -~~~~~~~~~~~~~~~~~~ - -* Added Silicon Labs EFM32 `siliconlabsefm32 `_ - development platform - (`issue #226 `_) -* Integrate PlatformIO with `Circle CI `_ and - `Shippable CI `_ -* Described in documentation how to `create/register own board `_ for PlatformIO -* Disabled "nano.specs" for ARM-based platforms - (`issue #219 `_) -* Fixed "ConnectionError" when PlatformIO SF Storage is off-line -* Fixed resolving of C/C++ std libs by Eclipse IDE - (`issue #220 `_) -* Fixed firmware uploading using USB programmer (USBasp) for - `atmelavr `_ - platform - (`issue #221 `_) - -2.0.2 (2015-05-27) -~~~~~~~~~~~~~~~~~~ - -* Fixed libraries order for "Library Dependency Finder" under Linux OS - -2.0.1 (2015-05-27) -~~~~~~~~~~~~~~~~~~ - -* Handle new environment variable - `PLATFORMIO_BUILD_FLAGS `_ -* Pass to API requests information about Continuous Integration system. This - information will be used by PlatformIO-API. -* Use ``include`` directories from toolchain when initialising project for IDE - (`issue #210 `_) -* Added support for new WildFire boards from - `Wicked Device `_ to - `atmelavr `__ - platform -* Updated `Arduino Framework `__ to - 1.6.4 version (`issue #212 `_) -* Handle Atmel AVR Symbols when initialising project for IDE - (`issue #216 `_) -* Fixed bug with converting ``*.ino`` to ``*.cpp`` -* Fixed failing with ``platformio init --ide eclipse`` without boards - (`issue #217 `_) - -2.0.0 (2015-05-22) -~~~~~~~~~~~~~~~~~~ - -*Made in* `Paradise `_ - -* PlatformIO as `Continuous Integration `_ - (CI) tool for embedded projects - (`issue #108 `_) -* Initialise PlatformIO project for the specified IDE - (`issue #151 `_) -* PlatformIO CLI 2.0: "platform" related commands have been - moved to ``platformio platforms`` subcommand - (`issue #158 `_) -* Created `PlatformIO gitter.im `_ room - (`issue #174 `_) -* Global ``-f, --force`` option which will force to accept any - confirmation prompts - (`issue #152 `_) -* Run project with `platformio run --project-dir `_ option without changing the current working - directory - (`issue #192 `_) -* Control verbosity of `platformio run `_ command via ``-v/--verbose`` option -* Add library dependencies for build environment using - `lib_install `_ - option in ``platformio.ini`` - (`issue #134 `_) -* Specify libraries which are compatible with build environment using - `lib_use `_ - option in ``platformio.ini`` - (`issue #148 `_) -* Add more boards to PlatformIO project with - `platformio init --board `__ - command - (`issue #167 `_) -* Choose which library to update - (`issue #168 `_) -* Specify `platformio init --env-prefix `__ when initialise/update project - (`issue #182 `_) -* Added new Armstrap boards - (`issue #204 `_) -* Updated SDK for `espressif `__ - development platform to v1.1 - (`issue #179 `_) -* Disabled automatic updates by default for platforms, packages and libraries - (`issue #171 `_) -* Fixed bug with creating copies of source files - (`issue #177 `_) +See `PlatformIO Core 2.0 history `__. PlatformIO Core 1 ----------------- -1.5.0 (2015-05-15) -~~~~~~~~~~~~~~~~~~ - -* Added support of `Framework mbed `_ - for Teensy 3.1 - (`issue #183 `_) -* Added GDB as alternative uploader to `ststm32 `__ platform - (`issue #175 `_) -* Added `examples `__ - with preconfigured IDE projects - (`issue #154 `_) -* Fixed firmware uploading under Linux OS for Arduino Leonardo board - (`issue #178 `_) -* Fixed invalid "mbed" firmware for Nucleo F411RE - (`issue #185 `_) -* Fixed parsing of includes for PlatformIO Library Dependency Finder - (`issue #189 `_) -* Fixed handling symbolic links within source code directory - (`issue #190 `_) -* Fixed cancelling any previous definition of name, either built in or provided - with a ``-D`` option - (`issue #191 `_) - -1.4.0 (2015-04-11) -~~~~~~~~~~~~~~~~~~ - -* Added `espressif `_ - development platform with ESP01 board -* Integrated PlatformIO with AppVeyor Windows based Continuous Integration system - (`issue #149 `_) -* Added support for Teensy LC board to - `teensy `__ - platform -* Added support for new Arduino based boards by *SparkFun, BQ, LightUp, - LowPowerLab, Quirkbot, RedBearLab, TinyCircuits* to - `atmelavr `__ - platform -* Upgraded `Arduino Framework `__ to - 1.6.3 version (`issue #156 `_) -* Upgraded `Energia Framework `__ to - 0101E0015 version (`issue #146 `_) -* Upgraded `Arduino Framework with Teensy Core `_ - to 1.22 version - (`issue #162 `_, - `issue #170 `_) -* Fixed exceptions with PlatformIO auto-updates when Internet connection isn't - active - - -1.3.0 (2015-03-27) -~~~~~~~~~~~~~~~~~~ - -* Moved PlatformIO source code and repositories from `Ivan Kravets `_ - account to `PlatformIO Organisation `_ - (`issue #138 `_) -* Added support for new Arduino based boards by *SparkFun, RepRap, Sanguino* to - `atmelavr `__ - platform - (`issue #127 `_, - `issue #131 `_) -* Added integration instructions for `Visual Studio `_ - and `Sublime Text `_ IDEs -* Improved handling of multi-file ``*.ino/pde`` sketches - (`issue #130 `_) -* Fixed wrong insertion of function prototypes converting ``*.ino/pde`` - (`issue #137 `_, - `issue #140 `_) - - - -1.2.0 (2015-03-20) -~~~~~~~~~~~~~~~~~~ - -* Added full support of `mbed `__ - framework including libraries: *RTOS, Ethernet, DSP, FAT, USB*. -* Added `freescalekinetis `_ - development platform with Freescale Kinetis Freedom boards -* Added `nordicnrf51 `_ - development platform with supported boards from *JKSoft, Nordic, RedBearLab, - Switch Science* -* Added `nxplpc `_ - development platform with supported boards from *CQ Publishing, Embedded - Artists, NGX Technologies, NXP, Outrageous Circuits, SeeedStudio, - Solder Splash Labs, Switch Science, u-blox* -* Added support for *ST Nucleo* boards to - `ststm32 `__ - development platform -* Created new `Frameworks `__ - page in documentation and added to `PlatformIO Web Site `_ - (`issue #115 `_) -* Introduced online `Embedded Boards Explorer `_ -* Automatically append define ``-DPLATFORMIO=%version%`` to - builder (`issue #105 `_) -* Renamed ``stm32`` development platform to - `ststm32 `__ -* Renamed ``opencm3`` framework to - `libopencm3 `__ -* Fixed uploading for `atmelsam `__ - development platform -* Fixed re-arranging the ``*.ino/pde`` files when converting to ``*.cpp`` - (`issue #100 `_) - -1.1.0 (2015-03-05) -~~~~~~~~~~~~~~~~~~ - -* Implemented ``PLATFORMIO_*`` environment variables - (`issue #102 `_) -* Added support for *SainSmart* boards to - `atmelsam `__ - development platform -* Added - `Project Configuration `__ - option named `envs_dir `__ -* Disabled "prompts" automatically for *Continuous Integration* systems - (`issue #103 `_) -* Fixed firmware uploading for - `atmelavr `__ - boards which work within ``usbtiny`` protocol -* Fixed uploading for *Digispark* board (`issue #106 `_) - -1.0.1 (2015-02-27) -~~~~~~~~~~~~~~~~~~ - -**PlatformIO 1.0 - recommended for production** - -* Changed development status from ``beta`` to ``Production/Stable`` -* Added support for *ARM*-based credit-card sized computers: - `Raspberry Pi `_, - `BeagleBone `_ and `CubieBoard `_ -* Added `atmelsam `__ - development platform with supported boards: *Arduino Due and Digistump DigiX* - (`issue #71 `_) -* Added `ststm32 `__ - development platform with supported boards: *Discovery kit for STM32L151/152, - STM32F303xx, STM32F407/417 lines* and `libOpenCM3 Framework `_ - (`issue #73 `_) -* Added `teensy `_ - development platform with supported boards: *Teensy 2.x & 3.x* - (`issue #72 `_) -* Added new *Arduino* boards to - `atmelavr `__ - platform: *Arduino NG, Arduino BT, Arduino Esplora, Arduino Ethernet, - Arduino Robot Control, Arduino Robot Motor and Arduino Yun* -* Added support for *Adafruit* boards to - `atmelavr `__ - platform: *Adafruit Flora and Adafruit Trinkets* - (`issue #65 `_) -* Added support for *Digispark* boards to - `atmelavr `__ - platform: *Digispark USB Development Board and Digispark Pro* - (`issue #47 `_) -* Covered code with tests (`issue #2 `_) -* Refactored *Library Dependency Finder* (issues - `#48 `_, - `#50 `_, - `#55 `_) -* Added `src_dir `__ - option to ``[platformio]`` section of - `platformio.ini `__ - which allows to redefine location to project's source directory - (`issue #83 `_) -* Added ``--json-output`` option to - `platformio boards `__ - and `platformio search `__ - commands which allows to return the output in `JSON `_ format - (`issue #42 `_) -* Allowed to ignore some libs from *Library Dependency Finder* via - `lib_ignore `_ option -* Improved `platformio run `__ - command: asynchronous output for build process, timing and detailed - information about environment configuration - (`issue #74 `_) -* Output compiled size and static memory usage with - `platformio run `__ - command (`issue #59 `_) -* Updated `framework-arduino` AVR & SAM to 1.6 stable version -* Fixed an issue with the libraries that are git repositories - (`issue #49 `_) -* Fixed handling of assembly files - (`issue #58 `_) -* Fixed compiling error if space is in user's folder - (`issue #56 `_) -* Fixed `AttributeError: 'module' object has no attribute 'disable_warnings'` - when a version of `requests` package is less then 2.4.0 -* Fixed bug with invalid process's "return code" when PlatformIO has internal - error (`issue #81 `_) -* Several bug fixes, increased stability and performance improvements +See `PlatformIO Core 1.0 history `__. PlatformIO Core Preview ----------------------- -0.10.2 (2015-01-06) -~~~~~~~~~~~~~~~~~~~ - -* Fixed an issue with ``--json-output`` - (`issue #42 `_) -* Fixed an exception during - `platformio upgrade `__ - under Windows OS (`issue #45 `_) - -0.10.1 (2015-01-02) -~~~~~~~~~~~~~~~~~~~ - -* Added ``--json-output`` option to - `platformio list `__, - `platformio serialports list `__ and - `platformio lib list `__ - commands which allows to return the output in `JSON `_ format - (`issue #42 `_) -* Fixed missing auto-uploading by default after `platformio init `__ - command - -0.10.0 (2015-01-01) -~~~~~~~~~~~~~~~~~~~ - -**Happy New Year!** - -* Implemented `platformio boards `_ - command (`issue #11 `_) -* Added support of *Engduino* boards for - `atmelavr `__ - platform (`issue #38 `_) -* Added ``--board`` option to `platformio init `__ - command which allows to initialise project with the specified embedded boards - (`issue #21 `_) -* Added `example with uploading firmware `_ - via USB programmer (USBasp) for - `atmelavr `_ - *MCUs* (`issue #35 `_) -* Automatic detection of port on `platformio serialports monitor `_ - (`issue #37 `_) -* Allowed auto-installation of platforms when prompts are disabled (`issue #43 `_) -* Fixed urllib3's *SSL* warning under Python <= 2.7.2 (`issue #39 `_) -* Fixed bug with *Arduino USB* boards (`issue #40 `_) - -0.9.2 (2014-12-10) -~~~~~~~~~~~~~~~~~~ - -* Replaced "dark blue" by "cyan" colour for the texts (`issue #33 `_) -* Added new setting ``enable_prompts`` and allowed to disable all *PlatformIO* prompts (useful for cloud compilers) - (`issue #34 `_) -* Fixed compilation bug on *Windows* with installed *MSVC* (`issue #18 `_) - -0.9.1 (2014-12-05) -~~~~~~~~~~~~~~~~~~ - -* Ask user to install platform (when it hasn't been installed yet) within - `platformio run `__ - and `platformio show `_ commands -* Improved main `documentation `_ -* Fixed "*OSError: [Errno 2] No such file or directory*" within - `platformio run `__ - command when PlatformIO isn't installed properly -* Fixed example for Eclipse IDE with Tiva board - (`issue #32 `_) -* Upgraded Eclipse Project Examples - to latest *Luna* and *PlatformIO* releases - -0.9.0 (2014-12-01) -~~~~~~~~~~~~~~~~~~ - -* Implemented `platformio settings `_ command -* Improved `platformio init `_ command. - Added new option ``--project-dir`` where you can specify another path to - directory where new project will be initialized (`issue #31 `_) -* Added *Migration Manager* which simplifies process with upgrading to a - major release -* Added *Telemetry Service* which should help us make *PlatformIO* better -* Implemented *PlatformIO AppState Manager* which allow to have multiple - ``.platformio`` states. -* Refactored *Package Manager* -* Download Manager: fixed SHA1 verification within *Cygwin Environment* - (`issue #26 `_) -* Fixed bug with code builder and built-in Arduino libraries - (`issue #28 `_) - -0.8.0 (2014-10-19) -~~~~~~~~~~~~~~~~~~ - -* Avoided trademark issues in `library.json `_ - with the new fields: `frameworks `_, - `platforms `_ - and `dependencies `_ - (`issue #17 `_) -* Switched logic from "Library Name" to "Library Registry ID" for all - `platformio lib `_ - commands (install, uninstall, update and etc.) -* Renamed ``author`` field to `authors `_ - and allowed to setup multiple authors per library in `library.json `_ -* Added option to specify "maintainer" status in `authors `_ field -* New filters/options for `platformio lib search `_ - command: ``--framework`` and ``--platform`` - -0.7.1 (2014-10-06) -~~~~~~~~~~~~~~~~~~ - -* Fixed bug with order for includes in conversation from INO/PDE to CPP -* Automatic detection of port on upload (`issue #15 `_) -* Fixed lib update crashing when no libs are installed (`issue #19 `_) - - -0.7.0 (2014-09-24) -~~~~~~~~~~~~~~~~~~ - -* Implemented new `[platformio] `_ - section for Configuration File with `home_dir `_ - option (`issue #14 `_) -* Implemented *Library Manager* (`issue #6 `_) - -0.6.0 (2014-08-09) -~~~~~~~~~~~~~~~~~~ - -* Implemented `platformio serialports monitor `_ (`issue #10 `_) -* Fixed an issue ``ImportError: No module named platformio.util`` (`issue #9 `_) -* Fixed bug with auto-conversation from Arduino \*.ino to \*.cpp - -0.5.0 (2014-08-04) -~~~~~~~~~~~~~~~~~~ - -* Improved nested lookups for libraries -* Disabled default warning flag "-Wall" -* Added auto-conversation from \*.ino to valid \*.cpp for Arduino/Energia - frameworks (`issue #7 `_) -* Added `Arduino example `_ - with external library (*Adafruit CC3000*) -* Implemented `platformio upgrade `_ - command and "auto-check" for the latest - version (`issue #8 `_) -* Fixed an issue with "auto-reset" for *Raspduino* board -* Fixed a bug with nested libs building - -0.4.0 (2014-07-31) -~~~~~~~~~~~~~~~~~~ - -* Implemented `platformio serialports `_ command -* Allowed to put special build flags only for ``src`` files via - `src_build_flags `_ - environment option -* Allowed to override some of settings via system environment variables - such as: ``PLATFORMIO_SRC_BUILD_FLAGS`` and ``PLATFORMIO_ENVS_DIR`` -* Added ``--upload-port`` option for `platformio run `__ command -* Implemented (especially for `SmartAnthill `_) - `platformio run -t uploadlazy `_ - target (no dependencies to framework libs, ELF and etc.) -* Allowed to skip default packages via `platformio install --skip-default-package `_ - option -* Added tools for *Raspberry Pi* platform -* Added support for *Microduino* and *Raspduino* boards in - `atmelavr `_ platform - -0.3.1 (2014-06-21) -~~~~~~~~~~~~~~~~~~ - -* Fixed auto-installer for Windows OS (bug with %PATH% custom installation) - - -0.3.0 (2014-06-21) -~~~~~~~~~~~~~~~~~~ - -* Allowed to pass multiple "SomePlatform" to install/uninstall commands -* Added "IDE Integration" section to README with Eclipse project examples -* Created auto installer script for *PlatformIO* (`issue #3 `_) -* Added "Super-Quick" way to Installation section (README) -* Implemented "build_flags" option for environments (`issue #4 `_) - - -0.2.0 (2014-06-15) -~~~~~~~~~~~~~~~~~~ - -* Resolved `issue #1 "Build referred libraries" `_ -* Renamed project's "libs" directory to "lib" -* Added `arduino-internal-library `_ example -* Changed to beta status - - -0.1.0 (2014-06-13) -~~~~~~~~~~~~~~~~~~ - -* Birth! First alpha release +See `PlatformIO Core Preview history `__. diff --git a/Makefile b/Makefile index 36b5d3963d..57aa76c40b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ lint: - pylint --rcfile=./.pylintrc ./platformio + pylint -j 6 --rcfile=./.pylintrc ./platformio + pylint -j 6 --rcfile=./.pylintrc ./tests isort: isort -rc ./platformio @@ -27,7 +28,7 @@ clean: clean-docs profile: # Usage $ > make PIOARGS="boards" profile - python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS} + python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS} snakeviz .tox/.tmp/cprofile.prof publish: diff --git a/README.rst b/README.rst index 44c251e0f1..bdd2858b7f 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,9 @@ PlatformIO .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png :target: https://platformio.org?utm_source=github&utm_medium=core -`PlatformIO `_ a new generation ecosystem for embedded development +`PlatformIO `_ is a professional collaborative platform for embedded development + +**A place where Developers and Teams have true Freedom! No more vendor lock-in!** * Open source, maximum permissive Apache 2.0 license * Cross-platform IDE and Unified Debugger @@ -64,10 +66,10 @@ Instruments Professional ------------ -* `PIO Check `_ -* `PIO Remote `_ -* `PIO Unified Debugger `_ -* `PIO Unit Testing `_ +* `Debugging `_ +* `Unit Testing `_ +* `Static Code Analysis `_ +* `Remote Development `_ Registry -------- @@ -81,6 +83,7 @@ Development Platforms --------------------- * `Aceinna IMU `_ +* `ASR Microelectronics ASR605x `_ * `Atmel AVR `_ * `Atmel SAM `_ * `Espressif 32 `_ @@ -144,7 +147,6 @@ Share minimal diagnostics and usage information to help us make PlatformIO bette It is enabled by default. For more information see: * `Telemetry Setting `_ -* `SSL Setting `_ License ------- diff --git a/docs b/docs index 683415246b..03a83c996f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 683415246be491a91c2f8e63fa46b0e6ab55f91b +Subproject commit 03a83c996f0c209ce0faaa2bcc285447a7780500 diff --git a/examples b/examples index 7793b677f7..84855946ea 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 7793b677f72ce3c3e9ed92b7915859ca2bfa313f +Subproject commit 84855946ea09b5e41ddbbae455f00e897060346d diff --git a/platformio/__init__.py b/platformio/__init__.py index 1e2e3fd173..74707d9cd2 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,18 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 3, 4) +import sys + +VERSION = (5, 0, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" __description__ = ( - "A new generation ecosystem for embedded development. " + "A professional collaborative platform for embedded development. " "Cross-platform IDE and Unified Debugger. " "Static Code Analyzer and Remote Unit Testing. " "Multi-platform and Multi-architecture Build System. " "Firmware File Explorer and Memory Inspection. " - "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, " - "RISC-V, FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3" + "IoT, Arduino, CMSIS, ESP-IDF, FreeRTOS, libOpenCM3, mbedOS, Pulp OS, SPL, " + "STM32Cube, Zephyr RTOS, ARM, AVR, Espressif (ESP8266/ESP32), FPGA, " + "MCS-51 (8051), MSP430, Nordic (nRF51/nRF52), NXP i.MX RT, PIC32, RISC-V, " + "STMicroelectronics (STM8/STM32), Teensy" ) __url__ = "https://platformio.org" @@ -33,6 +37,29 @@ __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" -__apiurl__ = "https://api.platformio.org" -__pioaccount_api__ = "https://api.accounts.platformio.org" +__accounts_api__ = "https://api.accounts.platformio.org" +__registry_api__ = [ + "https://api.registry.platformio.org", + "https://api.registry.ns1.platformio.org", +] __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" + +__default_requests_timeout__ = (10, None) # (connect, read) + +__core_packages__ = { + "contrib-piohome": "~3.3.0", + "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), + "tool-unity": "~1.20500.0", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40001.0", + "tool-cppcheck": "~1.210.0", + "tool-clangtidy": "~1.100000.0", + "tool-pvs-studio": "~7.9.0", +} + +__check_internet_hosts__ = [ + "140.82.118.3", # Github.com + "35.231.145.151", # Gitlab.com + "88.198.170.159", # platformio.org + "github.com", + "platformio.org", +] diff --git a/platformio/app.py b/platformio/app.py index 6c7c7b1ad0..0196fac41e 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -12,27 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import codecs +from __future__ import absolute_import + import getpass import hashlib +import json import os import platform import socket import uuid -from os import environ, getenv, listdir, remove from os.path import dirname, isdir, isfile, join, realpath -from time import time - -import requests -from platformio import __version__, exception, fs, lockfile +from platformio import __version__, exception, fs, proc from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data -from platformio.proc import is_ci -from platformio.project.helpers import ( - get_default_projects_dir, - get_project_cache_dir, - get_project_core_dir, -) +from platformio.package.lockfile import LockFile +from platformio.project.helpers import get_default_projects_dir, get_project_core_dir def projects_dir_validate(projects_dir): @@ -62,10 +56,9 @@ def projects_dir_validate(projects_dir): "value": 7, }, "enable_cache": { - "description": "Enable caching for API requests and Library Manager", + "description": "Enable caching for HTTP API requests", "value": True, }, - "strict_ssl": {"description": "Strict SSL for PlatformIO Services", "value": False}, "enable_telemetry": { "description": ("Telemetry service (Yes/No)"), "value": True, @@ -75,7 +68,7 @@ def projects_dir_validate(projects_dir): "value": False, }, "projects_dir": { - "description": "Default location for PlatformIO projects (PIO Home)", + "description": "Default location for PlatformIO projects (PlatformIO Home)", "value": get_default_projects_dir(), "validator": projects_dir_validate, }, @@ -126,7 +119,7 @@ def __exit__(self, type_, value, traceback): def _lock_state_file(self): if not self.lock: return - self._lockfile = lockfile.LockFile(self.path) + self._lockfile = LockFile(self.path) try: self._lockfile.acquire() except IOError: @@ -144,6 +137,9 @@ def __del__(self): def as_dict(self): return self._storage + def keys(self): + return self._storage.keys() + def get(self, key, default=True): return self._storage.get(key, default) @@ -169,146 +165,6 @@ def __contains__(self, item): return item in self._storage -class ContentCache(object): - def __init__(self, cache_dir=None): - self.cache_dir = None - self._db_path = None - self._lockfile = None - - self.cache_dir = cache_dir or get_project_cache_dir() - self._db_path = join(self.cache_dir, "db.data") - - def __enter__(self): - self.delete() - return self - - def __exit__(self, type_, value, traceback): - pass - - def _lock_dbindex(self): - if not self.cache_dir: - os.makedirs(self.cache_dir) - self._lockfile = lockfile.LockFile(self.cache_dir) - try: - self._lockfile.acquire() - except: # pylint: disable=bare-except - return False - - return True - - def _unlock_dbindex(self): - if self._lockfile: - self._lockfile.release() - return True - - def get_cache_path(self, key): - assert "/" not in key and "\\" not in key - key = str(key) - assert len(key) > 3 - return join(self.cache_dir, key[-2:], key) - - @staticmethod - def key_from_args(*args): - h = hashlib.md5() - for arg in args: - if arg: - h.update(hashlib_encode_data(arg)) - return h.hexdigest() - - def get(self, key): - cache_path = self.get_cache_path(key) - if not isfile(cache_path): - return None - with codecs.open(cache_path, "rb", encoding="utf8") as fp: - return fp.read() - - def set(self, key, data, valid): - if not get_setting("enable_cache"): - return False - cache_path = self.get_cache_path(key) - if isfile(cache_path): - self.delete(key) - if not data: - return False - if not isdir(self.cache_dir): - os.makedirs(self.cache_dir) - tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} - assert valid.endswith(tuple(tdmap)) - expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) - - if not self._lock_dbindex(): - return False - - if not isdir(dirname(cache_path)): - os.makedirs(dirname(cache_path)) - try: - with codecs.open(cache_path, "wb", encoding="utf8") as fp: - fp.write(data) - with open(self._db_path, "a") as fp: - fp.write("%s=%s\n" % (str(expire_time), cache_path)) - except UnicodeError: - if isfile(cache_path): - try: - remove(cache_path) - except OSError: - pass - - return self._unlock_dbindex() - - def delete(self, keys=None): - """ Keys=None, delete expired items """ - if not isfile(self._db_path): - return None - if not keys: - keys = [] - if not isinstance(keys, list): - keys = [keys] - paths_for_delete = [self.get_cache_path(k) for k in keys] - found = False - newlines = [] - with open(self._db_path) as fp: - for line in fp.readlines(): - line = line.strip() - if "=" not in line: - continue - expire, path = line.split("=") - try: - if ( - time() < int(expire) - and isfile(path) - and path not in paths_for_delete - ): - newlines.append(line) - continue - except ValueError: - pass - found = True - if isfile(path): - try: - remove(path) - if not listdir(dirname(path)): - fs.rmtree(dirname(path)) - except OSError: - pass - - if found and self._lock_dbindex(): - with open(self._db_path, "w") as fp: - fp.write("\n".join(newlines) + "\n") - self._unlock_dbindex() - - return True - - def clean(self): - if not self.cache_dir or not isdir(self.cache_dir): - return - fs.rmtree(self.cache_dir) - - -def clean_cache(): - with ContentCache() as cc: - cc.clean() - - def sanitize_setting(name, value): if name not in DEFAULT_SETTINGS: raise exception.InvalidSettingName(name) @@ -346,8 +202,8 @@ def delete_state_item(name): def get_setting(name): _env_name = "PLATFORMIO_SETTING_%s" % name.upper() - if _env_name in environ: - return sanitize_setting(name, getenv(_env_name)) + if _env_name in os.environ: + return sanitize_setting(name, os.getenv(_env_name)) with State() as state: if "settings" in state and name in state["settings"]: @@ -383,31 +239,32 @@ def is_disabled_progressbar(): return any( [ get_session_var("force_option"), - is_ci(), - getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", + proc.is_ci(), + os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true", ] ) def get_cid(): + # pylint: disable=import-outside-toplevel + from platformio.clients.http import fetch_remote_content + cid = get_state_item("cid") if cid: return cid uid = None - if getenv("C9_UID"): - uid = getenv("C9_UID") - elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): + if os.getenv("C9_UID"): + uid = os.getenv("C9_UID") + elif os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")): try: - uid = ( - requests.get( + uid = json.loads( + fetch_remote_content( "{api}/user?token={token}".format( - api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), - token=getenv("USER_TOKEN"), + api=os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")), + token=os.getenv("USER_TOKEN"), ) ) - .json() - .get("id") - ) + ).get("id") except: # pylint: disable=bare-except pass if not uid: @@ -420,7 +277,11 @@ def get_cid(): def get_user_agent(): - data = ["PlatformIO/%s" % __version__, "CI/%d" % int(is_ci())] + data = [ + "PlatformIO/%s" % __version__, + "CI/%d" % int(proc.is_ci()), + "Container/%d" % int(proc.is_container()), + ] if get_session_var("caller_id"): data.append("Caller/%s" % get_session_var("caller_id")) if os.getenv("PLATFORMIO_IDE"): diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 7184da7c21..1547cf99aa 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -30,7 +30,7 @@ from platformio import compat, fs from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformBase +from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_dir @@ -55,6 +55,7 @@ "c++", "link", "platformio", + "piotarget", "pioplatform", "pioproject", "piomaxlen", @@ -159,7 +160,7 @@ env.SConscriptChdir(0) env.SConsignFile( - join("$BUILD_DIR", ".sconsign%d%d.db" % (sys.version_info[0], sys.version_info[1])) + join("$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1])) ) for item in env.GetExtraScripts("pre"): @@ -217,7 +218,7 @@ click.echo( "\n%s\n" % dump_json_to_unicode( - projenv.DumpIDEData() # pylint: disable=undefined-variable + projenv.DumpIDEData(env) # pylint: disable=undefined-variable ) ) env.Exit(0) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 65203ab785..c21b150022 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -20,7 +20,7 @@ from SCons.Defaults import processDefines # pylint: disable=import-error from platformio.compat import glob_escape -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command, where_is_program @@ -45,10 +45,10 @@ def _dump_includes(env): # includes from toolchains p = env.PioPlatform() includes["toolchain"] = [] - for name in p.get_installed_packages(): - if p.get_package_type(name) != "toolchain": + for pkg in p.get_installed_packages(): + if p.get_package_type(pkg.metadata.name) != "toolchain": continue - toolchain_dir = glob_escape(p.get_package_dir(name)) + toolchain_dir = glob_escape(pkg.path) toolchain_incglobs = [ os.path.join(toolchain_dir, "*", "include", "c++", "*"), os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"), @@ -143,7 +143,8 @@ def _escape_build_flag(flags): return [flag if " " not in flag else '"%s"' % flag for flag in flags] -def DumpIDEData(env): +def DumpIDEData(env, globalenv): + """ env here is `projenv`""" env["__escape_build_flag"] = _escape_build_flag @@ -169,6 +170,7 @@ def DumpIDEData(env): ], "svd_path": _get_svd_path(env), "compiler_type": env.GetCompilerType(), + "targets": globalenv.DumpTargets(), } env_ = env.Clone() diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 3aa6b36d5c..f6b9824be6 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=no-member, no-self-use, unused-argument, too-many-lines +# pylint: disable=no-self-use, unused-argument, too-many-lines # pylint: disable=too-many-instance-attributes, too-many-public-methods # pylint: disable=assignment-from-no-return @@ -23,7 +23,6 @@ import os import re import sys -from os.path import basename, commonprefix, isdir, isfile, join, realpath, sep import click import SCons.Scanner # pylint: disable=import-error @@ -33,12 +32,15 @@ from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool +from platformio.clients.http import InternetIsOffline from platformio.compat import WINDOWS, hashlib_encode_data, string_types -from platformio.managers.lib import LibraryManager +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.library import LibraryPackageManager from platformio.package.manifest.parser import ( ManifestParserError, ManifestParserFactory, ) +from platformio.package.meta import PackageItem from platformio.project.options import ProjectOptions @@ -46,7 +48,7 @@ class LibBuilderFactory(object): @staticmethod def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): clsname = "UnknownLibBuilder" - if isfile(join(path, "library.json")): + if os.path.isfile(os.path.join(path, "library.json")): clsname = "PlatformIOLibBuilder" else: used_frameworks = LibBuilderFactory.get_used_frameworks(env, path) @@ -63,12 +65,12 @@ def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): @staticmethod def get_used_frameworks(env, path): if any( - isfile(join(path, fname)) + os.path.isfile(os.path.join(path, fname)) for fname in ("library.properties", "keywords.txt") ): return ["arduino"] - if isfile(join(path, "module.json")): + if os.path.isfile(os.path.join(path, "module.json")): return ["mbed"] include_re = re.compile( @@ -84,7 +86,7 @@ def get_used_frameworks(env, path): fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT ): continue - with io.open(join(root, fname), errors="ignore") as fp: + with io.open(os.path.join(root, fname), errors="ignore") as fp: content = fp.read() if not content: continue @@ -111,7 +113,7 @@ class LibBuilderBase(object): def __init__(self, env, path, manifest=None, verbose=False): self.env = env.Clone() self.envorigin = env.Clone() - self.path = realpath(env.subst(path)) + self.path = os.path.realpath(env.subst(path)) self.verbose = verbose try: @@ -145,11 +147,11 @@ def __contains__(self, path): p2 = p2.lower() if p1 == p2: return True - return commonprefix((p1 + sep, p2)) == p1 + sep + return os.path.commonprefix((p1 + os.path.sep, p2)) == p1 + os.path.sep @property def name(self): - return self._manifest.get("name", basename(self.path)) + return self._manifest.get("name", os.path.basename(self.path)) @property def version(self): @@ -170,13 +172,19 @@ def src_filter(self): @property def include_dir(self): - if not all(isdir(join(self.path, d)) for d in ("include", "src")): + if not all( + os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src") + ): return None - return join(self.path, "include") + return os.path.join(self.path, "include") @property def src_dir(self): - return join(self.path, "src") if isdir(join(self.path, "src")) else self.path + return ( + os.path.join(self.path, "src") + if os.path.isdir(os.path.join(self.path, "src")) + else self.path + ) def get_include_dirs(self): items = [] @@ -189,7 +197,9 @@ def get_include_dirs(self): @property def build_dir(self): lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3] - return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) + return os.path.join( + "$BUILD_DIR", "lib%s" % lib_hash, os.path.basename(self.path) + ) @property def build_flags(self): @@ -268,7 +278,7 @@ def process_extra_options(self): if self.extra_script: self.env.SConscriptChdir(1) self.env.SConscript( - realpath(self.extra_script), + os.path.realpath(self.extra_script), exports={"env": self.env, "pio_lib_builder": self}, ) self.env.ProcessUnFlags(self.build_unflags) @@ -294,14 +304,14 @@ def process_dependencies(self): def get_search_files(self): items = [ - join(self.src_dir, item) + os.path.join(self.src_dir, item) for item in self.env.MatchSourceFiles(self.src_dir, self.src_filter) ] include_dir = self.include_dir if include_dir: items.extend( [ - join(include_dir, item) + os.path.join(include_dir, item) for item in self.env.MatchSourceFiles(include_dir) ] ) @@ -370,7 +380,7 @@ def _get_found_includes( # pylint: disable=too-many-branches continue _f_part = _h_path[: _h_path.rindex(".")] for ext in piotool.SRC_C_EXT + piotool.SRC_CXX_EXT: - if not isfile("%s.%s" % (_f_part, ext)): + if not os.path.isfile("%s.%s" % (_f_part, ext)): continue _c_path = self.env.File("%s.%s" % (_f_part, ext)) if _c_path not in result: @@ -464,23 +474,24 @@ class UnknownLibBuilder(LibBuilderBase): class ArduinoLibBuilder(LibBuilderBase): def load_manifest(self): - manifest_path = join(self.path, "library.properties") - if not isfile(manifest_path): + manifest_path = os.path.join(self.path, "library.properties") + if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) - if isdir(join(self.path, "src")): + if os.path.isdir(os.path.join(self.path, "src")): return include_dirs - if isdir(join(self.path, "utility")): - include_dirs.append(join(self.path, "utility")) + if os.path.isdir(os.path.join(self.path, "utility")): + include_dirs.append(os.path.join(self.path, "utility")) return include_dirs @property def src_filter(self): - src_dir = join(self.path, "src") - if isdir(src_dir): + src_dir = os.path.join(self.path, "src") + if os.path.isdir(src_dir): + # pylint: disable=no-member src_filter = LibBuilderBase.src_filter.fget(self) for root, _, files in os.walk(src_dir, followlinks=True): found = False @@ -491,50 +502,68 @@ def src_filter(self): if not found: continue rel_path = root.replace(src_dir, "") - if rel_path.startswith(sep): - rel_path = rel_path[1:] + sep + if rel_path.startswith(os.path.sep): + rel_path = rel_path[1:] + os.path.sep src_filter.append("-<%s*.[aA][sS][mM]>" % rel_path) return src_filter src_filter = [] - is_utility = isdir(join(self.path, "utility")) + is_utility = os.path.isdir(os.path.join(self.path, "utility")) for ext in piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT: # arduino ide ignores files with .asm or .ASM extensions if ext.lower() == "asm": continue src_filter.append("+<*.%s>" % ext) if is_utility: - src_filter.append("+" % (sep, ext)) + src_filter.append("+" % (os.path.sep, ext)) return src_filter + @property + def dependencies(self): + # do not include automatically all libraries for build + # chain+ will decide later + return None + + @property + def lib_ldf_mode(self): + # pylint: disable=no-member + if not self._manifest.get("dependencies"): + return LibBuilderBase.lib_ldf_mode.fget(self) + missing = object() + global_value = self.env.GetProjectConfig().getraw( + "env:" + self.env["PIOENV"], "lib_ldf_mode", missing + ) + if global_value != missing: + return LibBuilderBase.lib_ldf_mode.fget(self) + # automatically enable C++ Preprocessing in runtime + # (Arduino IDE has this behavior) + return "chain+" + def is_frameworks_compatible(self, frameworks): return util.items_in_list(frameworks, ["arduino", "energia"]) def is_platforms_compatible(self, platforms): - items = self._manifest.get("platforms", []) - if not items: - return LibBuilderBase.is_platforms_compatible(self, platforms) - return util.items_in_list(platforms, items) + return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) class MbedLibBuilder(LibBuilderBase): def load_manifest(self): - manifest_path = join(self.path, "module.json") - if not isfile(manifest_path): + manifest_path = os.path.join(self.path, "module.json") + if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() @property def include_dir(self): - if isdir(join(self.path, "include")): - return join(self.path, "include") + if os.path.isdir(os.path.join(self.path, "include")): + return os.path.join(self.path, "include") return None @property def src_dir(self): - if isdir(join(self.path, "source")): - return join(self.path, "source") - return LibBuilderBase.src_dir.fget(self) + if os.path.isdir(os.path.join(self.path, "source")): + return os.path.join(self.path, "source") + return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) @@ -543,13 +572,13 @@ def get_include_dirs(self): # library with module.json for p in self._manifest.get("extraIncludes", []): - include_dirs.append(join(self.path, p)) + include_dirs.append(os.path.join(self.path, p)) # old mbed library without manifest, add to CPPPATH all folders if not self._manifest: for root, _, __ in os.walk(self.path): part = root.replace(self.path, "").lower() - if any(s in part for s in ("%s." % sep, "test", "example")): + if any(s in part for s in ("%s." % os.path.sep, "test", "example")): continue if root not in include_dirs: include_dirs.append(root) @@ -565,7 +594,7 @@ def process_extra_options(self): def _process_mbed_lib_confs(self): mbed_lib_paths = [ - join(root, "mbed_lib.json") + os.path.join(root, "mbed_lib.json") for root, _, files in os.walk(self.path) if "mbed_lib.json" in files ] @@ -574,8 +603,8 @@ def _process_mbed_lib_confs(self): mbed_config_path = None for p in self.env.get("CPPPATH"): - mbed_config_path = join(self.env.subst(p), "mbed_config.h") - if isfile(mbed_config_path): + mbed_config_path = os.path.join(self.env.subst(p), "mbed_config.h") + if os.path.isfile(mbed_config_path): break mbed_config_path = None if not mbed_config_path: @@ -667,30 +696,31 @@ def _mbed_conf_append_macros(self, mbed_config_path, macros): class PlatformIOLibBuilder(LibBuilderBase): def load_manifest(self): - manifest_path = join(self.path, "library.json") - if not isfile(manifest_path): + manifest_path = os.path.join(self.path, "library.json") + if not os.path.isfile(manifest_path): return {} return ManifestParserFactory.new_from_file(manifest_path).as_dict() def _has_arduino_manifest(self): - return isfile(join(self.path, "library.properties")) + return os.path.isfile(os.path.join(self.path, "library.properties")) @property def include_dir(self): if "includeDir" in self._manifest.get("build", {}): with fs.cd(self.path): - return realpath(self._manifest.get("build").get("includeDir")) - return LibBuilderBase.include_dir.fget(self) + return os.path.realpath(self._manifest.get("build").get("includeDir")) + return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): with fs.cd(self.path): - return realpath(self._manifest.get("build").get("srcDir")) - return LibBuilderBase.src_dir.fget(self) + return os.path.realpath(self._manifest.get("build").get("srcDir")) + return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member @property def src_filter(self): + # pylint: disable=no-member if "srcFilter" in self._manifest.get("build", {}): return self._manifest.get("build").get("srcFilter") if self.env["SRC_FILTER"]: @@ -703,19 +733,19 @@ def src_filter(self): def build_flags(self): if "flags" in self._manifest.get("build", {}): return self._manifest.get("build").get("flags") - return LibBuilderBase.build_flags.fget(self) + return LibBuilderBase.build_flags.fget(self) # pylint: disable=no-member @property def build_unflags(self): if "unflags" in self._manifest.get("build", {}): return self._manifest.get("build").get("unflags") - return LibBuilderBase.build_unflags.fget(self) + return LibBuilderBase.build_unflags.fget(self) # pylint: disable=no-member @property def extra_script(self): if "extraScript" in self._manifest.get("build", {}): return self._manifest.get("build").get("extraScript") - return LibBuilderBase.extra_script.fget(self) + return LibBuilderBase.extra_script.fget(self) # pylint: disable=no-member @property def lib_archive(self): @@ -727,12 +757,14 @@ def lib_archive(self): return self.env.GetProjectConfig().get( "env:" + self.env["PIOENV"], "lib_archive" ) + # pylint: disable=no-member return self._manifest.get("build", {}).get( "libArchive", LibBuilderBase.lib_archive.fget(self) ) @property def lib_ldf_mode(self): + # pylint: disable=no-member return self.validate_ldf_mode( self._manifest.get("build", {}).get( "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self) @@ -741,6 +773,7 @@ def lib_ldf_mode(self): @property def lib_compat_mode(self): + # pylint: disable=no-member return self.validate_compat_mode( self._manifest.get("build", {}).get( "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self) @@ -748,16 +781,10 @@ def lib_compat_mode(self): ) def is_platforms_compatible(self, platforms): - items = self._manifest.get("platforms") - if not items: - return LibBuilderBase.is_platforms_compatible(self, platforms) - return util.items_in_list(platforms, items) + return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"]) def is_frameworks_compatible(self, frameworks): - items = self._manifest.get("frameworks") - if not items: - return LibBuilderBase.is_frameworks_compatible(self, frameworks) - return util.items_in_list(frameworks, items) + return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"]) def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) @@ -766,10 +793,10 @@ def get_include_dirs(self): if ( "build" not in self._manifest and self._has_arduino_manifest() - and not isdir(join(self.path, "src")) - and isdir(join(self.path, "utility")) + and not os.path.isdir(os.path.join(self.path, "src")) + and os.path.isdir(os.path.join(self.path, "utility")) ): - include_dirs.append(join(self.path, "utility")) + include_dirs.append(os.path.join(self.path, "utility")) for path in self.env.get("CPPPATH", []): if path not in self.envorigin.get("CPPPATH", []): @@ -788,7 +815,7 @@ def __init__(self, env, *args, **kwargs): @property def include_dir(self): include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") - return include_dir if isdir(include_dir) else None + return include_dir if os.path.isdir(include_dir) else None @property def src_dir(self): @@ -797,7 +824,7 @@ def src_dir(self): def get_include_dirs(self): include_dirs = [] project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") - if isdir(project_include_dir): + if os.path.isdir(project_include_dir): include_dirs.append(project_include_dir) for include_dir in LibBuilderBase.get_include_dirs(self): if include_dir not in include_dirs: @@ -811,7 +838,7 @@ def get_search_files(self): if "__test" in COMMAND_LINE_TARGETS: items.extend( [ - join("$PROJECT_TEST_DIR", item) + os.path.join("$PROJECT_TEST_DIR", item) for item in self.env.MatchSourceFiles( "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) @@ -821,7 +848,7 @@ def get_search_files(self): @property def lib_ldf_mode(self): - mode = LibBuilderBase.lib_ldf_mode.fget(self) + mode = LibBuilderBase.lib_ldf_mode.fget(self) # pylint: disable=no-member if not mode.startswith("chain"): return mode # parse all project files @@ -829,6 +856,7 @@ def lib_ldf_mode(self): @property def src_filter(self): + # pylint: disable=no-member return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self) @property @@ -840,34 +868,36 @@ def process_extra_options(self): pass def install_dependencies(self): - def _is_builtin(uri): + def _is_builtin(spec): for lb in self.env.GetLibBuilders(): - if lb.name == uri: + if lb.name == spec: return True return False - not_found_uri = [] - for uri in self.dependencies: + not_found_specs = [] + for spec in self.dependencies: # check if built-in library - if _is_builtin(uri): + if _is_builtin(spec): continue found = False for storage_dir in self.env.GetLibSourceDirs(): - lm = LibraryManager(storage_dir) - if lm.get_package_dir(*lm.parse_pkg_uri(uri)): + lm = LibraryPackageManager(storage_dir) + if lm.get_package(spec): found = True break if not found: - not_found_uri.append(uri) + not_found_specs.append(spec) did_install = False - lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))) - for uri in not_found_uri: + lm = LibraryPackageManager( + self.env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) + ) + for spec in not_found_specs: try: - lm.install(uri) + lm.install(spec) did_install = True - except (exception.LibNotFound, exception.InternetIsOffline) as e: + except (UnknownPackageError, InternetIsOffline) as e: click.secho("Warning! %s" % e, fg="yellow") # reset cache @@ -875,17 +905,17 @@ def _is_builtin(uri): DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None) def process_dependencies(self): # pylint: disable=too-many-branches - for uri in self.dependencies: + for spec in self.dependencies: found = False for storage_dir in self.env.GetLibSourceDirs(): if found: break - lm = LibraryManager(storage_dir) - lib_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri)) - if not lib_dir: + lm = LibraryPackageManager(storage_dir) + pkg = lm.get_package(spec) + if not pkg: continue for lb in self.env.GetLibBuilders(): - if lib_dir != lb.path: + if pkg.path != lb.path: continue if lb not in self.depbuilders: self.depend_recursive(lb) @@ -897,7 +927,7 @@ def process_dependencies(self): # pylint: disable=too-many-branches # look for built-in libraries by a name # which don't have package manifest for lb in self.env.GetLibBuilders(): - if lb.name != uri: + if lb.name != spec: continue if lb not in self.depbuilders: self.depend_recursive(lb) @@ -952,12 +982,12 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches found_incompat = False for storage_dir in env.GetLibSourceDirs(): - storage_dir = realpath(storage_dir) - if not isdir(storage_dir): + storage_dir = os.path.realpath(storage_dir) + if not os.path.isdir(storage_dir): continue for item in sorted(os.listdir(storage_dir)): - lib_dir = join(storage_dir, item) - if item == "__cores__" or not isdir(lib_dir): + lib_dir = os.path.join(storage_dir, item) + if item == "__cores__" or not os.path.isdir(lib_dir): continue try: lb = LibBuilderFactory.new(env, lib_dir) @@ -989,10 +1019,6 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches def ConfigureProjectLibBuilder(env): - def _get_vcs_info(lb): - path = LibraryManager.get_src_manifest_path(lb.path) - return fs.load_json(path) if path else None - def _correct_found_libs(lib_builders): # build full dependency graph found_lbs = [lb for lb in lib_builders if lb.dependent] @@ -1008,15 +1034,15 @@ def _print_deps_tree(root, level=0): margin = "| " * (level) for lb in root.depbuilders: title = "<%s>" % lb.name - vcs_info = _get_vcs_info(lb) - if lb.version: + pkg = PackageItem(lb.path) + if pkg.metadata: + title += " %s" % pkg.metadata.version + elif lb.version: title += " %s" % lb.version - if vcs_info and vcs_info.get("version"): - title += " #%s" % vcs_info.get("version") click.echo("%s|-- %s" % (margin, title), nl=False) if int(ARGUMENTS.get("PIOVERBOSE", 0)): - if vcs_info: - click.echo(" [%s]" % vcs_info.get("url"), nl=False) + if pkg.metadata and pkg.metadata.spec.external: + click.echo(" [%s]" % pkg.metadata.spec.url, nl=False) click.echo(" (", nl=False) click.echo(lb.path, nl=False) click.echo(")", nl=False) @@ -1025,7 +1051,7 @@ def _print_deps_tree(root, level=0): _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") - ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) + ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) # pylint: disable=no-member click.echo("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") click.echo( diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 1079f40252..799b192fbc 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -16,19 +16,16 @@ import atexit import io +import os import re import sys -from os import environ, remove, walk -from os.path import basename, isdir, isfile, join, realpath, relpath, sep from tempfile import mkstemp import click -from SCons.Action import Action # pylint: disable=import-error -from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import fs, util from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command @@ -126,11 +123,11 @@ def _gcc_preprocess(self, contents, out_file): '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( out_file, tmp_path ), - "Converting " + basename(out_file[:-4]), + "Converting " + os.path.basename(out_file[:-4]), ) ) atexit.register(_delete_file, tmp_path) - return isfile(out_file) + return os.path.isfile(out_file) def _join_multiline_strings(self, contents): if "\\\n" not in contents: @@ -233,7 +230,9 @@ def append_prototypes(self, contents): def ConvertInoToCpp(env): src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR")) - ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde")) + ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob( + os.path.join(src_dir, "*.pde") + ) if not ino_nodes: return c = InoToCPPConverter(env) @@ -244,8 +243,8 @@ def ConvertInoToCpp(env): def _delete_file(path): try: - if isfile(path): - remove(path) + if os.path.isfile(path): + os.remove(path) except: # pylint: disable=bare-except pass @@ -255,7 +254,7 @@ def _get_compiler_type(env): if env.subst("$CC").endswith("-gcc"): return "gcc" try: - sysenv = environ.copy() + sysenv = os.environ.copy() sysenv["PATH"] = str(env["ENV"]["PATH"]) result = exec_command([env.subst("$CC"), "-v"], env=sysenv) except OSError: @@ -277,8 +276,8 @@ def GetCompilerType(env): def GetActualLDScript(env): def _lookup_in_ldpath(script): for d in env.get("LIBPATH", []): - path = join(env.subst(d), script) - if isfile(path): + path = os.path.join(env.subst(d), script) + if os.path.isfile(path): return path return None @@ -297,7 +296,7 @@ def _lookup_in_ldpath(script): else: continue script = env.subst(raw_script.replace('"', "").strip()) - if isfile(script): + if os.path.isfile(script): return script path = _lookup_in_ldpath(script) if path: @@ -319,29 +318,6 @@ def _lookup_in_ldpath(script): env.Exit(1) -def VerboseAction(_, act, actstr): - if int(ARGUMENTS.get("PIOVERBOSE", 0)): - return act - return Action(act, actstr) - - -def PioClean(env, clean_dir): - if not isdir(clean_dir): - print("Build environment is clean") - env.Exit(0) - clean_rel_path = relpath(clean_dir) - for root, _, files in walk(clean_dir): - for f in files: - dst = join(root, f) - remove(dst) - print( - "Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst)) - ) - print("Done cleaning") - fs.rmtree(clean_dir) - env.Exit(0) - - def ConfigureDebugFlags(env): def _cleanup_debug_flags(scope): if scope not in env: @@ -370,16 +346,16 @@ def _cleanup_debug_flags(scope): def ConfigureTestTarget(env): env.Append( CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], - CPPPATH=[join("$BUILD_DIR", "UnityTestLib")], + CPPPATH=[os.path.join("$BUILD_DIR", "UnityTestLib")], ) unitylib = env.BuildLibrary( - join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity") + os.path.join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity") ) env.Prepend(LIBS=[unitylib]) src_filter = ["+<*.cpp>", "+<*.c>"] if "PIOTEST_RUNNING_NAME" in env: - src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], sep)) + src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], os.path.sep)) env.Replace(PIOTEST_SRC_FILTER=src_filter) @@ -393,7 +369,7 @@ def GetExtraScripts(env, scope): if not items: return items with fs.cd(env.subst("$PROJECT_DIR")): - return [realpath(item) for item in items] + return [os.path.realpath(item) for item in items] def exists(_): @@ -404,8 +380,6 @@ def generate(env): env.AddMethod(ConvertInoToCpp) env.AddMethod(GetCompilerType) env.AddMethod(GetActualLDScript) - env.AddMethod(VerboseAction) - env.AddMethod(PioClean) env.AddMethod(ConfigureDebugFlags) env.AddMethod(ConfigureTestTarget) env.AddMethod(GetExtraScripts) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index e280372b7d..7dd36a64ef 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -14,15 +14,18 @@ from __future__ import absolute_import +import os import sys -from os.path import isdir, isfile, join from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error -from platformio import exception, fs, util +from platformio import fs, util from platformio.compat import WINDOWS -from platformio.managers.platform import PlatformFactory +from platformio.package.meta import PackageItem +from platformio.package.version import get_original_version +from platformio.platform.exception import UnknownBoard +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectOptions # pylint: disable=too-many-branches, too-many-locals @@ -34,7 +37,7 @@ def PioPlatform(env): if "framework" in variables: # support PIO Core 3.0 dev/platforms variables["pioframework"] = variables["framework"] - p = PlatformFactory.newPlatform(env["PLATFORM_MANIFEST"]) + p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"])) p.configure_default_packages(variables, COMMAND_LINE_TARGETS) return p @@ -46,7 +49,7 @@ def BoardConfig(env, board=None): board = board or env.get("BOARD") assert board, "BoardConfig: Board is not defined" return p.board_config(board) - except (AssertionError, exception.UnknownBoard) as e: + except (AssertionError, UnknownBoard) as e: sys.stderr.write("Error: %s\n" % str(e)) env.Exit(1) @@ -55,37 +58,42 @@ def GetFrameworkScript(env, framework): p = env.PioPlatform() assert p.frameworks and framework in p.frameworks script_path = env.subst(p.frameworks[framework]["script"]) - if not isfile(script_path): - script_path = join(p.get_dir(), script_path) + if not os.path.isfile(script_path): + script_path = os.path.join(p.get_dir(), script_path) return script_path def LoadPioPlatform(env): p = env.PioPlatform() - installed_packages = p.get_installed_packages() # Ensure real platform name env["PIOPLATFORM"] = p.name # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH systype = util.get_systype() - for name in installed_packages: - type_ = p.get_package_type(name) + for pkg in p.get_installed_packages(): + type_ = p.get_package_type(pkg.metadata.name) if type_ not in ("toolchain", "uploader", "debugger"): continue - pkg_dir = p.get_package_dir(name) env.PrependENVPath( - "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir + "PATH", + os.path.join(pkg.path, "bin") + if os.path.isdir(os.path.join(pkg.path, "bin")) + else pkg.path, ) - if not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain": + if ( + not WINDOWS + and os.path.isdir(os.path.join(pkg.path, "lib")) + and type_ != "toolchain" + ): env.PrependENVPath( "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH", - join(pkg_dir, "lib"), + os.path.join(pkg.path, "lib"), ) # Platform specific LD Scripts - if isdir(join(p.get_dir(), "ldscripts")): - env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")]) + if os.path.isdir(os.path.join(p.get_dir(), "ldscripts")): + env.Prepend(LIBPATH=[os.path.join(p.get_dir(), "ldscripts")]) if "BOARD" not in env: return @@ -125,6 +133,7 @@ def LoadPioPlatform(env): def PrintConfiguration(env): # pylint: disable=too-many-statements platform = env.PioPlatform() + pkg_metadata = PackageItem(platform.get_dir()).metadata board_config = env.BoardConfig() if "BOARD" in env else None def _get_configuration_data(): @@ -139,11 +148,19 @@ def _get_configuration_data(): ) def _get_plaform_data(): - data = ["PLATFORM: %s %s" % (platform.title, platform.version)] - if platform.src_version: - data.append("#" + platform.src_version) - if int(ARGUMENTS.get("PIOVERBOSE", 0)) and platform.src_url: - data.append("(%s)" % platform.src_url) + data = [ + "PLATFORM: %s (%s)" + % ( + platform.title, + pkg_metadata.version if pkg_metadata else platform.version, + ) + ] + if ( + int(ARGUMENTS.get("PIOVERBOSE", 0)) + and pkg_metadata + and pkg_metadata.spec.external + ): + data.append("(%s)" % pkg_metadata.spec.url) if board_config: data.extend([">", board_config.get("name")]) return data @@ -162,7 +179,8 @@ def _get_hardware_data(): ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") data.append( - "%s RAM, %s Flash" % (fs.format_filesize(ram), fs.format_filesize(flash)) + "%s RAM, %s Flash" + % (fs.humanize_file_size(ram), fs.humanize_file_size(flash)) ) return data @@ -194,7 +212,7 @@ def _get_debug_data(): def _get_packages_data(): data = [] for item in platform.dump_used_packages(): - original_version = util.get_original_version(item["version"]) + original_version = get_original_version(item["version"]) info = "%s %s" % (item["name"], item["version"]) extra = [] if original_version: diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py new file mode 100644 index 0000000000..948776fc0b --- /dev/null +++ b/platformio/builder/tools/piotarget.py @@ -0,0 +1,119 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +from SCons.Action import Action # pylint: disable=import-error +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import AlwaysBuild # pylint: disable=import-error + +from platformio import compat, fs + + +def VerboseAction(_, act, actstr): + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + return act + return Action(act, actstr) + + +def PioClean(env, clean_dir): + def _relpath(path): + if compat.WINDOWS: + prefix = os.getcwd()[:2].lower() + if ( + ":" not in prefix + or not path.lower().startswith(prefix) + or os.path.relpath(path).startswith("..") + ): + return path + return os.path.relpath(path) + + if not os.path.isdir(clean_dir): + print("Build environment is clean") + env.Exit(0) + clean_rel_path = _relpath(clean_dir) + for root, _, files in os.walk(clean_dir): + for f in files: + dst = os.path.join(root, f) + os.remove(dst) + print( + "Removed %s" + % (dst if not clean_rel_path.startswith(".") else _relpath(dst)) + ) + print("Done cleaning") + fs.rmtree(clean_dir) + env.Exit(0) + + +def AddTarget( # pylint: disable=too-many-arguments + env, + name, + dependencies, + actions, + title=None, + description=None, + group="Generic", + always_build=True, +): + if "__PIO_TARGETS" not in env: + env["__PIO_TARGETS"] = {} + assert name not in env["__PIO_TARGETS"] + env["__PIO_TARGETS"][name] = dict( + name=name, title=title, description=description, group=group + ) + target = env.Alias(name, dependencies, actions) + if always_build: + AlwaysBuild(target) + return target + + +def AddPlatformTarget(env, *args, **kwargs): + return env.AddTarget(group="Platform", *args, **kwargs) + + +def AddCustomTarget(env, *args, **kwargs): + return env.AddTarget(group="Custom", *args, **kwargs) + + +def DumpTargets(env): + targets = env.get("__PIO_TARGETS") or {} + # pre-fill default targets if embedded dev-platform + if env.PioPlatform().is_embedded() and not any( + t["group"] == "Platform" for t in targets.values() + ): + targets["upload"] = dict(name="upload", group="Platform", title="Upload") + targets["compiledb"] = dict( + name="compiledb", + title="Compilation Database", + description="Generate compilation database `compile_commands.json`", + group="Advanced", + ) + targets["clean"] = dict(name="clean", title="Clean", group="Generic") + return list(targets.values()) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(VerboseAction) + env.AddMethod(PioClean) + env.AddMethod(AddTarget) + env.AddMethod(AddPlatformTarget) + env.AddMethod(AddCustomTarget) + env.AddMethod(DumpTargets) + return env diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index c0cc11de71..aac4742621 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -26,9 +26,9 @@ from SCons.Script import Export # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error -from platformio import fs +from platformio import __version__, fs from platformio.compat import string_types -from platformio.util import pioversion_to_intstr +from platformio.package.version import pepver_to_semver SRC_HEADER_EXT = ["h", "hpp"] SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"] @@ -66,7 +66,11 @@ def BuildProgram(env): env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")]) # enable "cyclic reference" for linker - if env.get("LIBS") and env.GetCompilerType() == "gcc": + if ( + env.get("LIBS") + and env.GetCompilerType() == "gcc" + and env.PioPlatform().is_embedded() + ): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") @@ -90,11 +94,16 @@ def BuildProgram(env): def ProcessProgramDeps(env): def _append_pio_macros(): + core_version = pepver_to_semver(__version__) env.AppendUnique( CPPDEFINES=[ ( "PLATFORMIO", - int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())), + int( + "{0:02d}{1:02d}{2:02d}".format( + core_version.major, core_version.minor, core_version.patch + ) + ), ) ] ) @@ -282,18 +291,21 @@ def CollectBuildFiles( if fs.path_endswith_ext(item, SRC_BUILD_EXT): sources.append(env.File(os.path.join(_var_dir, os.path.basename(item)))) - for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []): - tmp = [] - for node in sources: + middlewares = env.get("__PIO_BUILD_MIDDLEWARES") + if not middlewares: + return sources + + new_sources = [] + for node in sources: + new_node = node + for callback, pattern in middlewares: if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern): - tmp.append(node) continue - n = callback(node) - if n: - tmp.append(n) - sources = tmp + new_node = callback(new_node) + if new_node: + new_sources.append(new_node) - return sources + return new_sources def AddBuildMiddleware(env, callback, pattern=None): diff --git a/platformio/cache.py b/platformio/cache.py new file mode 100644 index 0000000000..bc817f61ef --- /dev/null +++ b/platformio/cache.py @@ -0,0 +1,165 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import codecs +import hashlib +import os +from time import time + +from platformio import app, fs +from platformio.compat import hashlib_encode_data +from platformio.package.lockfile import LockFile +from platformio.project.helpers import get_project_cache_dir + + +class ContentCache(object): + def __init__(self, namespace=None): + self.cache_dir = os.path.join(get_project_cache_dir(), namespace or "content") + self._db_path = os.path.join(self.cache_dir, "db.data") + self._lockfile = None + if not os.path.isdir(self.cache_dir): + os.makedirs(self.cache_dir) + + def __enter__(self): + # cleanup obsolete items + self.delete() + return self + + def __exit__(self, type_, value, traceback): + pass + + @staticmethod + def key_from_args(*args): + h = hashlib.sha1() + for arg in args: + if arg: + h.update(hashlib_encode_data(arg)) + return h.hexdigest() + + def get_cache_path(self, key): + assert "/" not in key and "\\" not in key + key = str(key) + assert len(key) > 3 + return os.path.join(self.cache_dir, key) + + def get(self, key): + cache_path = self.get_cache_path(key) + if not os.path.isfile(cache_path): + return None + with codecs.open(cache_path, "rb", encoding="utf8") as fp: + return fp.read() + + def set(self, key, data, valid): + if not app.get_setting("enable_cache"): + return False + cache_path = self.get_cache_path(key) + if os.path.isfile(cache_path): + self.delete(key) + if not data: + return False + tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} + assert valid.endswith(tuple(tdmap)) + expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) + + if not self._lock_dbindex(): + return False + + if not os.path.isdir(os.path.dirname(cache_path)): + os.makedirs(os.path.dirname(cache_path)) + try: + with codecs.open(cache_path, "wb", encoding="utf8") as fp: + fp.write(data) + with open(self._db_path, "a") as fp: + fp.write("%s=%s\n" % (str(expire_time), os.path.basename(cache_path))) + except UnicodeError: + if os.path.isfile(cache_path): + try: + os.remove(cache_path) + except OSError: + pass + + return self._unlock_dbindex() + + def delete(self, keys=None): + """ Keys=None, delete expired items """ + if not os.path.isfile(self._db_path): + return None + if not keys: + keys = [] + if not isinstance(keys, list): + keys = [keys] + paths_for_delete = [self.get_cache_path(k) for k in keys] + found = False + newlines = [] + with open(self._db_path) as fp: + for line in fp.readlines(): + line = line.strip() + if "=" not in line: + continue + expire, fname = line.split("=") + path = os.path.join(self.cache_dir, fname) + try: + if ( + time() < int(expire) + and os.path.isfile(path) + and path not in paths_for_delete + ): + newlines.append(line) + continue + except ValueError: + pass + found = True + if os.path.isfile(path): + try: + os.remove(path) + if not os.listdir(os.path.dirname(path)): + fs.rmtree(os.path.dirname(path)) + except OSError: + pass + + if found and self._lock_dbindex(): + with open(self._db_path, "w") as fp: + fp.write("\n".join(newlines) + "\n") + self._unlock_dbindex() + + return True + + def clean(self): + if not os.path.isdir(self.cache_dir): + return + fs.rmtree(self.cache_dir) + + def _lock_dbindex(self): + self._lockfile = LockFile(self.cache_dir) + try: + self._lockfile.acquire() + except: # pylint: disable=bare-except + return False + + return True + + def _unlock_dbindex(self): + if self._lockfile: + self._lockfile.release() + return True + + +# +# Helpers +# + + +def cleanup_content_cache(namespace=None): + with ContentCache(namespace) as cc: + cc.clean() diff --git a/platformio/commands/account/__init__.py b/platformio/clients/__init__.py similarity index 100% rename from platformio/commands/account/__init__.py rename to platformio/clients/__init__.py diff --git a/platformio/clients/account.py b/platformio/clients/account.py new file mode 100644 index 0000000000..1c4b6755fa --- /dev/null +++ b/platformio/clients/account.py @@ -0,0 +1,290 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time + +from platformio import __accounts_api__, app +from platformio.clients.http import HTTPClient +from platformio.exception import PlatformioException + + +class AccountError(PlatformioException): + + MESSAGE = "{0}" + + +class AccountNotAuthorized(AccountError): + + MESSAGE = "You are not authorized! Please log in to PlatformIO Account." + + +class AccountAlreadyAuthorized(AccountError): + + MESSAGE = "You are already authorized with {0} account." + + +class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods + + SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 + + def __init__(self): + super(AccountClient, self).__init__(__accounts_api__) + + @staticmethod + def get_refresh_token(): + try: + return app.get_state_item("account").get("auth").get("refresh_token") + except: # pylint:disable=bare-except + raise AccountNotAuthorized() + + @staticmethod + def delete_local_session(): + app.delete_state_item("account") + + @staticmethod + def delete_local_state(key): + account = app.get_state_item("account") + if not account or key not in account: + return + del account[key] + app.set_state_item("account", account) + + def send_auth_request(self, *args, **kwargs): + headers = kwargs.get("headers", {}) + if "Authorization" not in headers: + token = self.fetch_authentication_token() + headers["Authorization"] = "Bearer %s" % token + kwargs["headers"] = headers + return self.fetch_json_data(*args, **kwargs) + + def login(self, username, password): + try: + self.fetch_authentication_token() + except: # pylint:disable=bare-except + pass + else: + raise AccountAlreadyAuthorized( + app.get_state_item("account", {}).get("email", "") + ) + + data = self.fetch_json_data( + "post", "/v1/login", data={"username": username, "password": password}, + ) + app.set_state_item("account", data) + return data + + def login_with_code(self, client_id, code, redirect_uri): + try: + self.fetch_authentication_token() + except: # pylint:disable=bare-except + pass + else: + raise AccountAlreadyAuthorized( + app.get_state_item("account", {}).get("email", "") + ) + + result = self.fetch_json_data( + "post", + "/v1/login/code", + data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, + ) + app.set_state_item("account", result) + return result + + def logout(self): + refresh_token = self.get_refresh_token() + self.delete_local_session() + try: + self.fetch_json_data( + "post", "/v1/logout", data={"refresh_token": refresh_token}, + ) + except AccountError: + pass + return True + + def change_password(self, old_password, new_password): + return self.send_auth_request( + "post", + "/v1/password", + data={"old_password": old_password, "new_password": new_password}, + ) + + def registration( + self, username, email, password, firstname, lastname + ): # pylint:disable=too-many-arguments + try: + self.fetch_authentication_token() + except: # pylint:disable=bare-except + pass + else: + raise AccountAlreadyAuthorized( + app.get_state_item("account", {}).get("email", "") + ) + + return self.fetch_json_data( + "post", + "/v1/registration", + data={ + "username": username, + "email": email, + "password": password, + "firstname": firstname, + "lastname": lastname, + }, + ) + + def auth_token(self, password, regenerate): + return self.send_auth_request( + "post", + "/v1/token", + data={"password": password, "regenerate": 1 if regenerate else 0}, + ).get("auth_token") + + def forgot_password(self, username): + return self.fetch_json_data("post", "/v1/forgot", data={"username": username},) + + def get_profile(self): + return self.send_auth_request("get", "/v1/profile",) + + def update_profile(self, profile, current_password): + profile["current_password"] = current_password + self.delete_local_state("summary") + response = self.send_auth_request("put", "/v1/profile", data=profile,) + return response + + def get_account_info(self, offline=False): + account = app.get_state_item("account") or {} + if ( + account.get("summary") + and account["summary"].get("expire_at", 0) > time.time() + ): + return account["summary"] + if offline and account.get("email"): + return { + "profile": { + "email": account.get("email"), + "username": account.get("username"), + } + } + result = self.send_auth_request("get", "/v1/summary",) + account["summary"] = dict( + profile=result.get("profile"), + packages=result.get("packages"), + subscriptions=result.get("subscriptions"), + user_id=result.get("user_id"), + expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL, + ) + app.set_state_item("account", account) + return result + + def destroy_account(self): + return self.send_auth_request("delete", "/v1/account") + + def create_org(self, orgname, email, displayname): + return self.send_auth_request( + "post", + "/v1/orgs", + data={"orgname": orgname, "email": email, "displayname": displayname}, + ) + + def get_org(self, orgname): + return self.send_auth_request("get", "/v1/orgs/%s" % orgname) + + def list_orgs(self): + return self.send_auth_request("get", "/v1/orgs",) + + def update_org(self, orgname, data): + return self.send_auth_request( + "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v} + ) + + def destroy_org(self, orgname): + return self.send_auth_request("delete", "/v1/orgs/%s" % orgname,) + + def add_org_owner(self, orgname, username): + return self.send_auth_request( + "post", "/v1/orgs/%s/owners" % orgname, data={"username": username}, + ) + + def list_org_owners(self, orgname): + return self.send_auth_request("get", "/v1/orgs/%s/owners" % orgname,) + + def remove_org_owner(self, orgname, username): + return self.send_auth_request( + "delete", "/v1/orgs/%s/owners" % orgname, data={"username": username}, + ) + + def create_team(self, orgname, teamname, description): + return self.send_auth_request( + "post", + "/v1/orgs/%s/teams" % orgname, + data={"name": teamname, "description": description}, + ) + + def destroy_team(self, orgname, teamname): + return self.send_auth_request( + "delete", "/v1/orgs/%s/teams/%s" % (orgname, teamname), + ) + + def get_team(self, orgname, teamname): + return self.send_auth_request( + "get", "/v1/orgs/%s/teams/%s" % (orgname, teamname), + ) + + def list_teams(self, orgname): + return self.send_auth_request("get", "/v1/orgs/%s/teams" % orgname,) + + def update_team(self, orgname, teamname, data): + return self.send_auth_request( + "put", + "/v1/orgs/%s/teams/%s" % (orgname, teamname), + data={k: v for k, v in data.items() if v}, + ) + + def add_team_member(self, orgname, teamname, username): + return self.send_auth_request( + "post", + "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), + data={"username": username}, + ) + + def remove_team_member(self, orgname, teamname, username): + return self.send_auth_request( + "delete", + "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), + data={"username": username}, + ) + + def fetch_authentication_token(self): + if os.environ.get("PLATFORMIO_AUTH_TOKEN"): + return os.environ.get("PLATFORMIO_AUTH_TOKEN") + auth = app.get_state_item("account", {}).get("auth", {}) + if auth.get("access_token") and auth.get("access_token_expire"): + if auth.get("access_token_expire") > time.time(): + return auth.get("access_token") + if auth.get("refresh_token"): + try: + data = self.fetch_json_data( + "post", + "/v1/login", + headers={ + "Authorization": "Bearer %s" % auth.get("refresh_token") + }, + ) + app.set_state_item("account", data) + return data.get("auth").get("access_token") + except AccountError: + self.delete_local_session() + raise AccountNotAuthorized() diff --git a/platformio/clients/http.py b/platformio/clients/http.py new file mode 100644 index 0000000000..4d59bcaa62 --- /dev/null +++ b/platformio/clients/http.py @@ -0,0 +1,206 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import math +import os +import socket + +import requests.adapters +from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error + +from platformio import __check_internet_hosts__, __default_requests_timeout__, app, util +from platformio.cache import ContentCache +from platformio.exception import PlatformioException, UserSideException + +try: + from urllib.parse import urljoin +except ImportError: + from urlparse import urljoin + + +class HTTPClientError(PlatformioException): + def __init__(self, message, response=None): + super(HTTPClientError, self).__init__() + self.message = message + self.response = response + + def __str__(self): # pragma: no cover + return self.message + + +class InternetIsOffline(UserSideException): + + MESSAGE = ( + "You are not connected to the Internet.\n" + "PlatformIO needs the Internet connection to" + " download dependent packages or to work with PlatformIO Account." + ) + + +class EndpointSession(requests.Session): + def __init__(self, base_url, *args, **kwargs): + super(EndpointSession, self).__init__(*args, **kwargs) + self.base_url = base_url + + def request( # pylint: disable=signature-differs,arguments-differ + self, method, url, *args, **kwargs + ): + # print(self.base_url, method, url, args, kwargs) + return super(EndpointSession, self).request( + method, urljoin(self.base_url, url), *args, **kwargs + ) + + +class EndpointSessionIterator(object): + def __init__(self, endpoints): + if not isinstance(endpoints, list): + endpoints = [endpoints] + self.endpoints = endpoints + self.endpoints_iter = iter(endpoints) + self.retry = Retry( + total=math.ceil(6 / len(self.endpoints)), + backoff_factor=1, + # method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], + status_forcelist=[413, 429, 500, 502, 503, 504], + ) + + def __iter__(self): # pylint: disable=non-iterator-returned + return self + + def next(self): + """ For Python 2 compatibility """ + return self.__next__() + + def __next__(self): + base_url = next(self.endpoints_iter) + session = EndpointSession(base_url) + session.headers.update({"User-Agent": app.get_user_agent()}) + adapter = requests.adapters.HTTPAdapter(max_retries=self.retry) + session.mount(base_url, adapter) + return session + + +class HTTPClient(object): + def __init__(self, endpoints): + self._session_iter = EndpointSessionIterator(endpoints) + self._session = None + self._next_session() + + def __del__(self): + if not self._session: + return + self._session.close() + self._session = None + + def _next_session(self): + if self._session: + self._session.close() + self._session = next(self._session_iter) + + @util.throttle(500) + def send_request(self, method, path, **kwargs): + # check Internet before and resolve issue with 60 seconds timeout + ensure_internet_on(raise_exception=True) + + # set default timeout + if "timeout" not in kwargs: + kwargs["timeout"] = __default_requests_timeout__ + + while True: + try: + return getattr(self._session, method)(path, **kwargs) + except ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + ) as e: + try: + self._next_session() + except: # pylint: disable=bare-except + raise HTTPClientError(str(e)) + + def fetch_json_data(self, method, path, **kwargs): + cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None + if not cache_valid: + return self.raise_error_from_response( + self.send_request(method, path, **kwargs) + ) + cache_key = ContentCache.key_from_args( + method, path, kwargs.get("params"), kwargs.get("data") + ) + with ContentCache("http") as cc: + result = cc.get(cache_key) + if result is not None: + return json.loads(result) + response = self.send_request(method, path, **kwargs) + cc.set(cache_key, response.text, cache_valid) + return self.raise_error_from_response(response) + + @staticmethod + def raise_error_from_response(response, expected_codes=(200, 201, 202)): + if response.status_code in expected_codes: + try: + return response.json() + except ValueError: + pass + try: + message = response.json()["message"] + except (KeyError, ValueError): + message = response.text + raise HTTPClientError(message, response) + + +# +# Helpers +# + + +@util.memoized(expire="10s") +def _internet_on(): + timeout = 2 + socket.setdefaulttimeout(timeout) + for host in __check_internet_hosts__: + try: + for var in ("HTTP_PROXY", "HTTPS_PROXY"): + if not os.getenv(var) and not os.getenv(var.lower()): + continue + requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) + return True + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((host, 80)) + s.close() + return True + except: # pylint: disable=bare-except + pass + return False + + +def ensure_internet_on(raise_exception=False): + result = _internet_on() + if raise_exception and not result: + raise InternetIsOffline() + return result + + +def fetch_remote_content(*args, **kwargs): + kwargs["headers"] = kwargs.get("headers", {}) + if "User-Agent" not in kwargs["headers"]: + kwargs["headers"]["User-Agent"] = app.get_user_agent() + + if "timeout" not in kwargs: + kwargs["timeout"] = __default_requests_timeout__ + + r = requests.get(*args, **kwargs) + r.raise_for_status() + return r.text diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py new file mode 100644 index 0000000000..c8fbeeeadf --- /dev/null +++ b/platformio/clients/registry.py @@ -0,0 +1,141 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import __registry_api__, fs +from platformio.clients.account import AccountClient +from platformio.clients.http import HTTPClient, HTTPClientError +from platformio.package.meta import PackageType + +# pylint: disable=too-many-arguments + + +class RegistryClient(HTTPClient): + def __init__(self): + super(RegistryClient, self).__init__(__registry_api__) + + def send_auth_request(self, *args, **kwargs): + headers = kwargs.get("headers", {}) + if "Authorization" not in headers: + token = AccountClient().fetch_authentication_token() + headers["Authorization"] = "Bearer %s" % token + kwargs["headers"] = headers + return self.fetch_json_data(*args, **kwargs) + + def publish_package( + self, archive_path, owner=None, released_at=None, private=False, notify=True + ): + account = AccountClient() + if not owner: + owner = ( + account.get_account_info(offline=True).get("profile").get("username") + ) + with open(archive_path, "rb") as fp: + return self.send_auth_request( + "post", + "/v3/packages/%s/%s" % (owner, PackageType.from_archive(archive_path)), + params={ + "private": 1 if private else 0, + "notify": 1 if notify else 0, + "released_at": released_at, + }, + headers={ + "Content-Type": "application/octet-stream", + "X-PIO-Content-SHA256": fs.calculate_file_hashsum( + "sha256", archive_path + ), + }, + data=fp, + ) + + def unpublish_package( # pylint: disable=redefined-builtin + self, type, name, owner=None, version=None, undo=False + ): + account = AccountClient() + if not owner: + owner = ( + account.get_account_info(offline=True).get("profile").get("username") + ) + path = "/v3/packages/%s/%s/%s" % (owner, type, name) + if version: + path += "/" + version + return self.send_auth_request( + "delete", path, params={"undo": 1 if undo else 0}, + ) + + def update_resource(self, urn, private): + return self.send_auth_request( + "put", "/v3/resources/%s" % urn, data={"private": int(private)}, + ) + + def grant_access_for_resource(self, urn, client, level): + return self.send_auth_request( + "put", + "/v3/resources/%s/access" % urn, + data={"client": client, "level": level}, + ) + + def revoke_access_from_resource(self, urn, client): + return self.send_auth_request( + "delete", "/v3/resources/%s/access" % urn, data={"client": client}, + ) + + def list_resources(self, owner): + return self.send_auth_request( + "get", "/v3/resources", params={"owner": owner} if owner else None + ) + + def list_packages(self, query=None, filters=None, page=None): + assert query or filters + search_query = [] + if filters: + valid_filters = ( + "authors", + "keywords", + "frameworks", + "platforms", + "headers", + "ids", + "names", + "owners", + "types", + ) + assert set(filters.keys()) <= set(valid_filters) + for name, values in filters.items(): + for value in set( + values if isinstance(values, (list, tuple)) else [values] + ): + search_query.append('%s:"%s"' % (name[:-1], value)) + if query: + search_query.append(query) + params = dict(query=" ".join(search_query)) + if page: + params["page"] = int(page) + return self.fetch_json_data( + "get", "/v3/packages", params=params, cache_valid="1h" + ) + + def get_package(self, type_, owner, name, version=None): + try: + return self.fetch_json_data( + "get", + "/v3/packages/{owner}/{type}/{name}".format( + type=type_, owner=owner.lower(), name=name.lower() + ), + params=dict(version=version) if version else None, + cache_valid="1h", + ) + except HTTPClientError as e: + if e.response.status_code == 404: + return None + raise e diff --git a/platformio/commands/access.py b/platformio/commands/access.py new file mode 100644 index 0000000000..8b65ba34a0 --- /dev/null +++ b/platformio/commands/access.py @@ -0,0 +1,138 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json +import re + +import click +from tabulate import tabulate + +from platformio.clients.registry import RegistryClient +from platformio.commands.account import validate_username +from platformio.commands.team import validate_orgname_teamname + + +def validate_client(value): + if ":" in value: + validate_orgname_teamname(value) + else: + validate_username(value) + return value + + +@click.group("access", short_help="Manage resource access") +def cli(): + pass + + +def validate_urn(value): + value = str(value).strip() + if not re.match(r"^prn:reg:pkg:(\d+):(\w+)$", value, flags=re.I): + raise click.BadParameter("Invalid URN format.") + return value + + +@cli.command("public", short_help="Make resource public") +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") +def access_public(urn, urn_type): + client = RegistryClient() + client.update_resource(urn=urn, private=0) + return click.secho( + "The resource %s has been successfully updated." % urn, fg="green", + ) + + +@cli.command("private", short_help="Make resource private") +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") +def access_private(urn, urn_type): + client = RegistryClient() + client.update_resource(urn=urn, private=1) + return click.secho( + "The resource %s has been successfully updated." % urn, fg="green", + ) + + +@cli.command("grant", short_help="Grant access") +@click.argument("level", type=click.Choice(["admin", "maintainer", "guest"])) +@click.argument( + "client", + metavar="[|]", + callback=lambda _, __, value: validate_client(value), +) +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") +def access_grant(level, client, urn, urn_type): + reg_client = RegistryClient() + reg_client.grant_access_for_resource(urn=urn, client=client, level=level) + return click.secho( + "Access for resource %s has been granted for %s" % (urn, client), fg="green", + ) + + +@cli.command("revoke", short_help="Revoke access") +@click.argument( + "client", + metavar="[ORGNAME:TEAMNAME|USERNAME]", + callback=lambda _, __, value: validate_client(value), +) +@click.argument( + "urn", callback=lambda _, __, value: validate_urn(value), +) +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") +def access_revoke(client, urn, urn_type): + reg_client = RegistryClient() + reg_client.revoke_access_from_resource(urn=urn, client=client) + return click.secho( + "Access for resource %s has been revoked for %s" % (urn, client), fg="green", + ) + + +@cli.command("list", short_help="List published resources") +@click.argument("owner", required=False) +@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg") +@click.option("--json-output", is_flag=True) +def access_list(owner, urn_type, json_output): + reg_client = RegistryClient() + resources = reg_client.list_resources(owner=owner) + if json_output: + return click.echo(json.dumps(resources)) + if not resources: + return click.secho("You do not have any resources.", fg="yellow") + for resource in resources: + click.echo() + click.secho(resource.get("name"), fg="cyan") + click.echo("-" * len(resource.get("name"))) + table_data = [] + table_data.append(("URN:", resource.get("urn"))) + table_data.append(("Owner:", resource.get("owner"))) + table_data.append( + ( + "Access level(s):", + ", ".join( + (level.capitalize() for level in resource.get("access_levels")) + ), + ) + ) + click.echo(tabulate(table_data, tablefmt="plain")) + return click.echo() diff --git a/platformio/commands/account/command.py b/platformio/commands/account.py similarity index 86% rename from platformio/commands/account/command.py rename to platformio/commands/account.py index 0177d00a24..88aab68b90 100644 --- a/platformio/commands/account/command.py +++ b/platformio/commands/account.py @@ -21,22 +21,23 @@ import click from tabulate import tabulate -from platformio.commands.account import exception -from platformio.commands.account.client import AccountClient +from platformio.clients.account import AccountClient, AccountNotAuthorized -@click.group("account", short_help="Manage PIO Account") +@click.group("account", short_help="Manage PlatformIO account") def cli(): pass -def validate_username(value): +def validate_username(value, field="username"): value = str(value).strip() - if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){3,38}$", value, flags=re.I): + if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I): raise click.BadParameter( - "Invalid username format. " - "Username must contain at least 4 characters including single hyphens," - " and cannot begin or end with a hyphen" + "Invalid %s format. " + "%s must contain only alphanumeric characters " + "or single hyphens, cannot begin or end with a hyphen, " + "and must not be longer than 38 characters." + % (field.lower(), field.capitalize()) ) return value @@ -59,7 +60,7 @@ def validate_password(value): return value -@cli.command("register", short_help="Create new PIO Account") +@cli.command("register", short_help="Create new PlatformIO Account") @click.option( "-u", "--username", @@ -89,7 +90,7 @@ def account_register(username, email, password, firstname, lastname): ) -@cli.command("login", short_help="Log in to PIO Account") +@cli.command("login", short_help="Log in to PlatformIO Account") @click.option("-u", "--username", prompt="Username or email") @click.option("-p", "--password", prompt=True, hide_input=True) def account_login(username, password): @@ -98,7 +99,7 @@ def account_login(username, password): return click.secho("Successfully logged in!", fg="green") -@cli.command("logout", short_help="Log out of PIO Account") +@cli.command("logout", short_help="Log out of PlatformIO Account") def account_logout(): client = AccountClient() client.logout() @@ -167,7 +168,7 @@ def account_update(current_password, **kwargs): return None try: client.logout() - except exception.AccountNotAuthorized: + except AccountNotAuthorized: pass if email_changed: return click.secho( @@ -177,7 +178,24 @@ def account_update(current_password, **kwargs): return click.secho("Please re-login.", fg="yellow") -@cli.command("show", short_help="PIO Account information") +@cli.command("destroy", short_help="Destroy account") +def account_destroy(): + client = AccountClient() + click.confirm( + "Are you sure you want to delete the %s user account?\n" + "Warning! All linked data will be permanently removed and can not be restored." + % client.get_account_info().get("profile").get("username"), + abort=True, + ) + client.destroy_account() + try: + client.logout() + except AccountNotAuthorized: + pass + return click.secho("User account has been destroyed.", fg="green",) + + +@cli.command("show", short_help="PlatformIO Account information") @click.option("--offline", is_flag=True) @click.option("--json-output", is_flag=True) def account_show(offline, json_output): diff --git a/platformio/commands/account/client.py b/platformio/commands/account/client.py deleted file mode 100644 index fb679dc03d..0000000000 --- a/platformio/commands/account/client.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=unused-argument - -import os -import time - -import requests.adapters -from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error - -from platformio import __pioaccount_api__, app -from platformio.commands.account import exception -from platformio.exception import InternetIsOffline - - -class AccountClient(object): - - SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 - - def __init__( - self, api_base_url=__pioaccount_api__, retries=3, - ): - if api_base_url.endswith("/"): - api_base_url = api_base_url[:-1] - self.api_base_url = api_base_url - self._session = requests.Session() - self._session.headers.update({"User-Agent": app.get_user_agent()}) - retry = Retry( - total=retries, - read=retries, - connect=retries, - backoff_factor=2, - method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"], - ) - adapter = requests.adapters.HTTPAdapter(max_retries=retry) - self._session.mount(api_base_url, adapter) - - @staticmethod - def get_refresh_token(): - try: - return app.get_state_item("account").get("auth").get("refresh_token") - except: # pylint:disable=bare-except - raise exception.AccountNotAuthorized() - - @staticmethod - def delete_local_session(): - app.delete_state_item("account") - - @staticmethod - def delete_local_state(key): - account = app.get_state_item("account") - if not account or key not in account: - return - del account[key] - app.set_state_item("account", account) - - def login(self, username, password): - try: - self.fetch_authentication_token() - except: # pylint:disable=bare-except - pass - else: - raise exception.AccountAlreadyAuthorized( - app.get_state_item("account", {}).get("email", "") - ) - - result = self.send_request( - "post", - self.api_base_url + "/v1/login", - data={"username": username, "password": password}, - ) - app.set_state_item("account", result) - return result - - def login_with_code(self, client_id, code, redirect_uri): - try: - self.fetch_authentication_token() - except: # pylint:disable=bare-except - pass - else: - raise exception.AccountAlreadyAuthorized( - app.get_state_item("account", {}).get("email", "") - ) - - result = self.send_request( - "post", - self.api_base_url + "/v1/login/code", - data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri}, - ) - app.set_state_item("account", result) - return result - - def logout(self): - refresh_token = self.get_refresh_token() - self.delete_local_session() - try: - self.send_request( - "post", - self.api_base_url + "/v1/logout", - data={"refresh_token": refresh_token}, - ) - except exception.AccountError: - pass - return True - - def change_password(self, old_password, new_password): - token = self.fetch_authentication_token() - self.send_request( - "post", - self.api_base_url + "/v1/password", - headers={"Authorization": "Bearer %s" % token}, - data={"old_password": old_password, "new_password": new_password}, - ) - return True - - def registration( - self, username, email, password, firstname, lastname - ): # pylint:disable=too-many-arguments - try: - self.fetch_authentication_token() - except: # pylint:disable=bare-except - pass - else: - raise exception.AccountAlreadyAuthorized( - app.get_state_item("account", {}).get("email", "") - ) - - return self.send_request( - "post", - self.api_base_url + "/v1/registration", - data={ - "username": username, - "email": email, - "password": password, - "firstname": firstname, - "lastname": lastname, - }, - ) - - def auth_token(self, password, regenerate): - token = self.fetch_authentication_token() - result = self.send_request( - "post", - self.api_base_url + "/v1/token", - headers={"Authorization": "Bearer %s" % token}, - data={"password": password, "regenerate": 1 if regenerate else 0}, - ) - return result.get("auth_token") - - def forgot_password(self, username): - return self.send_request( - "post", self.api_base_url + "/v1/forgot", data={"username": username}, - ) - - def get_profile(self): - token = self.fetch_authentication_token() - return self.send_request( - "get", - self.api_base_url + "/v1/profile", - headers={"Authorization": "Bearer %s" % token}, - ) - - def update_profile(self, profile, current_password): - token = self.fetch_authentication_token() - profile["current_password"] = current_password - self.delete_local_state("summary") - response = self.send_request( - "put", - self.api_base_url + "/v1/profile", - headers={"Authorization": "Bearer %s" % token}, - data=profile, - ) - return response - - def get_account_info(self, offline): - account = app.get_state_item("account") - if not account: - raise exception.AccountNotAuthorized() - if ( - account.get("summary") - and account["summary"].get("expire_at", 0) > time.time() - ): - return account["summary"] - if offline: - return { - "profile": { - "email": account.get("email"), - "username": account.get("username"), - } - } - token = self.fetch_authentication_token() - result = self.send_request( - "get", - self.api_base_url + "/v1/summary", - headers={"Authorization": "Bearer %s" % token}, - ) - account["summary"] = dict( - profile=result.get("profile"), - packages=result.get("packages"), - subscriptions=result.get("subscriptions"), - user_id=result.get("user_id"), - expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL, - ) - app.set_state_item("account", account) - return result - - def fetch_authentication_token(self): - if "PLATFORMIO_AUTH_TOKEN" in os.environ: - return os.environ["PLATFORMIO_AUTH_TOKEN"] - auth = app.get_state_item("account", {}).get("auth", {}) - if auth.get("access_token") and auth.get("access_token_expire"): - if auth.get("access_token_expire") > time.time(): - return auth.get("access_token") - if auth.get("refresh_token"): - try: - result = self.send_request( - "post", - self.api_base_url + "/v1/login", - headers={ - "Authorization": "Bearer %s" % auth.get("refresh_token") - }, - ) - app.set_state_item("account", result) - return result.get("auth").get("access_token") - except exception.AccountError: - self.delete_local_session() - raise exception.AccountNotAuthorized() - - def send_request(self, method, url, headers=None, data=None): - try: - response = getattr(self._session, method)( - url, headers=headers or {}, data=data or {} - ) - except requests.exceptions.ConnectionError: - raise InternetIsOffline() - return self.raise_error_from_response(response) - - def raise_error_from_response(self, response, expected_codes=(200, 201, 202)): - if response.status_code in expected_codes: - try: - return response.json() - except ValueError: - pass - try: - message = response.json()["message"] - except (KeyError, ValueError): - message = response.text - if "Authorization session has been expired" in message: - self.delete_local_session() - raise exception.AccountError(message) diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index e7e2ecd6bb..4170b32ff7 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -19,10 +19,10 @@ from platformio import fs from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager -@click.command("boards", short_help="Embedded Board Explorer") +@click.command("boards", short_help="Embedded board explorer") @click.argument("query", required=False) @click.option("--installed", is_flag=True) @click.option("--json-output", is_flag=True) @@ -59,8 +59,8 @@ def print_boards(boards): click.style(b["id"], fg="cyan"), b["mcu"], "%dMHz" % (b["fcpu"] / 1000000), - fs.format_filesize(b["rom"]), - fs.format_filesize(b["ram"]), + fs.humanize_file_size(b["rom"]), + fs.humanize_file_size(b["ram"]), b["name"], ) for b in boards @@ -71,7 +71,7 @@ def print_boards(boards): def _get_boards(installed=False): - pm = PlatformManager() + pm = PlatformPackageManager() return pm.get_installed_boards() if installed else pm.get_all_boards() diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index a5c4e1e7f2..8f9a6dcae2 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -31,7 +31,7 @@ from platformio.project.helpers import find_project_dir_above, get_project_dir -@click.command("check", short_help="Run a static analysis tool on code") +@click.command("check", short_help="Static code analysis") @click.option("-e", "--environment", multiple=True) @click.option( "-d", diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index d6f5d4f124..dc9f476fc4 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob import os from tempfile import NamedTemporaryFile import click -from platformio import fs, proc +from platformio import compat, fs, proc from platformio.commands.check.defect import DefectItem from platformio.project.helpers import load_project_ide_data @@ -84,7 +83,9 @@ def _extract_defines(language, includes_file): cmd = "echo | %s -x %s %s %s -dM -E -" % ( self.cc_path, language, - " ".join([f for f in build_flags if f.startswith(("-m", "-f"))]), + " ".join( + [f for f in build_flags if f.startswith(("-m", "-f", "-std"))] + ), includes_file, ) result = proc.exec_command(cmd, shell=True) @@ -183,7 +184,7 @@ def _add_file(path): result["c++"].append(os.path.realpath(path)) for pattern in patterns: - for item in glob.glob(pattern): + for item in compat.glob_recursive(pattern): if not os.path.isdir(item): _add_file(item) for root, _, files in os.walk(item, followlinks=True): diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index f16104527f..06f3ff7616 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -17,7 +17,7 @@ from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class ClangtidyCheckTool(CheckToolBase): @@ -63,10 +63,7 @@ def configure_command(self): for scope in project_files: src_files.extend(project_files[scope]) - cmd.extend(flags) - cmd.extend(src_files) - cmd.append("--") - + cmd.extend(flags + src_files + ["--"]) cmd.extend( ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines["c++"]] ) @@ -79,6 +76,6 @@ def configure_command(self): continue includes.append(inc) - cmd.append("--extra-arg=" + self._long_includes_hook(includes)) + cmd.extend(["-I%s" % inc for inc in includes]) return cmd diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index 3412971428..b38bb8d69f 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -19,11 +19,13 @@ from platformio import proc from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class CppcheckCheckTool(CheckToolBase): def __init__(self, *args, **kwargs): + self._field_delimiter = "<&PIO&>" + self._buffer = "" self.defect_fields = [ "severity", "message", @@ -55,13 +57,15 @@ def tool_output_filter(self, line): return line def parse_defect(self, raw_line): - if "<&PIO&>" not in raw_line or any( - f not in raw_line for f in self.defect_fields - ): + if self._field_delimiter not in raw_line: + return None + + self._buffer += raw_line + if any(f not in self._buffer for f in self.defect_fields): return None args = dict() - for field in raw_line.split("<&PIO&>"): + for field in self._buffer.split(self._field_delimiter): field = field.strip().replace('"', "") name, value = field.split("=", 1) args[name] = value @@ -94,6 +98,7 @@ def parse_defect(self, raw_line): self._bad_input = True return None + self._buffer = "" return DefectItem(**args) def configure_command( @@ -103,13 +108,16 @@ def configure_command( cmd = [ tool_path, + "--addon-python=%s" % proc.get_pythonexe_path(), "--error-exitcode=1", "--verbose" if self.options.get("verbose") else "--quiet", ] cmd.append( '--template="%s"' - % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields]) + % self._field_delimiter.join( + ["{0}={{{0}}}".format(f) for f in self.defect_fields] + ) ) flags = self.get_flags("cppcheck") diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index 871ec4bc03..ce5d93ec62 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -22,7 +22,7 @@ from platformio import proc, util from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools.base import CheckToolBase -from platformio.managers.core import get_core_package_dir +from platformio.package.manager.core import get_core_package_dir class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-attributes diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 9a48f2622f..e72ddf76fb 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from glob import glob from os import getenv, makedirs, remove from os.path import basename, isdir, isfile, join, realpath from shutil import copyfile, copytree @@ -20,11 +19,10 @@ import click -from platformio import app, fs +from platformio import app, compat, fs from platformio.commands.project import project_init as cmd_project_init from platformio.commands.project import validate_boards from platformio.commands.run.command import cli as cmd_run -from platformio.compat import glob_escape from platformio.exception import CIBuildEnvsEmpty from platformio.project.config import ProjectConfig @@ -36,7 +34,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument if p.startswith("~"): value[i] = fs.expanduser(p) value[i] = realpath(value[i]) - if not glob(value[i]): + if not compat.glob_recursive(value[i]): invalid_path = p break try: @@ -46,7 +44,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument raise click.BadParameter("Found invalid path: %s" % invalid_path) -@click.command("ci", short_help="Continuous Integration") +@click.command("ci", short_help="Continuous integration") @click.argument("src", nargs=-1, callback=validate_path) @click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY") @click.option("--exclude", multiple=True) @@ -98,7 +96,7 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches continue contents = [] for p in patterns: - contents += glob(p) + contents += compat.glob_recursive(p) _copy_contents(join(build_dir, dir_name), contents) if project_conf and isfile(project_conf): @@ -159,7 +157,7 @@ def _copy_contents(dst_dir, contents): def _exclude_contents(dst_dir, patterns): contents = [] for p in patterns: - contents += glob(join(glob_escape(dst_dir), p)) + contents += compat.glob_recursive(join(compat.glob_escape(dst_dir), p)) for path in contents: path = realpath(path) if isdir(path): diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 2528611161..fc83405c0e 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -21,10 +21,10 @@ import click -from platformio import app, exception, fs, proc, util +from platformio import app, exception, fs, proc from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError -from platformio.managers.core import inject_contrib_pysite +from platformio.package.manager.core import inject_contrib_pysite from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project, load_project_ide_data @@ -33,7 +33,7 @@ @click.command( "debug", context_settings=dict(ignore_unknown_options=True), - short_help="PIO Unified Debugger", + short_help="Unified debugger", ) @click.option( "-d", @@ -130,7 +130,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro nl=False, ) stream = helpers.GDBMIConsoleStream() - with util.capture_std_streams(stream): + with proc.capture_std_streams(stream): helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) stream.close() else: diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 4604a86106..657e8c48ea 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -20,13 +20,14 @@ from io import BytesIO from os.path import isfile -from platformio import exception, fs, util +from platformio import fs, util from platformio.commands import PlatformioCLI from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.run.command import cli as cmd_run from platformio.compat import is_bytes -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.options import ProjectOptions @@ -94,14 +95,14 @@ def _cleanup_cmds(items): return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] try: - platform = PlatformFactory.newPlatform(env_options["platform"]) - except exception.UnknownPlatform: + platform = PlatformFactory.new(env_options["platform"]) + except UnknownPlatform: cmd_ctx.invoke( cmd_platform_install, platforms=[env_options["platform"]], skip_default_package=True, ) - platform = PlatformFactory.newPlatform(env_options["platform"]) + platform = PlatformFactory.new(env_options["platform"]) board_config = platform.board_config(env_options["board"]) tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) diff --git a/platformio/commands/debug/process/client.py b/platformio/commands/debug/process/client.py index a58438b755..45374727c1 100644 --- a/platformio/commands/debug/process/client.py +++ b/platformio/commands/debug/process/client.py @@ -26,7 +26,8 @@ from twisted.internet import stdio # pylint: disable=import-error from twisted.internet import task # pylint: disable=import-error -from platformio import app, fs, proc, telemetry, util +from platformio import fs, proc, telemetry, util +from platformio.cache import ContentCache from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.debug.initcfgs import get_gdb_init_config @@ -252,7 +253,7 @@ def _handle_error(self, data): def _kill_previous_session(self): assert self._session_id pid = None - with app.ContentCache() as cc: + with ContentCache() as cc: pid = cc.get(self._session_id) cc.delete(self._session_id) if not pid: @@ -269,11 +270,11 @@ def _kill_previous_session(self): def _lock_session(self, pid): if not self._session_id: return - with app.ContentCache() as cc: + with ContentCache() as cc: cc.set(self._session_id, str(pid), "1h") def _unlock_session(self): if not self._session_id: return - with app.ContentCache() as cc: + with ContentCache() as cc: cc.delete(self._session_id) diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index e93b1214d0..a66cb9961a 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -22,11 +22,11 @@ from platformio import exception, fs, util from platformio.commands.device import helpers as device_helpers from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformFactory +from platformio.platform.factory import PlatformFactory from platformio.project.exception import NotPlatformIOProjectError -@click.group(short_help="Monitor device or list existing") +@click.group(short_help="Device manager & serial/socket monitor") def cli(): pass @@ -192,7 +192,7 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches platform = None if "platform" in project_options: with fs.cd(kwargs["project_dir"]): - platform = PlatformFactory.newPlatform(project_options["platform"]) + platform = PlatformFactory.new(project_options["platform"]) device_helpers.register_platform_filters( platform, kwargs["project_dir"], kwargs["environment"] ) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 32d2806335..6cb26ed955 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -22,10 +22,10 @@ from platformio import exception from platformio.compat import WINDOWS -from platformio.managers.core import get_core_package_dir, inject_contrib_pysite +from platformio.package.manager.core import get_core_package_dir, inject_contrib_pysite -@click.command("home", short_help="PIO Home") +@click.command("home", short_help="UI to manage PlatformIO") @click.option("--port", type=int, default=8008, help="HTTP port, default=8008") @click.option( "--host", diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index 018d5da645..aff9228126 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -14,9 +14,6 @@ # pylint: disable=keyword-arg-before-vararg,arguments-differ,signature-differs -import os -import socket - import requests from twisted.internet import defer # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error @@ -52,18 +49,3 @@ def get_core_fullpath(): return where_is_program( "platformio" + (".exe" if "windows" in util.get_systype() else "") ) - - -@util.memoized(expire="10s") -def is_twitter_blocked(): - ip = "104.244.42.1" - timeout = 2 - try: - if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): - requests.get("http://%s" % ip, allow_redirects=False, timeout=timeout) - else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) - return False - except: # pylint: disable=bare-except - pass - return True diff --git a/platformio/commands/home/rpc/handlers/account.py b/platformio/commands/home/rpc/handlers/account.py index 911006bc41..d28379f83f 100644 --- a/platformio/commands/home/rpc/handlers/account.py +++ b/platformio/commands/home/rpc/handlers/account.py @@ -14,7 +14,7 @@ import jsonrpc # pylint: disable=import-error -from platformio.commands.account.client import AccountClient +from platformio.clients.account import AccountClient class AccountRPC(object): diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index d1851d1397..a4bdc6522a 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -17,15 +17,15 @@ from twisted.internet import defer, reactor # pylint: disable=import-error -from platformio import app +from platformio.cache import ContentCache from platformio.commands.home.rpc.handlers.os import OSRPC class MiscRPC(object): def load_latest_tweets(self, data_url): - cache_key = app.ContentCache.key_from_args(data_url, "tweets") - cache_valid = "7d" - with app.ContentCache() as cc: + cache_key = ContentCache.key_from_args(data_url, "tweets") + cache_valid = "180d" + with ContentCache() as cc: cache_data = cc.get(cache_key) if cache_data: cache_data = json.loads(cache_data) @@ -43,7 +43,7 @@ def load_latest_tweets(self, data_url): @defer.inlineCallbacks def _preload_latest_tweets(data_url, cache_key, cache_valid): result = json.loads((yield OSRPC.fetch_content(data_url))) - with app.ContentCache() as cc: + with ContentCache() as cc: cc.set( cache_key, json.dumps({"time": int(time.time()), "result": result}), diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 2997e8aaba..448c633a50 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -14,7 +14,6 @@ from __future__ import absolute_import -import glob import io import os import shutil @@ -23,9 +22,11 @@ import click from twisted.internet import defer # pylint: disable=import-error -from platformio import app, fs, util +from platformio import __default_requests_timeout__, fs, util +from platformio.cache import ContentCache +from platformio.clients.http import ensure_internet_on from platformio.commands.home import helpers -from platformio.compat import PY2, get_filesystem_encoding +from platformio.compat import PY2, get_filesystem_encoding, glob_recursive class OSRPC(object): @@ -40,26 +41,30 @@ def fetch_content(uri, data=None, headers=None, cache_valid=None): "Safari/603.3.8" ) } - cache_key = app.ContentCache.key_from_args(uri, data) if cache_valid else None - with app.ContentCache() as cc: + cache_key = ContentCache.key_from_args(uri, data) if cache_valid else None + with ContentCache() as cc: if cache_key: result = cc.get(cache_key) if result is not None: defer.returnValue(result) # check internet before and resolve issue with 60 seconds timeout - util.internet_on(raise_exception=True) + ensure_internet_on(raise_exception=True) session = helpers.requests_session() if data: - r = yield session.post(uri, data=data, headers=headers) + r = yield session.post( + uri, data=data, headers=headers, timeout=__default_requests_timeout__ + ) else: - r = yield session.get(uri, headers=headers) + r = yield session.get( + uri, headers=headers, timeout=__default_requests_timeout__ + ) r.raise_for_status() result = r.text if cache_valid: - with app.ContentCache() as cc: + with ContentCache() as cc: cc.set(cache_key, result, cache_valid) defer.returnValue(result) @@ -115,7 +120,9 @@ def glob(pathnames, root=None): pathnames = [pathnames] result = set() for pathname in pathnames: - result |= set(glob.glob(os.path.join(root, pathname) if root else pathname)) + result |= set( + glob_recursive(os.path.join(root, pathname) if root else pathname) + ) return list(result) @staticmethod diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 3f4cdc88f5..2db966b699 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -25,7 +25,7 @@ from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.compat import PY2, get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError from platformio.project.helpers import get_project_dir, is_platformio_project @@ -105,7 +105,7 @@ def _path_to_name(path): return (os.path.sep).join(path.split(os.path.sep)[-2:]) result = [] - pm = PlatformManager() + pm = PlatformPackageManager() for project_dir in AppRPC.load_state()["storage"]["recentProjects"]: if not os.path.isdir(project_dir): continue @@ -148,8 +148,9 @@ def _path_to_name(path): @staticmethod def get_project_examples(): result = [] - for manifest in PlatformManager().get_installed(): - examples_dir = os.path.join(manifest["__pkg_dir"], "examples") + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + examples_dir = os.path.join(pkg.path, "examples") if not os.path.isdir(examples_dir): continue items = [] @@ -172,6 +173,7 @@ def get_project_examples(): "description": project_description, } ) + manifest = pm.load_manifest(pkg) result.append( { "platform": { diff --git a/platformio/commands/account/exception.py b/platformio/commands/lib/__init__.py similarity index 64% rename from platformio/commands/account/exception.py rename to platformio/commands/lib/__init__.py index a1a0059e27..b051490361 100644 --- a/platformio/commands/account/exception.py +++ b/platformio/commands/lib/__init__.py @@ -11,20 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from platformio.exception import PlatformioException - - -class AccountError(PlatformioException): - - MESSAGE = "{0}" - - -class AccountNotAuthorized(AccountError): - - MESSAGE = "You are not authorized! Please log in to PIO Account." - - -class AccountAlreadyAuthorized(AccountError): - - MESSAGE = "You are already authorized with {0} account." diff --git a/platformio/commands/lib.py b/platformio/commands/lib/command.py similarity index 76% rename from platformio/commands/lib.py rename to platformio/commands/lib/command.py index d08529fdce..543e439ccf 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib/command.py @@ -18,18 +18,21 @@ import time import click -import semantic_version from tabulate import tabulate from platformio import exception, fs, util from platformio.commands import PlatformioCLI +from platformio.commands.lib.helpers import ( + get_builtin_libs, + is_builtin_lib, + save_project_libdeps, +) from platformio.compat import dump_json_to_unicode -from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib -from platformio.package.manifest.parser import ManifestParserFactory -from platformio.package.manifest.schema import ManifestSchema +from platformio.package.exception import NotGlobalLibDir, UnknownPackageError +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.meta import PackageItem, PackageSpec from platformio.proc import is_ci from platformio.project.config import ProjectConfig -from platformio.project.exception import InvalidProjectConfError from platformio.project.helpers import get_project_dir, is_platformio_project try: @@ -47,7 +50,7 @@ def get_project_global_lib_dir(): return ProjectConfig.get_instance().get_optional_dir("globallib") -@click.group(short_help="Library Manager") +@click.group(short_help="Library manager") @click.option( "-d", "--storage-dir", @@ -94,7 +97,7 @@ def cli(ctx, **options): ) if not storage_dirs: - raise exception.NotGlobalLibDir( + raise NotGlobalLibDir( get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand ) @@ -126,89 +129,106 @@ def cli(ctx, **options): @cli.command("install", short_help="Install library") @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") @click.option( - "--save", + "--save/--no-save", is_flag=True, - help="Save installed libraries into the `platformio.ini` dependency list", + default=True, + help="Save installed libraries into the `platformio.ini` dependency list" + " (enabled by default)", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option( - "--interactive", is_flag=True, help="Allow to make a choice for all prompts" + "--interactive", + is_flag=True, + help="Deprecated! Please use a strict dependency specification (owner/libname)", ) @click.option( "-f", "--force", is_flag=True, help="Reinstall/redownload library if exists" ) @click.pass_context -def lib_install( # pylint: disable=too-many-arguments +def lib_install( # pylint: disable=too-many-arguments,unused-argument ctx, libraries, save, silent, interactive, force ): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, []) - installed_manifests = {} + installed_pkgs = {} for storage_dir in storage_dirs: if not silent and (libraries or storage_dir in storage_libdeps): print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) + lm = LibraryPackageManager(storage_dir) + if libraries: - for library in libraries: - pkg_dir = lm.install( - library, silent=silent, interactive=interactive, force=force - ) - installed_manifests[library] = lm.load_manifest(pkg_dir) + installed_pkgs = { + library: lm.install(library, silent=silent, force=force) + for library in libraries + } + elif storage_dir in storage_libdeps: builtin_lib_storages = None for library in storage_libdeps[storage_dir]: try: - pkg_dir = lm.install( - library, silent=silent, interactive=interactive, force=force - ) - installed_manifests[library] = lm.load_manifest(pkg_dir) - except exception.LibNotFound as e: + lm.install(library, silent=silent, force=force) + except UnknownPackageError as e: if builtin_lib_storages is None: builtin_lib_storages = get_builtin_libs() if not silent or not is_builtin_lib(builtin_lib_storages, library): click.secho("Warning! %s" % e, fg="yellow") - if not save or not libraries: - return + if save and installed_pkgs: + _save_deps(ctx, installed_pkgs) + + +def _save_deps(ctx, pkgs, action="add"): + specs = [] + for library, pkg in pkgs.items(): + spec = PackageSpec(library) + if spec.external: + specs.append(spec) + else: + specs.append( + PackageSpec( + owner=pkg.metadata.spec.owner, + name=pkg.metadata.spec.name, + requirements=spec.requirements + or ( + ("^%s" % pkg.metadata.version) + if not pkg.metadata.version.build + else pkg.metadata.version + ), + ) + ) input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] for input_dir in input_dirs: - config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini")) - config.validate(project_environments) - for env in config.envs(): - if project_environments and env not in project_environments: - continue - config.expand_interpolations = False - try: - lib_deps = config.get("env:" + env, "lib_deps") - except InvalidProjectConfError: - lib_deps = [] - for library in libraries: - if library in lib_deps: - continue - manifest = installed_manifests[library] - try: - assert library.lower() == manifest["name"].lower() - assert semantic_version.Version(manifest["version"]) - lib_deps.append("{name}@^{version}".format(**manifest)) - except (AssertionError, ValueError): - lib_deps.append(library) - config.set("env:" + env, "lib_deps", lib_deps) - config.save() + if not is_platformio_project(input_dir): + continue + save_project_libdeps(input_dir, specs, project_environments, action=action) -@cli.command("uninstall", short_help="Uninstall libraries") +@cli.command("uninstall", short_help="Remove libraries") @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]") +@click.option( + "--save/--no-save", + is_flag=True, + default=True, + help="Remove libraries from the `platformio.ini` dependency list and save changes" + " (enabled by default)", +) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.pass_context -def lib_uninstall(ctx, libraries): +def lib_uninstall(ctx, libraries, save, silent): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] + uninstalled_pkgs = {} for storage_dir in storage_dirs: print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) - for library in libraries: - lm.uninstall(library) + lm = LibraryPackageManager(storage_dir) + uninstalled_pkgs = { + library: lm.uninstall(library, silent=silent) for library in libraries + } + + if save and uninstalled_pkgs: + _save_deps(ctx, uninstalled_pkgs, action="remove") @cli.command("update", short_help="Update installed libraries") @@ -222,42 +242,58 @@ def lib_uninstall(ctx, libraries): @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option("--json-output", is_flag=True) @click.pass_context -def lib_update(ctx, libraries, only_check, dry_run, json_output): +def lib_update( # pylint: disable=too-many-arguments + ctx, libraries, only_check, dry_run, silent, json_output +): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] only_check = dry_run or only_check json_result = {} for storage_dir in storage_dirs: if not json_output: print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) - - _libraries = libraries - if not _libraries: - _libraries = [manifest["__pkg_dir"] for manifest in lm.get_installed()] + lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, []) + lm = LibraryPackageManager(storage_dir) + _libraries = libraries or lib_deps or lm.get_installed() if only_check and json_output: result = [] for library in _libraries: - pkg_dir = library if os.path.isdir(library) else None - requirements = None - url = None - if not pkg_dir: - name, requirements, url = lm.parse_pkg_uri(library) - pkg_dir = lm.get_package_dir(name, requirements, url) - if not pkg_dir: + spec = None + pkg = None + if isinstance(library, PackageItem): + pkg = library + else: + spec = PackageSpec(library) + pkg = lm.get_package(spec) + if not pkg: continue - latest = lm.outdated(pkg_dir, requirements) - if not latest: + outdated = lm.outdated(pkg, spec) + if not outdated.is_outdated(allow_incompatible=True): continue - manifest = lm.load_manifest(pkg_dir) - manifest["versionLatest"] = latest + manifest = lm.legacy_load_manifest(pkg) + manifest["versionWanted"] = ( + str(outdated.wanted) if outdated.wanted else None + ) + manifest["versionLatest"] = ( + str(outdated.latest) if outdated.latest else None + ) result.append(manifest) json_result[storage_dir] = result else: for library in _libraries: - lm.update(library, only_check=only_check) + to_spec = ( + None if isinstance(library, PackageItem) else PackageSpec(library) + ) + try: + lm.update( + library, to_spec=to_spec, only_check=only_check, silent=silent + ) + except UnknownPackageError as e: + if library not in lib_deps: + raise e if json_output: return click.echo( @@ -278,8 +314,8 @@ def lib_list(ctx, json_output): for storage_dir in storage_dirs: if not json_output: print_storage_header(storage_dirs, storage_dir) - lm = LibraryManager(storage_dir) - items = lm.get_installed() + lm = LibraryPackageManager(storage_dir) + items = lm.legacy_get_installed() if json_output: json_result[storage_dir] = items elif items: @@ -303,6 +339,7 @@ def lib_list(ctx, json_output): @click.option("--json-output", is_flag=True) @click.option("--page", type=click.INT, default=1) @click.option("--id", multiple=True) +@click.option("-o", "--owner", multiple=True) @click.option("-n", "--name", multiple=True) @click.option("-a", "--author", multiple=True) @click.option("-k", "--keyword", multiple=True) @@ -315,6 +352,7 @@ def lib_list(ctx, json_output): help="Do not prompt, automatically paginate with delay", ) def lib_search(query, json_output, page, noninteractive, **filters): + regclient = LibraryPackageManager().get_registry_client_instance() if not query: query = [] if not isinstance(query, list): @@ -324,8 +362,11 @@ def lib_search(query, json_output, page, noninteractive, **filters): for value in values: query.append('%s:"%s"' % (key, value)) - result = util.get_api_result( - "/v2/lib/search", dict(query=" ".join(query), page=page), cache_valid="1d" + result = regclient.fetch_json_data( + "get", + "/v2/lib/search", + params=dict(query=" ".join(query), page=page), + cache_valid="1d", ) if json_output: @@ -374,9 +415,10 @@ def lib_search(query, json_output, page, noninteractive, **filters): time.sleep(5) elif not click.confirm("Show next libraries?"): break - result = util.get_api_result( + result = regclient.fetch_json_data( + "get", "/v2/lib/search", - {"query": " ".join(query), "page": int(result["page"]) + 1}, + params=dict(query=" ".join(query), page=int(result["page"]) + 1), cache_valid="1d", ) @@ -406,23 +448,20 @@ def lib_builtin(storage, json_output): @click.argument("library", metavar="[LIBRARY]") @click.option("--json-output", is_flag=True) def lib_show(library, json_output): - lm = LibraryManager() - name, requirements, _ = lm.parse_pkg_uri(library) - lib_id = lm.search_lib_id( - {"name": name, "requirements": requirements}, - silent=json_output, - interactive=not json_output, - ) - lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") + lm = LibraryPackageManager() + lib_id = lm.reveal_registry_package_id(library, silent=json_output) + regclient = lm.get_registry_client_instance() + lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h") if json_output: return click.echo(dump_json_to_unicode(lib)) - click.secho(lib["name"], fg="cyan") - click.echo("=" * len(lib["name"])) - click.secho("#ID: %d" % lib["id"], bold=True) + title = "{ownername}/{name}".format(**lib) + click.secho(title, fg="cyan") + click.echo("=" * len(title)) click.echo(lib["description"]) click.echo() + click.secho("ID: %d" % lib["id"]) click.echo( "Version: %s, released %s" % ( @@ -445,7 +484,7 @@ def lib_show(library, json_output): for author in lib.get("authors", []): _data = [] for key in ("name", "email", "url", "maintainer"): - if not author[key]: + if not author.get(key): continue if key == "email": _data.append("<%s>" % author[key]) @@ -495,29 +534,19 @@ def lib_show(library, json_output): return True -@cli.command("register", short_help="Register a new library") +@cli.command("register", short_help="Deprecated") @click.argument("config_url") -def lib_register(config_url): - if not config_url.startswith("http://") and not config_url.startswith("https://"): - raise exception.InvalidLibConfURL(config_url) - - # Validate manifest - ManifestSchema().load_manifest( - ManifestParserFactory.new_from_url(config_url).as_dict() +def lib_register(config_url): # pylint: disable=unused-argument + raise exception.UserSideException( + "This command is deprecated. Please use `pio package publish` command." ) - result = util.get_api_result("/lib/register", data=dict(config_url=config_url)) - if "message" in result and result["message"]: - click.secho( - result["message"], - fg="green" if "successed" in result and result["successed"] else "red", - ) - @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) def lib_stats(json_output): - result = util.get_api_result("/lib/stats", cache_valid="1h") + regclient = LibraryPackageManager().get_registry_client_instance() + result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h") if json_output: return click.echo(dump_json_to_unicode(result)) diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py new file mode 100644 index 0000000000..7a156e0f0a --- /dev/null +++ b/platformio/commands/lib/helpers.py @@ -0,0 +1,94 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio.compat import ci_strings_are_equal +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.meta import PackageSpec +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig +from platformio.project.exception import InvalidProjectConfError + + +def get_builtin_libs(storage_names=None): + # pylint: disable=import-outside-toplevel + from platformio.package.manager.library import LibraryPackageManager + + items = [] + storage_names = storage_names or [] + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + p = PlatformFactory.new(pkg) + for storage in p.get_lib_storages(): + if storage_names and storage["name"] not in storage_names: + continue + lm = LibraryPackageManager(storage["path"]) + items.append( + { + "name": storage["name"], + "path": storage["path"], + "items": lm.legacy_get_installed(), + } + ) + return items + + +def is_builtin_lib(storages, name): + for storage in storages or []: + if any(lib.get("name") == name for lib in storage["items"]): + return True + return False + + +def ignore_deps_by_specs(deps, specs): + result = [] + for dep in deps: + depspec = PackageSpec(dep) + if depspec.external: + result.append(dep) + continue + ignore_conditions = [] + for spec in specs: + if depspec.owner: + ignore_conditions.append( + ci_strings_are_equal(depspec.owner, spec.owner) + and ci_strings_are_equal(depspec.name, spec.name) + ) + else: + ignore_conditions.append(ci_strings_are_equal(depspec.name, spec.name)) + if not any(ignore_conditions): + result.append(dep) + return result + + +def save_project_libdeps(project_dir, specs, environments=None, action="add"): + config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) + config.validate(environments) + for env in config.envs(): + if environments and env not in environments: + continue + config.expand_interpolations = False + lib_deps = [] + try: + lib_deps = ignore_deps_by_specs(config.get("env:" + env, "lib_deps"), specs) + except InvalidProjectConfError: + pass + if action == "add": + lib_deps.extend(spec.as_dependency() for spec in specs) + if lib_deps: + config.set("env:" + env, "lib_deps", lib_deps) + elif config.has_option("env:" + env, "lib_deps"): + config.remove_option("env:" + env, "lib_deps") + config.save() diff --git a/platformio/commands/org.py b/platformio/commands/org.py new file mode 100644 index 0000000000..ac13d13f74 --- /dev/null +++ b/platformio/commands/org.py @@ -0,0 +1,150 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json + +import click +from tabulate import tabulate + +from platformio.clients.account import AccountClient +from platformio.commands.account import validate_email, validate_username + + +@click.group("org", short_help="Manage organizations") +def cli(): + pass + + +def validate_orgname(value): + return validate_username(value, "Organization name") + + +@cli.command("create", short_help="Create a new organization") +@click.argument( + "orgname", callback=lambda _, __, value: validate_orgname(value), +) +@click.option( + "--email", callback=lambda _, __, value: validate_email(value) if value else value +) +@click.option("--displayname",) +def org_create(orgname, email, displayname): + client = AccountClient() + client.create_org(orgname, email, displayname) + return click.secho( + "The organization `%s` has been successfully created." % orgname, fg="green", + ) + + +@cli.command("list", short_help="List organizations and their members") +@click.option("--json-output", is_flag=True) +def org_list(json_output): + client = AccountClient() + orgs = client.list_orgs() + if json_output: + return click.echo(json.dumps(orgs)) + if not orgs: + return click.echo("You do not have any organization") + for org in orgs: + click.echo() + click.secho(org.get("orgname"), fg="cyan") + click.echo("-" * len(org.get("orgname"))) + data = [] + if org.get("displayname"): + data.append(("Display Name:", org.get("displayname"))) + if org.get("email"): + data.append(("Email:", org.get("email"))) + data.append( + ( + "Owners:", + ", ".join((owner.get("username") for owner in org.get("owners"))), + ) + ) + click.echo(tabulate(data, tablefmt="plain")) + return click.echo() + + +@cli.command("update", short_help="Update organization") +@click.argument("cur_orgname") +@click.option( + "--orgname", + callback=lambda _, __, value: validate_orgname(value), + help="A new orgname", +) +@click.option("--email") +@click.option("--displayname") +def org_update(cur_orgname, **kwargs): + client = AccountClient() + org = client.get_org(cur_orgname) + del org["owners"] + new_org = org.copy() + if not any(kwargs.values()): + for field in org: + new_org[field] = click.prompt( + field.replace("_", " ").capitalize(), default=org[field] + ) + if field == "email": + validate_email(new_org[field]) + if field == "orgname": + validate_orgname(new_org[field]) + else: + new_org.update( + {key.replace("new_", ""): value for key, value in kwargs.items() if value} + ) + client.update_org(cur_orgname, new_org) + return click.secho( + "The organization `%s` has been successfully updated." % cur_orgname, + fg="green", + ) + + +@cli.command("destroy", short_help="Destroy organization") +@click.argument("orgname") +def account_destroy(orgname): + client = AccountClient() + click.confirm( + "Are you sure you want to delete the `%s` organization account?\n" + "Warning! All linked data will be permanently removed and can not be restored." + % orgname, + abort=True, + ) + client.destroy_org(orgname) + return click.secho("Organization `%s` has been destroyed." % orgname, fg="green",) + + +@cli.command("add", short_help="Add a new owner to organization") +@click.argument("orgname",) +@click.argument("username",) +def org_add_owner(orgname, username): + client = AccountClient() + client.add_org_owner(orgname, username) + return click.secho( + "The new owner `%s` has been successfully added to the `%s` organization." + % (username, orgname), + fg="green", + ) + + +@cli.command("remove", short_help="Remove an owner from organization") +@click.argument("orgname",) +@click.argument("username",) +def org_remove_owner(orgname, username): + client = AccountClient() + client.remove_org_owner(orgname, username) + return click.secho( + "The `%s` owner has been successfully removed from the `%s` organization." + % (username, orgname), + fg="green", + ) diff --git a/platformio/commands/package.py b/platformio/commands/package.py new file mode 100644 index 0000000000..88f6c0d3ed --- /dev/null +++ b/platformio/commands/package.py @@ -0,0 +1,113 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from datetime import datetime + +import click + +from platformio.clients.registry import RegistryClient +from platformio.package.meta import PackageSpec, PackageType +from platformio.package.pack import PackagePacker + + +def validate_datetime(ctx, param, value): # pylint: disable=unused-argument + if not value: + return value + try: + datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + except ValueError as e: + raise click.BadParameter(e) + return value + + +@click.group("package", short_help="Package manager") +def cli(): + pass + + +@cli.command("pack", short_help="Create a tarball from a package") +@click.argument( + "package", + required=True, + default=os.getcwd, + metavar="", +) +@click.option( + "-o", "--output", help="A destination path (folder or a full path to file)" +) +def package_pack(package, output): + p = PackagePacker(package) + archive_path = p.pack(output) + click.secho('Wrote a tarball to "%s"' % archive_path, fg="green") + + +@cli.command("publish", short_help="Publish a package to the registry") +@click.argument( + "package", + required=True, + default=os.getcwd, + metavar="", +) +@click.option( + "--owner", + help="PIO Account username (can be organization username). " + "Default is set to a username of the authorized PIO Account", +) +@click.option( + "--released-at", + callback=validate_datetime, + help="Custom release date and time in the next format (UTC): 2014-06-13 17:08:52", +) +@click.option("--private", is_flag=True, help="Restricted access (not a public)") +@click.option( + "--notify/--no-notify", + default=True, + help="Notify by email when package is processed", +) +def package_publish(package, owner, released_at, private, notify): + p = PackagePacker(package) + archive_path = p.pack() + response = RegistryClient().publish_package( + archive_path, owner, released_at, private, notify + ) + os.remove(archive_path) + click.secho(response.get("message"), fg="green") + + +@cli.command("unpublish", short_help="Remove a pushed package from the registry") +@click.argument( + "package", required=True, metavar="[/][@]" +) +@click.option( + "--type", + type=click.Choice(list(PackageType.items().values())), + default="library", + help="Package type, default is set to `library`", +) +@click.option( + "--undo", + is_flag=True, + help="Undo a remove, putting a version back into the registry", +) +def package_unpublish(package, type, undo): # pylint: disable=redefined-builtin + spec = PackageSpec(package) + response = RegistryClient().unpublish_package( + type=type, + name=spec.name, + owner=spec.owner, + version=spec.requirements, + undo=undo, + ) + click.secho(response.get("message"), fg="green") diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index d4ff49309a..054a7a1217 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -12,18 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import dirname, isdir +import os import click -from platformio import app, exception, util +from platformio.cache import cleanup_content_cache from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformFactory, PlatformManager -from platformio.package.pack import PackagePacker +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.meta import PackageItem, PackageSpec +from platformio.package.version import get_original_version +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory -@click.group(short_help="Platform Manager") +@click.group(short_help="Platform manager") def cli(): pass @@ -47,7 +50,7 @@ def _print_platforms(platforms): if "version" in platform: if "__src_url" in platform: click.echo( - "Version: #%s (%s)" % (platform["version"], platform["__src_url"]) + "Version: %s (%s)" % (platform["version"], platform["__src_url"]) ) else: click.echo("Version: " + platform["version"]) @@ -55,31 +58,27 @@ def _print_platforms(platforms): def _get_registry_platforms(): - platforms = util.get_api_result("/platforms", cache_valid="7d") - pm = PlatformManager() - for platform in platforms or []: - platform["versions"] = pm.get_all_repo_versions(platform["name"]) - return platforms + regclient = PlatformPackageManager().get_registry_client_instance() + return regclient.fetch_json_data("get", "/v2/platforms", cache_valid="1d") def _get_platform_data(*args, **kwargs): try: return _get_installed_platform_data(*args, **kwargs) - except exception.UnknownPlatform: + except UnknownPlatform: return _get_registry_platform_data(*args, **kwargs) def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): - p = PlatformFactory.newPlatform(platform) + p = PlatformFactory.new(platform) data = dict( name=p.name, title=p.title, description=p.description, version=p.version, homepage=p.homepage, + url=p.homepage, repository=p.repository_url, - url=p.vendor_url, - docs=p.docs_url, license=p.license, forDesktop=not p.is_embedded(), frameworks=sorted(list(p.frameworks) if p.frameworks else []), @@ -91,7 +90,9 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru # return data # overwrite VCS version and add extra fields - manifest = PlatformManager().load_manifest(dirname(p.manifest_path)) + manifest = PlatformPackageManager().legacy_load_manifest( + os.path.dirname(p.manifest_path) + ) assert manifest for key in manifest: if key == "version" or key.startswith("__"): @@ -104,13 +105,15 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru return data data["packages"] = [] - installed_pkgs = p.get_installed_packages() - for name, opts in p.packages.items(): + installed_pkgs = { + pkg.metadata.name: p.pm.load_manifest(pkg) for pkg in p.get_installed_packages() + } + for name, options in p.packages.items(): item = dict( name=name, type=p.get_package_type(name), - requirements=opts.get("version"), - optional=opts.get("optional") is True, + requirements=options.get("version"), + optional=options.get("optional") is True, ) if name in installed_pkgs: for key, value in installed_pkgs[name].items(): @@ -118,7 +121,7 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=Tru continue item[key] = value if key == "version": - item["originalVersion"] = util.get_original_version(value) + item["originalVersion"] = get_original_version(value) data["packages"].append(item) return data @@ -137,6 +140,7 @@ def _get_registry_platform_data( # pylint: disable=unused-argument return None data = dict( + ownername=_data.get("ownername"), name=_data["name"], title=_data["title"], description=_data["description"], @@ -147,13 +151,13 @@ def _get_registry_platform_data( # pylint: disable=unused-argument forDesktop=_data["forDesktop"], frameworks=_data["frameworks"], packages=_data["packages"], - versions=_data["versions"], + versions=_data.get("versions"), ) if with_boards: data["boards"] = [ board - for board in PlatformManager().get_registered_boards() + for board in PlatformPackageManager().get_registered_boards() if board["platform"] == _data["name"] ] @@ -187,8 +191,11 @@ def platform_search(query, json_output): @click.argument("query", required=False) @click.option("--json-output", is_flag=True) def platform_frameworks(query, json_output): + regclient = PlatformPackageManager().get_registry_client_instance() frameworks = [] - for framework in util.get_api_result("/frameworks", cache_valid="7d"): + for framework in regclient.fetch_json_data( + "get", "/v2/frameworks", cache_valid="1d" + ): if query == "all": query = "" search_data = dump_json_to_unicode(framework) @@ -213,12 +220,10 @@ def platform_frameworks(query, json_output): @click.option("--json-output", is_flag=True) def platform_list(json_output): platforms = [] - pm = PlatformManager() - for manifest in pm.get_installed(): + pm = PlatformPackageManager() + for pkg in pm.get_installed(): platforms.append( - _get_installed_platform_data( - manifest["__pkg_dir"], with_boards=False, expose_packages=False - ) + _get_installed_platform_data(pkg, with_boards=False, expose_packages=False) ) platforms = sorted(platforms, key=lambda manifest: manifest["name"]) @@ -234,16 +239,15 @@ def platform_list(json_output): def platform_show(platform, json_output): # pylint: disable=too-many-branches data = _get_platform_data(platform) if not data: - raise exception.UnknownPlatform(platform) + raise UnknownPlatform(platform) if json_output: return click.echo(dump_json_to_unicode(data)) + dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"] click.echo( - "{name} ~ {title}".format( - name=click.style(data["name"], fg="cyan"), title=data["title"] - ) + "{dep} ~ {title}".format(dep=click.style(dep, fg="cyan"), title=data["title"]) ) - click.echo("=" * (3 + len(data["name"] + data["title"]))) + click.echo("=" * (3 + len(dep + data["title"]))) click.echo(data["description"]) click.echo() if "version" in data: @@ -300,6 +304,7 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches @click.option("--without-package", multiple=True) @click.option("--skip-default-package", is_flag=True) @click.option("--with-all-packages", is_flag=True) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option( "-f", "--force", @@ -312,21 +317,24 @@ def platform_install( # pylint: disable=too-many-arguments without_package, skip_default_package, with_all_packages, + silent, force, ): - pm = PlatformManager() + pm = PlatformPackageManager() for platform in platforms: - if pm.install( - name=platform, + pkg = pm.install( + spec=platform, with_packages=with_package, without_packages=without_package, skip_default_package=skip_default_package, with_all_packages=with_all_packages, + silent=silent, force=force, - ): + ) + if pkg and not silent: click.secho( "The platform '%s' has been successfully installed!\n" - "The rest of packages will be installed automatically " + "The rest of the packages will be installed later " "depending on your build environment." % platform, fg="green", ) @@ -335,11 +343,11 @@ def platform_install( # pylint: disable=too-many-arguments @cli.command("uninstall", short_help="Uninstall development platform") @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]") def platform_uninstall(platforms): - pm = PlatformManager() + pm = PlatformPackageManager() for platform in platforms: if pm.uninstall(platform): click.secho( - "The platform '%s' has been successfully uninstalled!" % platform, + "The platform '%s' has been successfully removed!" % platform, fg="green", ) @@ -358,66 +366,60 @@ def platform_uninstall(platforms): @click.option( "--dry-run", is_flag=True, help="Do not update, only check for the new versions" ) +@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option("--json-output", is_flag=True) -def platform_update( # pylint: disable=too-many-locals - platforms, only_packages, only_check, dry_run, json_output +def platform_update( # pylint: disable=too-many-locals, too-many-arguments + platforms, only_packages, only_check, dry_run, silent, json_output ): - pm = PlatformManager() - pkg_dir_to_name = {} - if not platforms: - platforms = [] - for manifest in pm.get_installed(): - platforms.append(manifest["__pkg_dir"]) - pkg_dir_to_name[manifest["__pkg_dir"]] = manifest.get( - "title", manifest["name"] - ) - + pm = PlatformPackageManager() + platforms = platforms or pm.get_installed() only_check = dry_run or only_check if only_check and json_output: result = [] for platform in platforms: - pkg_dir = platform if isdir(platform) else None - requirements = None - url = None - if not pkg_dir: - name, requirements, url = pm.parse_pkg_uri(platform) - pkg_dir = pm.get_package_dir(name, requirements, url) - if not pkg_dir: + spec = None + pkg = None + if isinstance(platform, PackageItem): + pkg = platform + else: + spec = PackageSpec(platform) + pkg = pm.get_package(spec) + if not pkg: continue - latest = pm.outdated(pkg_dir, requirements) + outdated = pm.outdated(pkg, spec) if ( - not latest - and not PlatformFactory.newPlatform(pkg_dir).are_outdated_packages() + not outdated.is_outdated(allow_incompatible=True) + and not PlatformFactory.new(pkg).are_outdated_packages() ): continue data = _get_installed_platform_data( - pkg_dir, with_boards=False, expose_packages=False + pkg, with_boards=False, expose_packages=False ) - if latest: - data["versionLatest"] = latest + if outdated.is_outdated(allow_incompatible=True): + data["versionLatest"] = ( + str(outdated.latest) if outdated.latest else None + ) result.append(data) return click.echo(dump_json_to_unicode(result)) # cleanup cached board and platform lists - app.clean_cache() + cleanup_content_cache("http") + for platform in platforms: click.echo( "Platform %s" - % click.style(pkg_dir_to_name.get(platform, platform), fg="cyan") + % click.style( + platform.metadata.name + if isinstance(platform, PackageItem) + else platform, + fg="cyan", + ) ) click.echo("--------") - pm.update(platform, only_packages=only_packages, only_check=only_check) + pm.update( + platform, only_packages=only_packages, only_check=only_check, silent=silent + ) click.echo() return True - - -@cli.command( - "pack", short_help="Create a tarball from development platform/tool package" -) -@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]") -def platform_pack(package): - p = PackagePacker(package) - tarball_path = p.pack() - click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green") diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 3d73f4ffcd..f861b5b199 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -12,23 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-arguments,too-many-locals, too-many-branches +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,line-too-long +import json import os import click from tabulate import tabulate -from platformio import exception, fs +from platformio import fs from platformio.commands.platform import platform_install as cli_platform_install from platformio.ide.projectgenerator import ProjectGenerator -from platformio.managers.platform import PlatformManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.exception import UnknownBoard from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError -from platformio.project.helpers import is_platformio_project +from platformio.project.helpers import is_platformio_project, load_project_ide_data -@click.group(short_help="Project Manager") +@click.group(short_help="Project manager") def cli(): pass @@ -38,9 +40,7 @@ def cli(): "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), ) @click.option("--json-output", is_flag=True) def project_config(project_dir, json_output): @@ -54,7 +54,6 @@ def project_config(project_dir, json_output): "Computed project configuration for %s" % click.style(project_dir, fg="cyan") ) for section, options in config.as_tuple(): - click.echo() click.secho(section, fg="cyan") click.echo("-" * len(section)) click.echo( @@ -66,15 +65,55 @@ def project_config(project_dir, json_output): tablefmt="plain", ) ) + click.echo() + return None + + +@cli.command("data", short_help="Dump data intended for IDE extensions/plugins") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), +) +@click.option("-e", "--environment", multiple=True) +@click.option("--json-output", is_flag=True) +def project_data(project_dir, environment, json_output): + if not is_platformio_project(project_dir): + raise NotPlatformIOProjectError(project_dir) + with fs.cd(project_dir): + config = ProjectConfig.get_instance() + config.validate(environment) + environment = list(environment or config.envs()) + + if json_output: + return click.echo(json.dumps(load_project_ide_data(project_dir, environment))) + + for envname in environment: + click.echo("Environment: " + click.style(envname, fg="cyan", bold=True)) + click.echo("=" * (13 + len(envname))) + click.echo( + tabulate( + [ + (click.style(name, bold=True), "=", json.dumps(value, indent=2)) + for name, value in load_project_ide_data( + project_dir, envname + ).items() + ], + tablefmt="plain", + ) + ) + click.echo() + return None def validate_boards(ctx, param, value): # pylint: disable=W0613 - pm = PlatformManager() + pm = PlatformPackageManager() for id_ in value: try: pm.board_config(id_) - except exception.UnknownBoard: + except UnknownBoard: raise click.BadParameter( "`%s`. Please search for board ID using `platformio boards` " "command" % id_ @@ -93,6 +132,7 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 ) @click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) @click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) +@click.option("-e", "--environment", help="Update using existing environment") @click.option("-O", "--project-option", multiple=True) @click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) @@ -102,6 +142,7 @@ def project_init( project_dir, board, ide, + environment, project_option, env_prefix, silent, @@ -139,11 +180,17 @@ def project_init( ) if ide: - pg = ProjectGenerator(project_dir, ide, board) + with fs.cd(project_dir): + config = ProjectConfig.get_instance( + os.path.join(project_dir, "platformio.ini") + ) + config.validate() + pg = ProjectGenerator( + config, environment or get_best_envname(config, board), ide + ) pg.generate() if is_new_project: - init_ci_conf(project_dir) init_cvs_ignore(project_dir) if silent: @@ -233,7 +280,6 @@ def init_include_readme(include_dir): def init_lib_readme(lib_dir): - # pylint: disable=line-too-long with open(os.path.join(lib_dir, "README"), "w") as fp: fp.write( """ @@ -290,7 +336,7 @@ def init_test_readme(test_dir): with open(os.path.join(test_dir, "README"), "w") as fp: fp.write( """ -This directory is intended for PIO Unit Testing and project tests. +This directory is intended for PlatformIO Unit Testing and project tests. Unit Testing is a software testing method by which individual units of source code, sets of one or more MCU program modules together with associated @@ -298,89 +344,12 @@ def init_test_readme(test_dir): determine whether they are fit for use. Unit testing finds problems early in the development cycle. -More information about PIO Unit Testing: +More information about PlatformIO Unit Testing: - https://docs.platformio.org/page/plus/unit-testing.html """, ) -def init_ci_conf(project_dir): - conf_path = os.path.join(project_dir, ".travis.yml") - if os.path.isfile(conf_path): - return - with open(conf_path, "w") as fp: - fp.write( - """# Continuous Integration (CI) is the practice, in software -# engineering, of merging all developer working copies with a shared mainline -# several times a day < https://docs.platformio.org/page/ci/index.html > -# -# Documentation: -# -# * Travis CI Embedded Builds with PlatformIO -# < https://docs.travis-ci.com/user/integration/platformio/ > -# -# * PlatformIO integration with Travis CI -# < https://docs.platformio.org/page/ci/travis.html > -# -# * User Guide for `platformio ci` command -# < https://docs.platformio.org/page/userguide/cmd_ci.html > -# -# -# Please choose one of the following templates (proposed below) and uncomment -# it (remove "# " before each line) or use own configuration according to the -# Travis CI documentation (see above). -# - - -# -# Template #1: General project. Test it using existing `platformio.ini`. -# - -# language: python -# python: -# - "2.7" -# -# sudo: false -# cache: -# directories: -# - "~/.platformio" -# -# install: -# - pip install -U platformio -# - platformio update -# -# script: -# - platformio run - - -# -# Template #2: The project is intended to be used as a library with examples. -# - -# language: python -# python: -# - "2.7" -# -# sudo: false -# cache: -# directories: -# - "~/.platformio" -# -# env: -# - PLATFORMIO_CI_SRC=path/to/test/file.c -# - PLATFORMIO_CI_SRC=examples/file.ino -# - PLATFORMIO_CI_SRC=path/to/test/directory -# -# install: -# - pip install -U platformio -# - platformio update -# -# script: -# - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N -""", - ) - - def init_cvs_ignore(project_dir): conf_path = os.path.join(project_dir, ".gitignore") if os.path.isfile(conf_path): @@ -401,7 +370,7 @@ def fill_project_envs( if all(cond): used_boards.append(config.get(section, "board")) - pm = PlatformManager() + pm = PlatformPackageManager() used_platforms = [] modified = False for id_ in board_ids: @@ -438,9 +407,31 @@ def fill_project_envs( def _install_dependent_platforms(ctx, platforms): - installed_platforms = [p["name"] for p in PlatformManager().get_installed()] + installed_platforms = [ + pkg.metadata.name for pkg in PlatformPackageManager().get_installed() + ] if set(platforms) <= set(installed_platforms): return ctx.invoke( cli_platform_install, platforms=list(set(platforms) - set(installed_platforms)) ) + + +def get_best_envname(config, board_ids=None): + envname = None + default_envs = config.default_envs() + if default_envs: + envname = default_envs[0] + if not board_ids: + return envname + + for env in config.envs(): + if not board_ids: + return env + if not envname: + envname = env + items = config.items(env=env, as_dict=True) + if "board" in items and items.get("board") in board_ids: + return env + + return envname diff --git a/platformio/commands/remote/client/base.py b/platformio/commands/remote/client/base.py index 806d7bda23..7ca7be3b1d 100644 --- a/platformio/commands/remote/client/base.py +++ b/platformio/commands/remote/client/base.py @@ -72,7 +72,7 @@ def _log_observer(self, event): def connect(self): self.log.info("Name: {name}", name=self.name) - self.log.info("Connecting to PIO Remote Cloud") + self.log.info("Connecting to PlatformIO Remote Development Cloud") # pylint: disable=protected-access proto, options = endpoints._parse(__pioremote_endpoint__) diff --git a/platformio/commands/remote/client/run_or_test.py b/platformio/commands/remote/client/run_or_test.py index c986ad0a79..10a9b008da 100644 --- a/platformio/commands/remote/client/run_or_test.py +++ b/platformio/commands/remote/client/run_or_test.py @@ -20,7 +20,7 @@ from twisted.spread import pb # pylint: disable=import-error -from platformio import util +from platformio import fs from platformio.commands.remote.client.async_base import AsyncClientBase from platformio.commands.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync from platformio.compat import hashlib_encode_data @@ -64,7 +64,7 @@ def generate_project_id(self, path): return "%s-%s" % (os.path.basename(path), h.hexdigest()) def add_project_items(self, psync): - with util.cd(self.options["project_dir"]): + with fs.cd(self.options["project_dir"]): cfg = ProjectConfig.get_instance( os.path.join(self.options["project_dir"], "platformio.ini") ) diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index f9e24c299c..cafbbd1f30 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -29,18 +29,19 @@ from platformio.commands.run.command import cli as cmd_run from platformio.commands.test.command import cli as cmd_test from platformio.compat import PY2 -from platformio.managers.core import inject_contrib_pysite +from platformio.package.manager.core import inject_contrib_pysite from platformio.project.exception import NotPlatformIOProjectError -@click.group("remote", short_help="PIO Remote") +@click.group("remote", short_help="Remote development") @click.option("-a", "--agent", multiple=True) @click.pass_context def cli(ctx, agent): if PY2: raise exception.UserSideException( - "PIO Remote requires Python 3.5 or above. \nPlease install the latest " - "Python 3 and reinstall PlatformIO Core using installation script:\n" + "PlatformIO Remote Development requires Python 3.5 or above. \n" + "Please install the latest Python 3 and reinstall PlatformIO Core using " + "installation script:\n" "https://docs.platformio.org/page/core/installation.html" ) ctx.obj = agent diff --git a/platformio/commands/remote/factory/client.py b/platformio/commands/remote/factory/client.py index 26abe08017..2b47ab01a6 100644 --- a/platformio/commands/remote/factory/client.py +++ b/platformio/commands/remote/factory/client.py @@ -17,7 +17,7 @@ from twisted.spread import pb # pylint: disable=import-error from platformio.app import get_host_id -from platformio.commands.account.client import AccountClient +from platformio.clients.account import AccountClient class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory): diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 378eaf0d9a..db4b412175 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import operator +import os from multiprocessing import cpu_count -from os import getcwd -from os.path import isfile from time import time import click @@ -26,7 +26,7 @@ from platformio.commands.run.processor import EnvironmentProcessor from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING from platformio.project.config import ProjectConfig -from platformio.project.helpers import find_project_dir_above +from platformio.project.helpers import find_project_dir_above, load_project_ide_data # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @@ -36,14 +36,14 @@ DEFAULT_JOB_NUMS = 1 -@click.command("run", short_help="Process project environments") +@click.command("run", short_help="Run project targets (build, upload, clean, etc.)") @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") @click.option( "-d", "--project-dir", - default=getcwd, + default=os.getcwd, type=click.Path( exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True ), @@ -68,6 +68,7 @@ @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--disable-auto-clean", is_flag=True) +@click.option("--list-targets", is_flag=True) @click.pass_context def cli( ctx, @@ -80,11 +81,12 @@ def cli( silent, verbose, disable_auto_clean, + list_targets, ): app.set_session_var("custom_project_conf", project_conf) # find project directory on upper level - if isfile(project_dir): + if os.path.isfile(project_dir): project_dir = find_project_dir_above(project_dir) is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta @@ -93,6 +95,9 @@ def cli( config = ProjectConfig.get_instance(project_conf) config.validate(environment) + if list_targets: + return print_target_list(list(environment) or config.envs()) + # clean obsolete build dir if not disable_auto_clean: build_dir = config.get_optional_dir("build") @@ -142,7 +147,7 @@ def cli( command_failed = any(r.get("succeeded") is False for r in results) if not is_test_running and (command_failed or not silent) and len(results) > 1: - print_processing_summary(results) + print_processing_summary(results, verbose) if command_failed: raise exception.ReturnErrorCode(1) @@ -215,7 +220,7 @@ def print_processing_footer(result): ) -def print_processing_summary(results): +def print_processing_summary(results, verbose=False): tabular_data = [] succeeded_nums = 0 failed_nums = 0 @@ -227,6 +232,8 @@ def print_processing_summary(results): failed_nums += 1 status_str = click.style("FAILED", fg="red") elif result.get("succeeded") is None: + if not verbose: + continue status_str = "IGNORED" else: succeeded_nums += 1 @@ -261,3 +268,33 @@ def print_processing_summary(results): is_error=failed_nums, fg="red" if failed_nums else "green", ) + + +def print_target_list(envs): + tabular_data = [] + for env, data in load_project_ide_data(os.getcwd(), envs).items(): + tabular_data.extend( + sorted( + [ + ( + click.style(env, fg="cyan"), + t["group"], + click.style(t.get("name"), fg="yellow"), + t["title"], + t.get("description"), + ) + for t in data.get("targets", []) + ], + key=operator.itemgetter(1, 2), + ) + ) + tabular_data.append((None, None, None, None, None)) + click.echo( + tabulate( + tabular_data, + headers=[ + click.style(s, bold=True) + for s in ("Environment", "Group", "Name", "Title", "Description") + ], + ), + ) diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 23ccc33329..d07c581ca4 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio import exception from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.exception import UndefinedEnvPlatformError # pylint: disable=too-many-instance-attributes @@ -67,14 +67,14 @@ def process(self): build_targets.remove("monitor") try: - p = PlatformFactory.newPlatform(self.options["platform"]) - except exception.UnknownPlatform: + p = PlatformFactory.new(self.options["platform"]) + except UnknownPlatform: self.cmd_ctx.invoke( cmd_platform_install, platforms=[self.options["platform"]], skip_default_package=True, ) - p = PlatformFactory.newPlatform(self.options["platform"]) + p = PlatformFactory.new(self.options["platform"]) result = p.run(build_vars, build_targets, self.silent, self.verbose, self.jobs) return result["returncode"] == 0 diff --git a/platformio/commands/settings.py b/platformio/commands/settings.py index 7f03f81bdf..695d902034 100644 --- a/platformio/commands/settings.py +++ b/platformio/commands/settings.py @@ -27,7 +27,7 @@ def format_value(raw): return str(raw) -@click.group(short_help="Manage PlatformIO settings") +@click.group(short_help="Manage system settings") def cli(): pass diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index 48336bfd27..cb31120566 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -12,17 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. - +import json +import os +import platform import subprocess +import sys import click +from tabulate import tabulate -from platformio import proc +from platformio import __version__, compat, fs, proc, util from platformio.commands.system.completion import ( get_completion_install_path, install_completion_code, uninstall_completion_code, ) +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_cache_dir @click.group("system", short_help="Miscellaneous system commands") @@ -30,6 +39,85 @@ def cli(): pass +@cli.command("info", short_help="Display system-wide information") +@click.option("--json-output", is_flag=True) +def system_info(json_output): + project_config = ProjectConfig() + data = {} + data["core_version"] = {"title": "PlatformIO Core", "value": __version__} + data["python_version"] = { + "title": "Python", + "value": "{0}.{1}.{2}-{3}.{4}".format(*list(sys.version_info)), + } + data["system"] = {"title": "System Type", "value": util.get_systype()} + data["platform"] = {"title": "Platform", "value": platform.platform(terse=True)} + data["filesystem_encoding"] = { + "title": "File System Encoding", + "value": compat.get_filesystem_encoding(), + } + data["locale_encoding"] = { + "title": "Locale Encoding", + "value": compat.get_locale_encoding(), + } + data["core_dir"] = { + "title": "PlatformIO Core Directory", + "value": project_config.get_optional_dir("core"), + } + data["platformio_exe"] = { + "title": "PlatformIO Core Executable", + "value": proc.where_is_program( + "platformio.exe" if proc.WINDOWS else "platformio" + ), + } + data["python_exe"] = { + "title": "Python Executable", + "value": proc.get_pythonexe_path(), + } + data["global_lib_nums"] = { + "title": "Global Libraries", + "value": len(LibraryPackageManager().get_installed()), + } + data["dev_platform_nums"] = { + "title": "Development Platforms", + "value": len(PlatformPackageManager().get_installed()), + } + data["package_tool_nums"] = { + "title": "Tools & Toolchains", + "value": len( + ToolPackageManager( + project_config.get_optional_dir("packages") + ).get_installed() + ), + } + + click.echo( + json.dumps(data) + if json_output + else tabulate([(item["title"], item["value"]) for item in data.values()]) + ) + + +@cli.command("prune", short_help="Remove unused data") +@click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation") +def system_prune(force): + click.secho("WARNING! This will remove:", fg="yellow") + click.echo(" - cached API requests") + click.echo(" - cached package downloads") + click.echo(" - temporary data") + if not force: + click.confirm("Do you want to continue?", abort=True) + + reclaimed_total = 0 + cache_dir = get_project_cache_dir() + if os.path.isdir(cache_dir): + reclaimed_total += fs.calculate_folder_size(cache_dir) + fs.rmtree(cache_dir) + + click.secho( + "Total reclaimed space: %s" % fs.humanize_file_size(reclaimed_total), fg="green" + ) + + @cli.group("completion", short_help="Shell completion support") def completion(): # pylint: disable=import-error,import-outside-toplevel diff --git a/platformio/commands/team.py b/platformio/commands/team.py new file mode 100644 index 0000000000..7c1e863816 --- /dev/null +++ b/platformio/commands/team.py @@ -0,0 +1,203 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import json +import re + +import click +from tabulate import tabulate + +from platformio.clients.account import AccountClient + + +def validate_orgname_teamname(value, teamname_validate=False): + if ":" not in value: + raise click.BadParameter( + "Please specify organization and team name in the next" + " format - orgname:teamname. For example, mycompany:DreamTeam" + ) + teamname = str(value.strip().split(":", 1)[1]) + if teamname_validate: + validate_teamname(teamname) + return value + + +def validate_teamname(value): + if not value: + return value + value = str(value).strip() + if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I): + raise click.BadParameter( + "Invalid team name format. " + "Team name must only contain alphanumeric characters, " + "single hyphens, underscores, spaces. It can not " + "begin or end with a hyphen or a underscore and must" + " not be longer than 20 characters." + ) + return value + + +@click.group("team", short_help="Manage organization teams") +def cli(): + pass + + +@cli.command("create", short_help="Create a new team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname( + value, teamname_validate=True + ), +) +@click.option("--description",) +def team_create(orgname_teamname, description): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.create_team(orgname, teamname, description) + return click.secho( + "The team %s has been successfully created." % teamname, fg="green", + ) + + +@cli.command("list", short_help="List teams") +@click.argument("orgname", required=False) +@click.option("--json-output", is_flag=True) +def team_list(orgname, json_output): + client = AccountClient() + data = {} + if not orgname: + for item in client.list_orgs(): + teams = client.list_teams(item.get("orgname")) + data[item.get("orgname")] = teams + else: + teams = client.list_teams(orgname) + data[orgname] = teams + if json_output: + return click.echo(json.dumps(data[orgname] if orgname else data)) + if not any(data.values()): + return click.secho("You do not have any teams.", fg="yellow") + for org_name in data: + for team in data[org_name]: + click.echo() + click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan") + click.echo("-" * len("%s:%s" % (org_name, team.get("name")))) + table_data = [] + if team.get("description"): + table_data.append(("Description:", team.get("description"))) + table_data.append( + ( + "Members:", + ", ".join( + (member.get("username") for member in team.get("members")) + ) + if team.get("members") + else "-", + ) + ) + click.echo(tabulate(table_data, tablefmt="plain")) + return click.echo() + + +@cli.command("update", short_help="Update team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.option( + "--name", + callback=lambda _, __, value: validate_teamname(value), + help="A new team name", +) +@click.option("--description",) +def team_update(orgname_teamname, **kwargs): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + team = client.get_team(orgname, teamname) + del team["id"] + del team["members"] + new_team = team.copy() + if not any(kwargs.values()): + for field in team: + new_team[field] = click.prompt( + field.replace("_", " ").capitalize(), default=team[field] + ) + if field == "name": + validate_teamname(new_team[field]) + else: + new_team.update({key: value for key, value in kwargs.items() if value}) + client.update_team(orgname, teamname, new_team) + return click.secho( + "The team %s has been successfully updated." % teamname, fg="green", + ) + + +@cli.command("destroy", short_help="Destroy a team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +def team_destroy(orgname_teamname): + orgname, teamname = orgname_teamname.split(":", 1) + click.confirm( + click.style( + "Are you sure you want to destroy the %s team?" % teamname, fg="yellow" + ), + abort=True, + ) + client = AccountClient() + client.destroy_team(orgname, teamname) + return click.secho( + "The team %s has been successfully destroyed." % teamname, fg="green", + ) + + +@cli.command("add", short_help="Add a new member to team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.argument("username",) +def team_add_member(orgname_teamname, username): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.add_team_member(orgname, teamname, username) + return click.secho( + "The new member %s has been successfully added to the %s team." + % (username, teamname), + fg="green", + ) + + +@cli.command("remove", short_help="Remove a member from team") +@click.argument( + "orgname_teamname", + metavar="ORGNAME:TEAMNAME", + callback=lambda _, __, value: validate_orgname_teamname(value), +) +@click.argument("username") +def team_remove_owner(orgname_teamname, username): + orgname, teamname = orgname_teamname.split(":", 1) + client = AccountClient() + client.remove_team_member(orgname, teamname, username) + return click.secho( + "The %s member has been successfully removed from the %s team." + % (username, teamname), + fg="green", + ) diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index b57b1d5918..13104bb2f9 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -28,7 +28,7 @@ from platformio.project.config import ProjectConfig -@click.command("test", short_help="Unit Testing") +@click.command("test", short_help="Unit testing") @click.option("--environment", "-e", multiple=True, metavar="") @click.option( "--filter", diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index 6f47eafcab..ca6584961b 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -19,7 +19,7 @@ from platformio import exception, util from platformio.commands.test.processor import TestProcessorBase -from platformio.managers.platform import PlatformFactory +from platformio.platform.factory import PlatformFactory class EmbeddedTestProcessor(TestProcessorBase): @@ -108,7 +108,7 @@ def get_test_port(self): return self.env_options.get("test_port") assert set(["platform", "board"]) & set(self.env_options.keys()) - p = PlatformFactory.newPlatform(self.env_options["platform"]) + p = PlatformFactory.new(self.env_options["platform"]) board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", []) port = None elapsed = 0 diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 9024ed0eec..de09b5f9a4 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -13,7 +13,7 @@ # limitations under the License. import atexit -from os import remove +from os import listdir, remove from os.path import isdir, isfile, join from string import Template @@ -25,33 +25,39 @@ "arduino": { "include": "#include ", "object": "", - "putchar": "Serial.write(c)", - "flush": "Serial.flush()", - "begin": "Serial.begin($baudrate)", - "end": "Serial.end()", + "putchar": "Serial.write(c);", + "flush": "Serial.flush();", + "begin": "Serial.begin($baudrate);", + "end": "Serial.end();", "language": "cpp", }, "mbed": { "include": "#include ", - "object": "Serial pc(USBTX, USBRX);", - "putchar": "pc.putc(c)", + "object": ( + "#if MBED_MAJOR_VERSION == 6\nUnbufferedSerial pc(USBTX, USBRX);\n" + "#else\nRawSerial pc(USBTX, USBRX);\n#endif" + ), + "putchar": ( + "#if MBED_MAJOR_VERSION == 6\npc.write(&c, 1);\n" + "#else\npc.putc(c);\n#endif" + ), "flush": "", - "begin": "pc.baud($baudrate)", + "begin": "pc.baud($baudrate);", "end": "", "language": "cpp", }, "espidf": { "include": "#include ", "object": "", - "putchar": "putchar(c)", - "flush": "fflush(stdout)", + "putchar": "putchar(c);", + "flush": "fflush(stdout);", "begin": "", "end": "", }, "zephyr": { "include": "#include ", "object": "", - "putchar": 'printk("%c", c)', + "putchar": 'printk("%c", c);', "flush": "", "begin": "", "end": "", @@ -59,18 +65,18 @@ "native": { "include": "#include ", "object": "", - "putchar": "putchar(c)", - "flush": "fflush(stdout)", + "putchar": "putchar(c);", + "flush": "fflush(stdout);", "begin": "", "end": "", }, "custom": { "include": '#include "unittest_transport.h"', "object": "", - "putchar": "unittest_uart_putchar(c)", - "flush": "unittest_uart_flush()", - "begin": "unittest_uart_begin()", - "end": "unittest_uart_end()", + "putchar": "unittest_uart_putchar(c);", + "flush": "unittest_uart_flush();", + "begin": "unittest_uart_begin();", + "end": "unittest_uart_end();", "language": "cpp", }, } @@ -132,6 +138,7 @@ def build_or_upload(self, target): return self.cmd_ctx.invoke( cmd_run, project_dir=self.options["project_dir"], + project_conf=self.options["project_config"].path, upload_port=self.options["upload_port"], verbose=self.options["verbose"], silent=self.options["silent"], @@ -174,44 +181,50 @@ def generate_output_file(self, test_dir): "void output_start(unsigned int baudrate)", "#endif", "{", - " $begin;", + " $begin", "}", "", "void output_char(int c)", "{", - " $putchar;", + " $putchar", "}", "", "void output_flush(void)", "{", - " $flush;", + " $flush", "}", "", "void output_complete(void)", "{", - " $end;", + " $end", "}", ] ) - def delete_tmptest_file(file_): - try: - remove(file_) - except: # pylint: disable=bare-except - if isfile(file_): - click.secho( - "Warning: Could not remove temporary file '%s'. " - "Please remove it manually." % file_, - fg="yellow", - ) + tmp_file_prefix = "tmp_pio_test_transport" + + def delete_tmptest_files(test_dir): + for item in listdir(test_dir): + if item.startswith(tmp_file_prefix) and isfile(join(test_dir, item)): + try: + remove(join(test_dir, item)) + except: # pylint: disable=bare-except + click.secho( + "Warning: Could not remove temporary file '%s'. " + "Please remove it manually." % join(test_dir, item), + fg="yellow", + ) transport_options = TRANSPORT_OPTIONS[self.get_transport()] tpl = Template(file_tpl).substitute(transport_options) data = Template(tpl).substitute(baudrate=self.get_baudrate()) + + delete_tmptest_files(test_dir) tmp_file = join( - test_dir, "output_export." + transport_options.get("language", "c") + test_dir, + "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")), ) with open(tmp_file, "w") as fp: fp.write(data) - atexit.register(delete_tmptest_file, tmp_file) + atexit.register(delete_tmptest_files, test_dir) diff --git a/platformio/commands/update.py b/platformio/commands/update.py index 1bac4f777f..ff88723efc 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -14,12 +14,12 @@ import click -from platformio import app -from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY -from platformio.commands.lib import lib_update as cmd_lib_update +from platformio.cache import cleanup_content_cache +from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY +from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update -from platformio.managers.core import update_core_packages -from platformio.managers.lib import LibraryManager +from platformio.package.manager.core import update_core_packages +from platformio.package.manager.library import LibraryPackageManager @click.command( @@ -38,7 +38,7 @@ @click.pass_context def cli(ctx, core_packages, only_check, dry_run): # cleanup lib search results, cached board and platform lists - app.clean_cache() + cleanup_content_cache("http") only_check = dry_run or only_check @@ -55,5 +55,5 @@ def cli(ctx, core_packages, only_check, dry_run): click.echo() click.echo("Library Manager") click.echo("===============") - ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryManager().package_dir] + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryPackageManager().package_dir] ctx.invoke(cmd_lib_update, only_check=only_check) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 6303ea69a5..2411f49cf5 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import re from zipfile import ZipFile import click -import requests from platformio import VERSION, __version__, app, exception +from platformio.clients.http import fetch_remote_content from platformio.compat import WINDOWS from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir @@ -130,13 +131,11 @@ def get_latest_version(): def get_develop_latest_version(): version = None - r = requests.get( + content = fetch_remote_content( "https://raw.githubusercontent.com/platformio/platformio" - "/develop/platformio/__init__.py", - headers={"User-Agent": app.get_user_agent()}, + "/develop/platformio/__init__.py" ) - r.raise_for_status() - for line in r.text.split("\n"): + for line in content.split("\n"): line = line.strip() if not line.startswith("VERSION"): continue @@ -152,9 +151,5 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - r = requests.get( - "https://pypi.org/pypi/platformio/json", - headers={"User-Agent": app.get_user_agent()}, - ) - r.raise_for_status() - return r.json()["info"]["version"] + content = fetch_remote_content("https://pypi.org/pypi/platformio/json") + return json.loads(content)["info"]["version"] diff --git a/platformio/compat.py b/platformio/compat.py index c812e98d00..59362d01b3 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -13,8 +13,9 @@ # limitations under the License. # pylint: disable=unused-import, no-name-in-module, import-error, -# pylint: disable=no-member, undefined-variable +# pylint: disable=no-member, undefined-variable, unexpected-keyword-arg +import glob import inspect import json import locale @@ -49,6 +50,14 @@ def get_object_members(obj, ignore_private=True): } +def ci_strings_are_equal(a, b): + if a == b: + return True + if not a or not b: + return False + return a.strip().lower() == b.strip().lower() + + if PY2: import imp @@ -81,6 +90,9 @@ def dump_json_to_unicode(obj): _magic_check = re.compile("([*?[])") _magic_check_bytes = re.compile(b"([*?[])") + def glob_recursive(pathname): + return glob.glob(pathname) + def glob_escape(pathname): """Escape all special characters.""" # https://github.com/python/cpython/blob/master/Lib/glob.py#L161 @@ -122,6 +134,9 @@ def dump_json_to_unicode(obj): return obj return json.dumps(obj, ensure_ascii=False, sort_keys=True) + def glob_recursive(pathname): + return glob.glob(pathname, recursive=True) + def load_python_module(name, pathname): spec = importlib.util.spec_from_file_location(name, pathname) module = importlib.util.module_from_spec(spec) diff --git a/platformio/exception.py b/platformio/exception.py index d291ad7fd0..ef1d3bab5a 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -30,10 +30,6 @@ class ReturnErrorCode(PlatformioException): MESSAGE = "{0}" -class LockFileTimeoutError(PlatformioException): - pass - - class MinitermException(PlatformioException): pass @@ -47,141 +43,6 @@ class AbortedByUser(UserSideException): MESSAGE = "Aborted by user" -# -# Development Platform -# - - -class UnknownPlatform(PlatformioException): - - MESSAGE = "Unknown development platform '{0}'" - - -class IncompatiblePlatform(PlatformioException): - - MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}" - - -class PlatformNotInstalledYet(PlatformioException): - - MESSAGE = ( - "The platform '{0}' has not been installed yet. " - "Use `platformio platform install {0}` command" - ) - - -class UnknownBoard(PlatformioException): - - MESSAGE = "Unknown board ID '{0}'" - - -class InvalidBoardManifest(PlatformioException): - - MESSAGE = "Invalid board JSON manifest '{0}'" - - -class UnknownFramework(PlatformioException): - - MESSAGE = "Unknown framework '{0}'" - - -# Package Manager - - -class PlatformIOPackageException(PlatformioException): - pass - - -class UnknownPackage(UserSideException): - - MESSAGE = "Detected unknown package '{0}'" - - -class MissingPackageManifest(PlatformIOPackageException): - - MESSAGE = "Could not find one of '{0}' manifest files in the package" - - -class UndefinedPackageVersion(PlatformIOPackageException): - - MESSAGE = ( - "Could not find a version that satisfies the requirement '{0}'" - " for your system '{1}'" - ) - - -class PackageInstallError(PlatformIOPackageException): - - MESSAGE = ( - "Could not install '{0}' with version requirements '{1}' " - "for your system '{2}'.\n\n" - "Please try this solution -> http://bit.ly/faq-package-manager" - ) - - -class ExtractArchiveItemError(PlatformIOPackageException): - - MESSAGE = ( - "Could not extract `{0}` to `{1}`. Try to disable antivirus " - "tool or check this solution -> http://bit.ly/faq-package-manager" - ) - - -class UnsupportedArchiveType(PlatformIOPackageException): - - MESSAGE = "Can not unpack file '{0}'" - - -class FDUnrecognizedStatusCode(PlatformIOPackageException): - - MESSAGE = "Got an unrecognized status code '{0}' when downloaded {1}" - - -class FDSizeMismatch(PlatformIOPackageException): - - MESSAGE = ( - "The size ({0:d} bytes) of downloaded file '{1}' " - "is not equal to remote size ({2:d} bytes)" - ) - - -class FDSHASumMismatch(PlatformIOPackageException): - - MESSAGE = ( - "The 'sha1' sum '{0}' of downloaded file '{1}' is not equal to remote '{2}'" - ) - - -# -# Library -# - - -class LibNotFound(PlatformioException): - - MESSAGE = ( - "Library `{0}` has not been found in PlatformIO Registry.\n" - "You can ignore this message, if `{0}` is a built-in library " - "(included in framework, SDK). E.g., SPI, Wire, etc." - ) - - -class NotGlobalLibDir(UserSideException): - - MESSAGE = ( - "The `{0}` is not a PlatformIO project.\n\n" - "To manage libraries in global storage `{1}`,\n" - "please use `platformio lib --global {2}` or specify custom storage " - "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" - "Check `platformio lib --help` for details." - ) - - -class InvalidLibConfURL(UserSideException): - - MESSAGE = "Invalid library config URL '{0}'" - - # # UDEV Rules # @@ -194,8 +55,8 @@ class InvalidUdevRules(PlatformioException): class MissedUdevRules(InvalidUdevRules): MESSAGE = ( - "Warning! Please install `99-platformio-udev.rules`. \nMode details: " - "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules" + "Warning! Please install `99-platformio-udev.rules`. \nMore details: " + "https://docs.platformio.org/page/faq.html#platformio-udev-rules" ) @@ -203,8 +64,8 @@ class OutdatedUdevRules(InvalidUdevRules): MESSAGE = ( "Warning! Your `{0}` are outdated. Please update or reinstall them." - "\n Mode details: https://docs.platformio.org" - "/en/latest/faq.html#platformio-udev-rules" + "\nMore details: " + "https://docs.platformio.org/page/faq.html#platformio-udev-rules" ) @@ -223,25 +84,6 @@ class GetLatestVersionError(PlatformioException): MESSAGE = "Can not retrieve the latest PlatformIO version" -class APIRequestError(PlatformioException): - - MESSAGE = "[API] {0}" - - -class InternetIsOffline(UserSideException): - - MESSAGE = ( - "You are not connected to the Internet.\n" - "PlatformIO needs the Internet connection to" - " download dependent packages or to work with PIO Account." - ) - - -class BuildScriptNotFound(PlatformioException): - - MESSAGE = "Invalid path '{0}' to build script" - - class InvalidSettingName(UserSideException): MESSAGE = "Invalid setting with the name '{0}'" diff --git a/platformio/fs.py b/platformio/fs.py index 575a14e57f..da2101c5fb 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -12,18 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import hashlib +import io import json import os import re import shutil import stat import sys -from glob import glob import click from platformio import exception -from platformio.compat import WINDOWS, glob_escape +from platformio.compat import WINDOWS, glob_escape, glob_recursive class cd(object): @@ -56,7 +57,7 @@ def load_json(file_path): raise exception.InvalidJSONFile(file_path) -def format_filesize(filesize): +def humanize_file_size(filesize): base = 1024 unit = 0 suffix = "B" @@ -73,6 +74,28 @@ def format_filesize(filesize): return "%d%sB" % ((base * filesize / unit), suffix) +def calculate_file_hashsum(algorithm, path): + h = hashlib.new(algorithm) + with io.open(path, "rb", buffering=0) as fp: + while True: + chunk = fp.read(io.DEFAULT_BUFFER_SIZE) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def calculate_folder_size(path): + assert os.path.isdir(path) + result = 0 + for root, __, files in os.walk(path): + for f in files: + file_path = os.path.join(root, f) + if not os.path.islink(file_path): + result += os.path.getsize(file_path) + return result + + def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel @@ -135,7 +158,7 @@ def _append_build_item(items, item, src_dir): src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter): items = set() - for item in glob(os.path.join(glob_escape(src_dir), pattern)): + for item in glob_recursive(os.path.join(glob_escape(src_dir), pattern)): if os.path.isdir(item): for root, _, files in os.walk(item, followlinks=followlinks): for f in files: @@ -164,6 +187,10 @@ def expanduser(path): return os.environ["USERPROFILE"] + path[1:] +def change_filemtime(path, mtime): + os.utime(path, (mtime, mtime)) + + def rmtree(path): def _onerror(func, path, __): try: diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 8bf735b0a7..30eaa97d62 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -15,47 +15,31 @@ import codecs import os import sys -from os.path import basename, isdir, isfile, join, realpath, relpath import bottle from platformio import fs, util from platformio.proc import where_is_program -from platformio.project.config import ProjectConfig from platformio.project.helpers import load_project_ide_data class ProjectGenerator(object): - def __init__(self, project_dir, ide, boards): - self.config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) - self.config.validate() - self.project_dir = project_dir + def __init__(self, config, env_name, ide): + self.config = config + self.project_dir = os.path.dirname(config.path) + self.env_name = str(env_name) self.ide = str(ide) - self.env_name = str(self.get_best_envname(boards)) @staticmethod def get_supported_ides(): - tpls_dir = join(fs.get_source_dir(), "ide", "tpls") - return sorted([d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) - - def get_best_envname(self, boards=None): - envname = None - default_envs = self.config.default_envs() - if default_envs: - envname = default_envs[0] - if not boards: - return envname - - for env in self.config.envs(): - if not boards: - return env - if not envname: - envname = env - items = self.config.items(env=env, as_dict=True) - if "board" in items and items.get("board") in boards: - return env - - return envname + tpls_dir = os.path.join(fs.get_source_dir(), "ide", "tpls") + return sorted( + [ + d + for d in os.listdir(tpls_dir) + if os.path.isdir(os.path.join(tpls_dir, d)) + ] + ) @staticmethod def filter_includes(includes_map, ignore_scopes=None, to_unix_path=True): @@ -75,12 +59,12 @@ def _load_tplvars(self): tpl_vars = { "config": self.config, "systype": util.get_systype(), - "project_name": basename(self.project_dir), + "project_name": os.path.basename(self.project_dir), "project_dir": self.project_dir, "env_name": self.env_name, - "user_home_dir": realpath(fs.expanduser("~")), + "user_home_dir": os.path.realpath(fs.expanduser("~")), "platformio_path": sys.argv[0] - if isfile(sys.argv[0]) + if os.path.isfile(sys.argv[0]) else where_is_program("platformio"), "env_path": os.getenv("PATH"), "env_pathsep": os.pathsep, @@ -97,7 +81,7 @@ def _load_tplvars(self): "src_files": self.get_src_files(), "project_src_dir": self.config.get_optional_dir("src"), "project_lib_dir": self.config.get_optional_dir("lib"), - "project_libdeps_dir": join( + "project_libdeps_dir": os.path.join( self.config.get_optional_dir("libdeps"), self.env_name ), } @@ -120,12 +104,12 @@ def get_src_files(self): with fs.cd(self.project_dir): for root, _, files in os.walk(self.config.get_optional_dir("src")): for f in files: - result.append(relpath(join(root, f))) + result.append(os.path.relpath(os.path.join(root, f))) return result def get_tpls(self): tpls = [] - tpls_dir = join(fs.get_source_dir(), "ide", "tpls", self.ide) + tpls_dir = os.path.join(fs.get_source_dir(), "ide", "tpls", self.ide) for root, _, files in os.walk(tpls_dir): for f in files: if not f.endswith(".tpl"): @@ -133,7 +117,7 @@ def get_tpls(self): _relpath = root.replace(tpls_dir, "") if _relpath.startswith(os.sep): _relpath = _relpath[1:] - tpls.append((_relpath, join(root, f))) + tpls.append((_relpath, os.path.join(root, f))) return tpls def generate(self): @@ -141,12 +125,12 @@ def generate(self): for tpl_relpath, tpl_path in self.get_tpls(): dst_dir = self.project_dir if tpl_relpath: - dst_dir = join(self.project_dir, tpl_relpath) - if not isdir(dst_dir): + dst_dir = os.path.join(self.project_dir, tpl_relpath) + if not os.path.isdir(dst_dir): os.makedirs(dst_dir) - file_name = basename(tpl_path)[:-4] + file_name = os.path.basename(tpl_path)[:-4] contents = self._render_tpl(tpl_path, tpl_vars) - self._merge_contents(join(dst_dir, file_name), contents) + self._merge_contents(os.path.join(dst_dir, file_name), contents) @staticmethod def _render_tpl(tpl_path, tpl_vars): @@ -155,7 +139,7 @@ def _render_tpl(tpl_path, tpl_vars): @staticmethod def _merge_contents(dst_path, contents): - if basename(dst_path) == ".gitignore" and isfile(dst_path): + if os.path.basename(dst_path) == ".gitignore" and os.path.isfile(dst_path): return with codecs.open(dst_path, "w", encoding="utf8") as fp: fp.write(contents) diff --git a/platformio/ide/tpls/emacs/.gitignore.tpl b/platformio/ide/tpls/emacs/.gitignore.tpl index b8e379fa64..6f8bafd31b 100644 --- a/platformio/ide/tpls/emacs/.gitignore.tpl +++ b/platformio/ide/tpls/emacs/.gitignore.tpl @@ -1,2 +1,3 @@ .pio .clang_complete +.ccls diff --git a/platformio/ide/tpls/vim/.gitignore.tpl b/platformio/ide/tpls/vim/.gitignore.tpl index bbdd36c798..1159b2d8b4 100644 --- a/platformio/ide/tpls/vim/.gitignore.tpl +++ b/platformio/ide/tpls/vim/.gitignore.tpl @@ -1,3 +1,4 @@ .pio .clang_complete .gcc-flags.json +.ccls diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index 930854d34e..e6dda8957d 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -10,10 +10,6 @@ % return to_unix_path(text).replace('"', '\\"') % end % -% def _escape_required(flag): -% return " " in flag and systype == "windows" -% end -% % def split_args(args_string): % return click.parser.split_arg_string(to_unix_path(args_string)) % end @@ -53,10 +49,7 @@ % def _find_forced_includes(flags, inc_paths): % result = [] % include_args = ("-include", "-imacros") -% for f in flags: -% if not f.startswith(include_args): -% continue -% end +% for f in filter_args(flags, include_args): % for arg in include_args: % inc = "" % if f.startswith(arg) and f.split(arg)[1].strip(): @@ -66,6 +59,7 @@ % end % if inc: % result.append(_find_abs_path(inc, inc_paths)) +% break % end % end % end @@ -134,8 +128,7 @@ "compilerPath": "{{ cc_path }}", "compilerArgs": [ % for flag in [ -% '"%s"' % _escape(f) if _escape_required(f) else f -% for f in filter_args(cc_m_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) +% f for f in filter_args(cc_m_flags, ["-m", "-i", "@"], ["-include", "-imacros"]) % ]: "{{ flag }}", % end diff --git a/platformio/maintenance.py b/platformio/maintenance.py index d2e7ea1cac..e038bcc097 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -19,15 +19,21 @@ import click import semantic_version -from platformio import __version__, app, exception, fs, telemetry, util +from platformio import __version__, app, exception, fs, telemetry +from platformio.cache import cleanup_content_cache +from platformio.clients import http from platformio.commands import PlatformioCLI -from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY -from platformio.commands.lib import lib_update as cmd_lib_update +from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY +from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.managers.core import update_core_packages -from platformio.managers.lib import LibraryManager -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.core import update_core_packages +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec +from platformio.package.version import pepver_to_semver +from platformio.platform.factory import PlatformFactory from platformio.proc import is_container @@ -50,9 +56,9 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument check_internal_updates(ctx, "platforms") check_internal_updates(ctx, "libraries") except ( - exception.InternetIsOffline, + http.HTTPClientError, + http.InternetIsOffline, exception.GetLatestVersionError, - exception.APIRequestError, ): click.secho( "Failed to check for PlatformIO upgrades. " @@ -66,10 +72,9 @@ def on_platformio_exception(e): def set_caller(caller=None): + caller = caller or getenv("PLATFORMIO_CALLER") if not caller: - if getenv("PLATFORMIO_CALLER"): - caller = getenv("PLATFORMIO_CALLER") - elif getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"): + if getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"): caller = "vscode" elif is_container(): if getenv("C9_UID"): @@ -83,15 +88,12 @@ def set_caller(caller=None): class Upgrader(object): def __init__(self, from_version, to_version): - self.from_version = semantic_version.Version.coerce( - util.pepver_to_semver(from_version) - ) - self.to_version = semantic_version.Version.coerce( - util.pepver_to_semver(to_version) - ) + self.from_version = pepver_to_semver(from_version) + self.to_version = pepver_to_semver(to_version) self._upgraders = [ - (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms) + (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms), + (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata), ] def run(self, ctx): @@ -111,6 +113,22 @@ def _update_dev_platforms(ctx): ctx.invoke(cmd_platform_update) return True + @staticmethod + def _update_pkg_metadata(_): + pm = ToolPackageManager() + for pkg in pm.get_installed(): + if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id: + continue + result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name)) + if len(result) != 1: + continue + result = result[0] + pkg.metadata.spec = PackageSpec( + id=result["id"], owner=result["owner"]["username"], name=result["name"], + ) + pkg.dump_meta() + return True + def after_upgrade(ctx): terminal_width, _ = click.get_terminal_size() @@ -120,9 +138,7 @@ def after_upgrade(ctx): if last_version == "0.0.0": app.set_state_item("last_version", __version__) - elif semantic_version.Version.coerce( - util.pepver_to_semver(last_version) - ) > semantic_version.Version.coerce(util.pepver_to_semver(__version__)): + elif pepver_to_semver(last_version) > pepver_to_semver(__version__): click.secho("*" * terminal_width, fg="yellow") click.secho( "Obsolete PIO Core v%s is used (previous was %s)" @@ -132,14 +148,17 @@ def after_upgrade(ctx): click.secho("Please remove multiple PIO Cores from a system:", fg="yellow") click.secho( "https://docs.platformio.org/page/faq.html" - "#multiple-pio-cores-in-a-system", + "#multiple-platformio-cores-in-a-system", fg="cyan", ) click.secho("*" * terminal_width, fg="yellow") return else: click.secho("Please wait while upgrading PlatformIO...", fg="yellow") - app.clean_cache() + try: + cleanup_content_cache("http") + except: # pylint: disable=bare-except + pass # Update PlatformIO's Core packages update_core_packages(silent=True) @@ -158,7 +177,6 @@ def after_upgrade(ctx): ) else: raise exception.UpgradeError("Auto upgrading...") - click.echo("") # PlatformIO banner click.echo("*" * terminal_width) @@ -200,15 +218,13 @@ def check_platformio_upgrade(): last_check["platformio_upgrade"] = int(time()) app.set_state_item("last_check", last_check) - util.internet_on(raise_exception=True) + http.ensure_internet_on(raise_exception=True) # Update PlatformIO's Core packages update_core_packages(silent=True) latest_version = get_latest_version() - if semantic_version.Version.coerce( - util.pepver_to_semver(latest_version) - ) <= semantic_version.Version.coerce(util.pepver_to_semver(__version__)): + if pepver_to_semver(latest_version) <= pepver_to_semver(__version__): return terminal_width, _ = click.get_terminal_size() @@ -238,7 +254,7 @@ def check_platformio_upgrade(): click.echo("") -def check_internal_updates(ctx, what): +def check_internal_updates(ctx, what): # pylint: disable=too-many-branches last_check = app.get_state_item("last_check", {}) interval = int(app.get_setting("check_%s_interval" % what)) * 3600 * 24 if (time() - interval) < last_check.get(what + "_update", 0): @@ -247,22 +263,19 @@ def check_internal_updates(ctx, what): last_check[what + "_update"] = int(time()) app.set_state_item("last_check", last_check) - util.internet_on(raise_exception=True) + http.ensure_internet_on(raise_exception=True) - pm = PlatformManager() if what == "platforms" else LibraryManager() outdated_items = [] - for manifest in pm.get_installed(): - if manifest["name"] in outdated_items: + pm = PlatformPackageManager() if what == "platforms" else LibraryPackageManager() + for pkg in pm.get_installed(): + if pkg.metadata.name in outdated_items: continue conds = [ - pm.outdated(manifest["__pkg_dir"]), - what == "platforms" - and PlatformFactory.newPlatform( - manifest["__pkg_dir"] - ).are_outdated_packages(), + pm.outdated(pkg).is_outdated(), + what == "platforms" and PlatformFactory.new(pkg).are_outdated_packages(), ] if any(conds): - outdated_items.append(manifest["name"]) + outdated_items.append(pkg.metadata.name) if not outdated_items: return diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py deleted file mode 100644 index 6e6b1b7dd0..0000000000 --- a/platformio/managers/lib.py +++ /dev/null @@ -1,374 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=too-many-arguments, too-many-locals, too-many-branches -# pylint: disable=too-many-return-statements - -import json -from glob import glob -from os.path import isdir, join - -import click -import semantic_version - -from platformio import app, exception, util -from platformio.compat import glob_escape -from platformio.managers.package import BasePkgManager -from platformio.managers.platform import PlatformFactory, PlatformManager -from platformio.package.exception import ManifestException -from platformio.package.manifest.parser import ManifestParserFactory -from platformio.project.config import ProjectConfig - - -class LibraryManager(BasePkgManager): - - FILE_CACHE_VALID = "30d" # 1 month - - def __init__(self, package_dir=None): - self.config = ProjectConfig.get_instance() - super(LibraryManager, self).__init__( - package_dir or self.config.get_optional_dir("globallib") - ) - - @property - def manifest_names(self): - return [".library.json", "library.json", "library.properties", "module.json"] - - def get_manifest_path(self, pkg_dir): - path = BasePkgManager.get_manifest_path(self, pkg_dir) - if path: - return path - - # if library without manifest, returns first source file - src_dir = join(glob_escape(pkg_dir)) - if isdir(join(pkg_dir, "src")): - src_dir = join(src_dir, "src") - chs_files = glob(join(src_dir, "*.[chS]")) - if chs_files: - return chs_files[0] - cpp_files = glob(join(src_dir, "*.cpp")) - if cpp_files: - return cpp_files[0] - - return None - - def max_satisfying_repo_version(self, versions, requirements=None): - def _cmp_dates(datestr1, datestr2): - date1 = util.parse_date(datestr1) - date2 = util.parse_date(datestr2) - if date1 == date2: - return 0 - return -1 if date1 < date2 else 1 - - semver_spec = None - try: - semver_spec = ( - semantic_version.SimpleSpec(requirements) if requirements else None - ) - except ValueError: - pass - - item = {} - - for v in versions: - semver_new = self.parse_semver_version(v["name"]) - if semver_spec: - # pylint: disable=unsupported-membership-test - if not semver_new or semver_new not in semver_spec: - continue - if not item or self.parse_semver_version(item["name"]) < semver_new: - item = v - elif requirements: - if requirements == v["name"]: - return v - - else: - if not item or _cmp_dates(item["released"], v["released"]) == -1: - item = v - return item - - def get_latest_repo_version(self, name, requirements, silent=False): - item = self.max_satisfying_repo_version( - util.get_api_result( - "/lib/info/%d" - % self.search_lib_id( - {"name": name, "requirements": requirements}, silent=silent - ), - cache_valid="1h", - )["versions"], - requirements, - ) - return item["name"] if item else None - - def _install_from_piorepo(self, name, requirements): - assert name.startswith("id="), name - version = self.get_latest_repo_version(name, requirements) - if not version: - raise exception.UndefinedPackageVersion( - requirements or "latest", util.get_systype() - ) - dl_data = util.get_api_result( - "/lib/download/" + str(name[3:]), dict(version=version), cache_valid="30d" - ) - assert dl_data - - return self._install_from_url( - name, - dl_data["url"].replace("http://", "https://") - if app.get_setting("strict_ssl") - else dl_data["url"], - requirements, - ) - - def search_lib_id( # pylint: disable=too-many-branches - self, filters, silent=False, interactive=False - ): - assert isinstance(filters, dict) - assert "name" in filters - - # try to find ID within installed packages - lib_id = self._get_lib_id_from_installed(filters) - if lib_id: - return lib_id - - # looking in PIO Library Registry - if not silent: - click.echo( - "Looking for %s library in registry" - % click.style(filters["name"], fg="cyan") - ) - query = [] - for key in filters: - if key not in ("name", "authors", "frameworks", "platforms"): - continue - values = filters[key] - if not isinstance(values, list): - values = [v.strip() for v in values.split(",") if v] - for value in values: - query.append( - '%s:"%s"' % (key[:-1] if key.endswith("s") else key, value) - ) - - lib_info = None - result = util.get_api_result( - "/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h" - ) - if result["total"] == 1: - lib_info = result["items"][0] - elif result["total"] > 1: - if silent and not interactive: - lib_info = result["items"][0] - else: - click.secho( - "Conflict: More than one library has been found " - "by request %s:" % json.dumps(filters), - fg="yellow", - err=True, - ) - # pylint: disable=import-outside-toplevel - from platformio.commands.lib import print_lib_item - - for item in result["items"]: - print_lib_item(item) - - if not interactive: - click.secho( - "Automatically chose the first available library " - "(use `--interactive` option to make a choice)", - fg="yellow", - err=True, - ) - lib_info = result["items"][0] - else: - deplib_id = click.prompt( - "Please choose library ID", - type=click.Choice([str(i["id"]) for i in result["items"]]), - ) - for item in result["items"]: - if item["id"] == int(deplib_id): - lib_info = item - break - - if not lib_info: - if list(filters) == ["name"]: - raise exception.LibNotFound(filters["name"]) - raise exception.LibNotFound(str(filters)) - if not silent: - click.echo( - "Found: %s" - % click.style( - "https://platformio.org/lib/show/{id}/{name}".format(**lib_info), - fg="blue", - ) - ) - return int(lib_info["id"]) - - def _get_lib_id_from_installed(self, filters): - if filters["name"].startswith("id="): - return int(filters["name"][3:]) - package_dir = self.get_package_dir( - filters["name"], filters.get("requirements", filters.get("version")) - ) - if not package_dir: - return None - manifest = self.load_manifest(package_dir) - if "id" not in manifest: - return None - - for key in ("frameworks", "platforms"): - if key not in filters: - continue - if key not in manifest: - return None - if not util.items_in_list( - util.items_to_list(filters[key]), util.items_to_list(manifest[key]) - ): - return None - - if "authors" in filters: - if "authors" not in manifest: - return None - manifest_authors = manifest["authors"] - if not isinstance(manifest_authors, list): - manifest_authors = [manifest_authors] - manifest_authors = [ - a["name"] - for a in manifest_authors - if isinstance(a, dict) and "name" in a - ] - filter_authors = filters["authors"] - if not isinstance(filter_authors, list): - filter_authors = [filter_authors] - if not set(filter_authors) <= set(manifest_authors): - return None - - return int(manifest["id"]) - - def install( # pylint: disable=arguments-differ - self, - name, - requirements=None, - silent=False, - after_update=False, - interactive=False, - force=False, - ): - _name, _requirements, _url = self.parse_pkg_uri(name, requirements) - if not _url: - name = "id=%d" % self.search_lib_id( - {"name": _name, "requirements": _requirements}, - silent=silent, - interactive=interactive, - ) - requirements = _requirements - pkg_dir = BasePkgManager.install( - self, - name, - requirements, - silent=silent, - after_update=after_update, - force=force, - ) - - if not pkg_dir: - return None - - manifest = None - try: - manifest = ManifestParserFactory.new_from_dir(pkg_dir).as_dict() - except ManifestException: - pass - if not manifest or not manifest.get("dependencies"): - return pkg_dir - - if not silent: - click.secho("Installing dependencies", fg="yellow") - - builtin_lib_storages = None - for filters in manifest["dependencies"]: - assert "name" in filters - - # avoid circle dependencies - if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = [] - history_key = str(filters) - if history_key in self.INSTALL_HISTORY: - continue - self.INSTALL_HISTORY.append(history_key) - - if any(s in filters.get("version", "") for s in ("\\", "/")): - self.install( - "{name}={version}".format(**filters), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force, - ) - else: - try: - lib_id = self.search_lib_id(filters, silent, interactive) - except exception.LibNotFound as e: - if builtin_lib_storages is None: - builtin_lib_storages = get_builtin_libs() - if not silent or is_builtin_lib( - builtin_lib_storages, filters["name"] - ): - click.secho("Warning! %s" % e, fg="yellow") - continue - - if filters.get("version"): - self.install( - lib_id, - filters.get("version"), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force, - ) - else: - self.install( - lib_id, - silent=silent, - after_update=after_update, - interactive=interactive, - force=force, - ) - return pkg_dir - - -def get_builtin_libs(storage_names=None): - items = [] - storage_names = storage_names or [] - pm = PlatformManager() - for manifest in pm.get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) - for storage in p.get_lib_storages(): - if storage_names and storage["name"] not in storage_names: - continue - lm = LibraryManager(storage["path"]) - items.append( - { - "name": storage["name"], - "path": storage["path"], - "items": lm.get_installed(), - } - ) - return items - - -def is_builtin_lib(storages, name): - for storage in storages or []: - if any(l.get("name") == name for l in storage["items"]): - return True - return False diff --git a/platformio/managers/package.py b/platformio/managers/package.py deleted file mode 100644 index 92ba451582..0000000000 --- a/platformio/managers/package.py +++ /dev/null @@ -1,818 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import hashlib -import json -import os -import re -import shutil -from os.path import basename, getsize, isdir, isfile, islink, join, realpath -from tempfile import mkdtemp - -import click -import requests -import semantic_version - -from platformio import __version__, app, exception, fs, util -from platformio.compat import hashlib_encode_data -from platformio.downloader import FileDownloader -from platformio.lockfile import LockFile -from platformio.package.exception import ManifestException -from platformio.package.manifest.parser import ManifestParserFactory -from platformio.unpacker import FileUnpacker -from platformio.vcsclient import VCSClientFactory - -# pylint: disable=too-many-arguments, too-many-return-statements - - -class PackageRepoIterator(object): - def __init__(self, package, repositories): - assert isinstance(repositories, list) - self.package = package - self.repositories = iter(repositories) - - def __iter__(self): - return self - - def __next__(self): - return self.next() # pylint: disable=not-callable - - @staticmethod - @util.memoized(expire="60s") - def load_manifest(url): - r = None - try: - r = requests.get(url, headers={"User-Agent": app.get_user_agent()}) - r.raise_for_status() - return r.json() - except: # pylint: disable=bare-except - pass - finally: - if r: - r.close() - return None - - def next(self): - repo = next(self.repositories) - manifest = repo if isinstance(repo, dict) else self.load_manifest(repo) - if manifest and self.package in manifest: - return manifest[self.package] - return next(self) - - -class PkgRepoMixin(object): - - PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) - - @staticmethod - def is_system_compatible(valid_systems): - if not valid_systems or "*" in valid_systems: - return True - if not isinstance(valid_systems, list): - valid_systems = list([valid_systems]) - return util.get_systype() in valid_systems - - def max_satisfying_repo_version(self, versions, requirements=None): - item = None - reqspec = None - try: - reqspec = ( - semantic_version.SimpleSpec(requirements) if requirements else None - ) - except ValueError: - pass - - for v in versions: - if not self.is_system_compatible(v.get("system")): - continue - # if "platformio" in v.get("engines", {}): - # if PkgRepoMixin.PIO_VERSION not in requirements.SimpleSpec( - # v['engines']['platformio']): - # continue - specver = semantic_version.Version(v["version"]) - if reqspec and specver not in reqspec: - continue - if not item or semantic_version.Version(item["version"]) < specver: - item = v - return item - - def get_latest_repo_version( # pylint: disable=unused-argument - self, name, requirements, silent=False - ): - version = None - for versions in PackageRepoIterator(name, self.repositories): - pkgdata = self.max_satisfying_repo_version(versions, requirements) - if not pkgdata: - continue - if ( - not version - or semantic_version.compare(pkgdata["version"], version) == 1 - ): - version = pkgdata["version"] - return version - - def get_all_repo_versions(self, name): - result = [] - for versions in PackageRepoIterator(name, self.repositories): - result.extend([semantic_version.Version(v["version"]) for v in versions]) - return [str(v) for v in sorted(set(result))] - - -class PkgInstallerMixin(object): - - SRC_MANIFEST_NAME = ".piopkgmanager.json" - TMP_FOLDER_PREFIX = "_tmp_installing-" - - FILE_CACHE_VALID = None # for example, 1 week = "7d" - FILE_CACHE_MAX_SIZE = 1024 * 1024 * 50 # 50 Mb - - MEMORY_CACHE = {} # cache for package manifests and read dirs - - def cache_get(self, key, default=None): - return self.MEMORY_CACHE.get(key, default) - - def cache_set(self, key, value): - self.MEMORY_CACHE[key] = value - - def cache_reset(self): - self.MEMORY_CACHE.clear() - - def read_dirs(self, src_dir): - cache_key = "read_dirs-%s" % src_dir - result = self.cache_get(cache_key) - if result: - return result - result = [ - join(src_dir, name) - for name in sorted(os.listdir(src_dir)) - if isdir(join(src_dir, name)) - ] - self.cache_set(cache_key, result) - return result - - def download(self, url, dest_dir, sha1=None): - cache_key_fname = app.ContentCache.key_from_args(url, "fname") - cache_key_data = app.ContentCache.key_from_args(url, "data") - if self.FILE_CACHE_VALID: - with app.ContentCache() as cc: - fname = str(cc.get(cache_key_fname)) - cache_path = cc.get_cache_path(cache_key_data) - if fname and isfile(cache_path): - dst_path = join(dest_dir, fname) - shutil.copy(cache_path, dst_path) - click.echo("Using cache: %s" % cache_path) - return dst_path - - with_progress = not app.is_disabled_progressbar() - try: - fd = FileDownloader(url, dest_dir) - fd.start(with_progress=with_progress) - except IOError as e: - raise_error = not with_progress - if with_progress: - try: - fd = FileDownloader(url, dest_dir) - fd.start(with_progress=False) - except IOError: - raise_error = True - if raise_error: - click.secho( - "Error: Please read http://bit.ly/package-manager-ioerror", - fg="red", - err=True, - ) - raise e - - if sha1: - fd.verify(sha1) - dst_path = fd.get_filepath() - if ( - not self.FILE_CACHE_VALID - or getsize(dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE - ): - return dst_path - - with app.ContentCache() as cc: - cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID) - cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID) - shutil.copy(dst_path, cc.get_cache_path(cache_key_data)) - return dst_path - - @staticmethod - def unpack(source_path, dest_dir): - with_progress = not app.is_disabled_progressbar() - try: - with FileUnpacker(source_path) as fu: - return fu.unpack(dest_dir, with_progress=with_progress) - except IOError as e: - if not with_progress: - raise e - with FileUnpacker(source_path) as fu: - return fu.unpack(dest_dir, with_progress=False) - - @staticmethod - def parse_semver_version(value, raise_exception=False): - try: - try: - return semantic_version.Version(value) - except ValueError: - if "." not in str(value) and not str(value).isdigit(): - raise ValueError("Invalid SemVer version %s" % value) - return semantic_version.Version.coerce(value) - except ValueError as e: - if raise_exception: - raise e - return None - - @staticmethod - def parse_pkg_uri(text, requirements=None): # pylint: disable=too-many-branches - text = str(text) - name, url = None, None - - # Parse requirements - req_conditions = [ - "@" in text, - not requirements, - ":" not in text or text.rfind("/") < text.rfind("@"), - ] - if all(req_conditions): - text, requirements = text.rsplit("@", 1) - - # Handle PIO Library Registry ID - if text.isdigit(): - text = "id=" + text - # Parse custom name - elif "=" in text and not text.startswith("id="): - name, text = text.split("=", 1) - - # Parse URL - # if valid URL with scheme vcs+protocol:// - if "+" in text and text.find("+") < text.find("://"): - url = text - elif "/" in text or "\\" in text: - git_conditions = [ - # Handle GitHub URL (https://github.com/user/package) - text.startswith("https://github.com/") - and not text.endswith((".zip", ".tar.gz")), - (text.split("#", 1)[0] if "#" in text else text).endswith(".git"), - ] - hg_conditions = [ - # Handle Developer Mbed URL - # (https://developer.mbed.org/users/user/code/package/) - # (https://os.mbed.com/users/user/code/package/) - text.startswith("https://developer.mbed.org"), - text.startswith("https://os.mbed.com"), - ] - if any(git_conditions): - url = "git+" + text - elif any(hg_conditions): - url = "hg+" + text - elif "://" not in text and (isfile(text) or isdir(text)): - url = "file://" + text - elif "://" in text: - url = text - # Handle short version of GitHub URL - elif text.count("/") == 1: - url = "git+https://github.com/" + text - - # Parse name from URL - if url and not name: - _url = url.split("#", 1)[0] if "#" in url else url - if _url.endswith(("\\", "/")): - _url = _url[:-1] - name = basename(_url) - if "." in name and not name.startswith("."): - name = name.rsplit(".", 1)[0] - - return (name or text, requirements, url) - - @staticmethod - def get_install_dirname(manifest): - name = re.sub(r"[^\da-z\_\-\. ]", "_", manifest["name"], flags=re.I) - if "id" in manifest: - name += "_ID%d" % manifest["id"] - return str(name) - - @classmethod - def get_src_manifest_path(cls, pkg_dir): - if not isdir(pkg_dir): - return None - for item in os.listdir(pkg_dir): - if not isdir(join(pkg_dir, item)): - continue - if isfile(join(pkg_dir, item, cls.SRC_MANIFEST_NAME)): - return join(pkg_dir, item, cls.SRC_MANIFEST_NAME) - return None - - def get_manifest_path(self, pkg_dir): - if not isdir(pkg_dir): - return None - for name in self.manifest_names: - manifest_path = join(pkg_dir, name) - if isfile(manifest_path): - return manifest_path - return None - - def manifest_exists(self, pkg_dir): - return self.get_manifest_path(pkg_dir) or self.get_src_manifest_path(pkg_dir) - - def load_manifest(self, pkg_dir): # pylint: disable=too-many-branches - cache_key = "load_manifest-%s" % pkg_dir - result = self.cache_get(cache_key) - if result: - return result - - manifest = {} - src_manifest = None - manifest_path = self.get_manifest_path(pkg_dir) - src_manifest_path = self.get_src_manifest_path(pkg_dir) - if src_manifest_path: - src_manifest = fs.load_json(src_manifest_path) - - if not manifest_path and not src_manifest_path: - return None - - try: - manifest = ManifestParserFactory.new_from_file(manifest_path).as_dict() - except ManifestException: - pass - - if src_manifest: - if "version" in src_manifest: - manifest["version"] = src_manifest["version"] - manifest["__src_url"] = src_manifest["url"] - # handle a custom package name - autogen_name = self.parse_pkg_uri(manifest["__src_url"])[0] - if "name" not in manifest or autogen_name != src_manifest["name"]: - manifest["name"] = src_manifest["name"] - - if "name" not in manifest: - manifest["name"] = basename(pkg_dir) - if "version" not in manifest: - manifest["version"] = "0.0.0" - - manifest["__pkg_dir"] = realpath(pkg_dir) - self.cache_set(cache_key, manifest) - return manifest - - def get_installed(self): - items = [] - for pkg_dir in self.read_dirs(self.package_dir): - if self.TMP_FOLDER_PREFIX in pkg_dir: - continue - manifest = self.load_manifest(pkg_dir) - if not manifest: - continue - assert "name" in manifest - items.append(manifest) - return items - - def get_package(self, name, requirements=None, url=None): - pkg_id = int(name[3:]) if name.startswith("id=") else 0 - best = None - for manifest in self.get_installed(): - if url: - if manifest.get("__src_url") != url: - continue - elif pkg_id and manifest.get("id") != pkg_id: - continue - elif not pkg_id and manifest["name"] != name: - continue - elif not PkgRepoMixin.is_system_compatible(manifest.get("system")): - continue - - # strict version or VCS HASH - if requirements and requirements == manifest["version"]: - return manifest - - try: - if requirements and not semantic_version.SimpleSpec(requirements).match( - self.parse_semver_version(manifest["version"], raise_exception=True) - ): - continue - if not best or ( - self.parse_semver_version(manifest["version"], raise_exception=True) - > self.parse_semver_version(best["version"], raise_exception=True) - ): - best = manifest - except ValueError: - pass - - return best - - def get_package_dir(self, name, requirements=None, url=None): - manifest = self.get_package(name, requirements, url) - return ( - manifest.get("__pkg_dir") - if manifest and isdir(manifest.get("__pkg_dir")) - else None - ) - - def get_package_by_dir(self, pkg_dir): - for manifest in self.get_installed(): - if manifest["__pkg_dir"] == realpath(pkg_dir): - return manifest - return None - - def find_pkg_root(self, src_dir): - if self.manifest_exists(src_dir): - return src_dir - for root, _, _ in os.walk(src_dir): - if self.manifest_exists(root): - return root - raise exception.MissingPackageManifest(", ".join(self.manifest_names)) - - def _install_from_piorepo(self, name, requirements): - pkg_dir = None - pkgdata = None - versions = None - last_exc = None - for versions in PackageRepoIterator(name, self.repositories): - pkgdata = self.max_satisfying_repo_version(versions, requirements) - if not pkgdata: - continue - try: - pkg_dir = self._install_from_url( - name, pkgdata["url"], requirements, pkgdata.get("sha1") - ) - break - except Exception as e: # pylint: disable=broad-except - last_exc = e - click.secho("Warning! Package Mirror: %s" % e, fg="yellow") - click.secho("Looking for another mirror...", fg="yellow") - - if versions is None: - util.internet_on(raise_exception=True) - raise exception.UnknownPackage( - name + (". Error -> %s" % last_exc if last_exc else "") - ) - if not pkgdata: - raise exception.UndefinedPackageVersion( - requirements or "latest", util.get_systype() - ) - return pkg_dir - - def _install_from_url(self, name, url, requirements=None, sha1=None, track=False): - tmp_dir = mkdtemp("-package", self.TMP_FOLDER_PREFIX, self.package_dir) - src_manifest_dir = None - src_manifest = {"name": name, "url": url, "requirements": requirements} - - try: - if url.startswith("file://"): - _url = url[7:] - if isfile(_url): - self.unpack(_url, tmp_dir) - else: - fs.rmtree(tmp_dir) - shutil.copytree(_url, tmp_dir, symlinks=True) - elif url.startswith(("http://", "https://")): - dlpath = self.download(url, tmp_dir, sha1) - assert isfile(dlpath) - self.unpack(dlpath, tmp_dir) - os.remove(dlpath) - else: - vcs = VCSClientFactory.newClient(tmp_dir, url) - assert vcs.export() - src_manifest_dir = vcs.storage_dir - src_manifest["version"] = vcs.get_current_revision() - - _tmp_dir = tmp_dir - if not src_manifest_dir: - _tmp_dir = self.find_pkg_root(tmp_dir) - src_manifest_dir = join(_tmp_dir, ".pio") - - # write source data to a special manifest - if track: - self._update_src_manifest(src_manifest, src_manifest_dir) - - return self._install_from_tmp_dir(_tmp_dir, requirements) - finally: - if isdir(tmp_dir): - fs.rmtree(tmp_dir) - return None - - def _update_src_manifest(self, data, src_dir): - if not isdir(src_dir): - os.makedirs(src_dir) - src_manifest_path = join(src_dir, self.SRC_MANIFEST_NAME) - _data = {} - if isfile(src_manifest_path): - _data = fs.load_json(src_manifest_path) - _data.update(data) - with open(src_manifest_path, "w") as fp: - json.dump(_data, fp) - - def _install_from_tmp_dir( # pylint: disable=too-many-branches - self, tmp_dir, requirements=None - ): - tmp_manifest = self.load_manifest(tmp_dir) - assert set(["name", "version"]) <= set(tmp_manifest) - - pkg_dirname = self.get_install_dirname(tmp_manifest) - pkg_dir = join(self.package_dir, pkg_dirname) - cur_manifest = self.load_manifest(pkg_dir) - - tmp_semver = self.parse_semver_version(tmp_manifest["version"]) - cur_semver = None - if cur_manifest: - cur_semver = self.parse_semver_version(cur_manifest["version"]) - - # package should satisfy requirements - if requirements: - mismatch_error = "Package version %s doesn't satisfy requirements %s" % ( - tmp_manifest["version"], - requirements, - ) - try: - assert tmp_semver and tmp_semver in semantic_version.SimpleSpec( - requirements - ), mismatch_error - except (AssertionError, ValueError): - assert tmp_manifest["version"] == requirements, mismatch_error - - # check if package already exists - if cur_manifest: - # 0-overwrite, 1-rename, 2-fix to a version - action = 0 - if "__src_url" in cur_manifest: - if cur_manifest["__src_url"] != tmp_manifest.get("__src_url"): - action = 1 - elif "__src_url" in tmp_manifest: - action = 2 - else: - if tmp_semver and (not cur_semver or tmp_semver > cur_semver): - action = 1 - elif tmp_semver and cur_semver and tmp_semver != cur_semver: - action = 2 - - # rename - if action == 1: - target_dirname = "%s@%s" % (pkg_dirname, cur_manifest["version"]) - if "__src_url" in cur_manifest: - target_dirname = "%s@src-%s" % ( - pkg_dirname, - hashlib.md5( - hashlib_encode_data(cur_manifest["__src_url"]) - ).hexdigest(), - ) - shutil.move(pkg_dir, join(self.package_dir, target_dirname)) - # fix to a version - elif action == 2: - target_dirname = "%s@%s" % (pkg_dirname, tmp_manifest["version"]) - if "__src_url" in tmp_manifest: - target_dirname = "%s@src-%s" % ( - pkg_dirname, - hashlib.md5( - hashlib_encode_data(tmp_manifest["__src_url"]) - ).hexdigest(), - ) - pkg_dir = join(self.package_dir, target_dirname) - - # remove previous/not-satisfied package - if isdir(pkg_dir): - fs.rmtree(pkg_dir) - shutil.copytree(tmp_dir, pkg_dir, symlinks=True) - try: - shutil.rmtree(tmp_dir) - except: # pylint: disable=bare-except - pass - assert isdir(pkg_dir) - self.cache_reset() - return pkg_dir - - -class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): - - # Handle circle dependencies - INSTALL_HISTORY = None - - def __init__(self, package_dir, repositories=None): - self.repositories = repositories - self.package_dir = package_dir - if not isdir(self.package_dir): - os.makedirs(self.package_dir) - assert isdir(self.package_dir) - - @property - def manifest_names(self): - raise NotImplementedError() - - def print_message(self, message, nl=True): - click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl) - - def outdated(self, pkg_dir, requirements=None): - """ - Has 3 different results: - `None` - unknown package, VCS is detached to commit - `False` - package is up-to-date - `String` - a found latest version - """ - if not isdir(pkg_dir): - return None - latest = None - manifest = self.load_manifest(pkg_dir) - # skip detached package to a specific version - if "@" in pkg_dir and "__src_url" not in manifest and not requirements: - return None - - if "__src_url" in manifest: - try: - vcs = VCSClientFactory.newClient( - pkg_dir, manifest["__src_url"], silent=True - ) - except (AttributeError, exception.PlatformioException): - return None - if not vcs.can_be_updated: - return None - latest = vcs.get_latest_revision() - else: - try: - latest = self.get_latest_repo_version( - "id=%d" % manifest["id"] if "id" in manifest else manifest["name"], - requirements, - silent=True, - ) - except (exception.PlatformioException, ValueError): - return None - - if not latest: - return None - - up_to_date = False - try: - assert "__src_url" not in manifest - up_to_date = self.parse_semver_version( - manifest["version"], raise_exception=True - ) >= self.parse_semver_version(latest, raise_exception=True) - except (AssertionError, ValueError): - up_to_date = latest == manifest["version"] - - return False if up_to_date else latest - - def install( - self, name, requirements=None, silent=False, after_update=False, force=False - ): # pylint: disable=unused-argument - pkg_dir = None - # interprocess lock - with LockFile(self.package_dir): - self.cache_reset() - - name, requirements, url = self.parse_pkg_uri(name, requirements) - package_dir = self.get_package_dir(name, requirements, url) - - # avoid circle dependencies - if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = [] - history_key = "%s-%s-%s" % (name, requirements or "", url or "") - if history_key in self.INSTALL_HISTORY: - return package_dir - self.INSTALL_HISTORY.append(history_key) - - if package_dir and force: - self.uninstall(package_dir) - package_dir = None - - if not package_dir or not silent: - msg = "Installing " + click.style(name, fg="cyan") - if requirements: - msg += " @ " + requirements - self.print_message(msg) - if package_dir: - if not silent: - click.secho( - "{name} @ {version} is already installed".format( - **self.load_manifest(package_dir) - ), - fg="yellow", - ) - return package_dir - - if url: - pkg_dir = self._install_from_url(name, url, requirements, track=True) - else: - pkg_dir = self._install_from_piorepo(name, requirements) - - if not pkg_dir or not self.manifest_exists(pkg_dir): - raise exception.PackageInstallError( - name, requirements or "*", util.get_systype() - ) - - manifest = self.load_manifest(pkg_dir) - assert manifest - - click.secho( - "{name} @ {version} has been successfully installed!".format( - **manifest - ), - fg="green", - ) - - return pkg_dir - - def uninstall( - self, package, requirements=None, after_update=False - ): # pylint: disable=unused-argument - # interprocess lock - with LockFile(self.package_dir): - self.cache_reset() - - if isdir(package) and self.get_package_by_dir(package): - pkg_dir = package - else: - name, requirements, url = self.parse_pkg_uri(package, requirements) - pkg_dir = self.get_package_dir(name, requirements, url) - - if not pkg_dir: - raise exception.UnknownPackage( - "%s @ %s" % (package, requirements or "*") - ) - - manifest = self.load_manifest(pkg_dir) - click.echo( - "Uninstalling %s @ %s: \t" - % (click.style(manifest["name"], fg="cyan"), manifest["version"]), - nl=False, - ) - - if islink(pkg_dir): - os.unlink(pkg_dir) - else: - fs.rmtree(pkg_dir) - self.cache_reset() - - # unfix package with the same name - pkg_dir = self.get_package_dir(manifest["name"]) - if pkg_dir and "@" in pkg_dir: - shutil.move( - pkg_dir, join(self.package_dir, self.get_install_dirname(manifest)) - ) - self.cache_reset() - - click.echo("[%s]" % click.style("OK", fg="green")) - - return True - - def update(self, package, requirements=None, only_check=False): - self.cache_reset() - if isdir(package) and self.get_package_by_dir(package): - pkg_dir = package - else: - pkg_dir = self.get_package_dir(*self.parse_pkg_uri(package)) - - if not pkg_dir: - raise exception.UnknownPackage("%s @ %s" % (package, requirements or "*")) - - manifest = self.load_manifest(pkg_dir) - name = manifest["name"] - - click.echo( - "{} {:<40} @ {:<15}".format( - "Checking" if only_check else "Updating", - click.style(manifest["name"], fg="cyan"), - manifest["version"], - ), - nl=False, - ) - if not util.internet_on(): - click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) - return None - - latest = self.outdated(pkg_dir, requirements) - if latest: - click.echo("[%s]" % (click.style(latest, fg="red"))) - elif latest is False: - click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) - else: - click.echo("[%s]" % (click.style("Detached", fg="yellow"))) - - if only_check or not latest: - return True - - if "__src_url" in manifest: - vcs = VCSClientFactory.newClient(pkg_dir, manifest["__src_url"]) - assert vcs.update() - self._update_src_manifest( - dict(version=vcs.get_current_revision()), vcs.storage_dir - ) - else: - self.uninstall(pkg_dir, after_update=True) - self.install(name, latest, after_update=True) - - return True - - -class PackageManager(BasePkgManager): - @property - def manifest_names(self): - return ["package.json"] diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index c667a956cb..9ef5e646c6 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -12,921 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-public-methods, too-many-instance-attributes - -import base64 -import os -import re -import subprocess -import sys -from os.path import basename, dirname, isdir, isfile, join - -import click -import semantic_version - -from platformio import __version__, app, exception, fs, proc, telemetry, util -from platformio.commands.debug.exception import ( - DebugInvalidOptionsError, - DebugSupportError, -) -from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module -from platformio.managers.core import get_core_package_dir -from platformio.managers.package import BasePkgManager, PackageManager -from platformio.project.config import ProjectConfig - -try: - from urllib.parse import quote -except ImportError: - from urllib import quote - - -class PlatformManager(BasePkgManager): - def __init__(self, package_dir=None, repositories=None): - if not repositories: - repositories = [ - "https://dl.bintray.com/platformio/dl-platforms/manifest.json", - "{0}://dl.platformio.org/platforms/manifest.json".format( - "https" if app.get_setting("strict_ssl") else "http" - ), - ] - self.config = ProjectConfig.get_instance() - BasePkgManager.__init__( - self, package_dir or self.config.get_optional_dir("platforms"), repositories - ) - - @property - def manifest_names(self): - return ["platform.json"] - - def get_manifest_path(self, pkg_dir): - if not isdir(pkg_dir): - return None - for name in self.manifest_names: - manifest_path = join(pkg_dir, name) - if isfile(manifest_path): - return manifest_path - return None - - def install( - self, - name, - requirements=None, - with_packages=None, - without_packages=None, - skip_default_package=False, - with_all_packages=False, - after_update=False, - silent=False, - force=False, - **_ - ): # pylint: disable=too-many-arguments, arguments-differ - platform_dir = BasePkgManager.install( - self, name, requirements, silent=silent, force=force - ) - p = PlatformFactory.newPlatform(platform_dir) - - if with_all_packages: - with_packages = list(p.packages.keys()) - - # don't cleanup packages or install them after update - # we check packages for updates in def update() - if after_update: - p.install_python_packages() - p.on_installed() - return True - - p.install_packages( - with_packages, - without_packages, - skip_default_package, - silent=silent, - force=force, - ) - p.install_python_packages() - p.on_installed() - return self.cleanup_packages(list(p.packages)) - - def uninstall(self, package, requirements=None, after_update=False): - if isdir(package): - pkg_dir = package - else: - name, requirements, url = self.parse_pkg_uri(package, requirements) - pkg_dir = self.get_package_dir(name, requirements, url) - - if not pkg_dir: - raise exception.UnknownPlatform(package) - - p = PlatformFactory.newPlatform(pkg_dir) - BasePkgManager.uninstall(self, pkg_dir, requirements) - p.uninstall_python_packages() - p.on_uninstalled() - - # don't cleanup packages or install them after update - # we check packages for updates in def update() - if after_update: - return True - - return self.cleanup_packages(list(p.packages)) - - def update( # pylint: disable=arguments-differ - self, package, requirements=None, only_check=False, only_packages=False - ): - if isdir(package): - pkg_dir = package - else: - name, requirements, url = self.parse_pkg_uri(package, requirements) - pkg_dir = self.get_package_dir(name, requirements, url) - - if not pkg_dir: - raise exception.UnknownPlatform(package) - - p = PlatformFactory.newPlatform(pkg_dir) - pkgs_before = list(p.get_installed_packages()) - - missed_pkgs = set() - if not only_packages: - BasePkgManager.update(self, pkg_dir, requirements, only_check) - p = PlatformFactory.newPlatform(pkg_dir) - missed_pkgs = set(pkgs_before) & set(p.packages) - missed_pkgs -= set(p.get_installed_packages()) - - p.update_packages(only_check) - self.cleanup_packages(list(p.packages)) - - if missed_pkgs: - p.install_packages( - with_packages=list(missed_pkgs), skip_default_package=True - ) - - return True - - def cleanup_packages(self, names): - self.cache_reset() - deppkgs = {} - for manifest in PlatformManager().get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) - for pkgname, pkgmanifest in p.get_installed_packages().items(): - if pkgname not in deppkgs: - deppkgs[pkgname] = set() - deppkgs[pkgname].add(pkgmanifest["version"]) - - pm = PackageManager(self.config.get_optional_dir("packages")) - for manifest in pm.get_installed(): - if manifest["name"] not in names: - continue - if ( - manifest["name"] not in deppkgs - or manifest["version"] not in deppkgs[manifest["name"]] - ): - try: - pm.uninstall(manifest["__pkg_dir"], after_update=True) - except exception.UnknownPackage: - pass - - self.cache_reset() - return True - - @util.memoized(expire="5s") - def get_installed_boards(self): - boards = [] - for manifest in self.get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) - for config in p.get_boards().values(): - board = config.get_brief_data() - if board not in boards: - boards.append(board) - return boards - - @staticmethod - def get_registered_boards(): - return util.get_api_result("/boards", cache_valid="7d") - - def get_all_boards(self): - boards = self.get_installed_boards() - know_boards = ["%s:%s" % (b["platform"], b["id"]) for b in boards] - try: - for board in self.get_registered_boards(): - key = "%s:%s" % (board["platform"], board["id"]) - if key not in know_boards: - boards.append(board) - except (exception.APIRequestError, exception.InternetIsOffline): - pass - return sorted(boards, key=lambda b: b["name"]) - - def board_config(self, id_, platform=None): - for manifest in self.get_installed_boards(): - if manifest["id"] == id_ and ( - not platform or manifest["platform"] == platform - ): - return manifest - for manifest in self.get_registered_boards(): - if manifest["id"] == id_ and ( - not platform or manifest["platform"] == platform - ): - return manifest - raise exception.UnknownBoard(id_) - - -class PlatformFactory(object): - @staticmethod - def get_clsname(name): - name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) - return "%s%sPlatform" % (name.upper()[0], name.lower()[1:]) - - @staticmethod - def load_module(name, path): - try: - return load_python_module("platformio.managers.platform.%s" % name, path) - except ImportError: - raise exception.UnknownPlatform(name) - - @classmethod - def newPlatform(cls, name, requirements=None): - pm = PlatformManager() - platform_dir = None - if isdir(name): - platform_dir = name - name = pm.load_manifest(platform_dir)["name"] - elif name.endswith("platform.json") and isfile(name): - platform_dir = dirname(name) - name = fs.load_json(name)["name"] - else: - name, requirements, url = pm.parse_pkg_uri(name, requirements) - platform_dir = pm.get_package_dir(name, requirements, url) - if platform_dir: - name = pm.load_manifest(platform_dir)["name"] - - if not platform_dir: - raise exception.UnknownPlatform( - name if not requirements else "%s@%s" % (name, requirements) - ) - - platform_cls = None - if isfile(join(platform_dir, "platform.py")): - platform_cls = getattr( - cls.load_module(name, join(platform_dir, "platform.py")), - cls.get_clsname(name), - ) - else: - platform_cls = type(str(cls.get_clsname(name)), (PlatformBase,), {}) - - _instance = platform_cls(join(platform_dir, "platform.json")) - assert isinstance(_instance, PlatformBase) - return _instance - - -class PlatformPackagesMixin(object): - def install_packages( # pylint: disable=too-many-arguments - self, - with_packages=None, - without_packages=None, - skip_default_package=False, - silent=False, - force=False, - ): - with_packages = set(self.find_pkg_names(with_packages or [])) - without_packages = set(self.find_pkg_names(without_packages or [])) - - upkgs = with_packages | without_packages - ppkgs = set(self.packages) - if not upkgs.issubset(ppkgs): - raise exception.UnknownPackage(", ".join(upkgs - ppkgs)) - - for name, opts in self.packages.items(): - version = opts.get("version", "") - if name in without_packages: - continue - if name in with_packages or not ( - skip_default_package or opts.get("optional", False) - ): - if ":" in version: - self.pm.install( - "%s=%s" % (name, version), silent=silent, force=force - ) - else: - self.pm.install(name, version, silent=silent, force=force) - - return True - - def find_pkg_names(self, candidates): - result = [] - for candidate in candidates: - found = False - - # lookup by package types - for _name, _opts in self.packages.items(): - if _opts.get("type") == candidate: - result.append(_name) - found = True - - if ( - self.frameworks - and candidate.startswith("framework-") - and candidate[10:] in self.frameworks - ): - result.append(self.frameworks[candidate[10:]]["package"]) - found = True - - if not found: - result.append(candidate) - - return result - - def update_packages(self, only_check=False): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - self.pm.update(manifest["__pkg_dir"], requirements, only_check) - - def get_installed_packages(self): - items = {} - for name in self.packages: - pkg_dir = self.get_package_dir(name) - if pkg_dir: - items[name] = self.pm.load_manifest(pkg_dir) - return items - - def are_outdated_packages(self): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - if self.pm.outdated(manifest["__pkg_dir"], requirements): - return True - return False - - def get_package_dir(self, name): - version = self.packages[name].get("version", "") - if ":" in version: - return self.pm.get_package_dir( - *self.pm.parse_pkg_uri("%s=%s" % (name, version)) - ) - return self.pm.get_package_dir(name, version) - - def get_package_version(self, name): - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - return None - return self.pm.load_manifest(pkg_dir).get("version") - - def dump_used_packages(self): - result = [] - for name, options in self.packages.items(): - if options.get("optional"): - continue - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - continue - manifest = self.pm.load_manifest(pkg_dir) - item = {"name": manifest["name"], "version": manifest["version"]} - if manifest.get("__src_url"): - item["src_url"] = manifest.get("__src_url") - result.append(item) - return result - - -class PlatformRunMixin(object): - - LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) - - @staticmethod - def encode_scons_arg(value): - data = base64.urlsafe_b64encode(hashlib_encode_data(value)) - return data.decode() if is_bytes(data) else data - - @staticmethod - def decode_scons_arg(data): - value = base64.urlsafe_b64decode(data) - return value.decode() if is_bytes(value) else value - - def run( # pylint: disable=too-many-arguments - self, variables, targets, silent, verbose, jobs - ): - assert isinstance(variables, dict) - assert isinstance(targets, list) - - options = self.config.items(env=variables["pioenv"], as_dict=True) - if "framework" in options: - # support PIO Core 3.0 dev/platforms - options["pioframework"] = options["framework"] - self.configure_default_packages(options, targets) - self.install_packages(silent=True) - - self._report_non_sensitive_data(options, targets) - - self.silent = silent - self.verbose = verbose or app.get_setting("force_verbose") - - if "clean" in targets: - targets = ["-c", "."] - - variables["platform_manifest"] = self.manifest_path - - if "build_script" not in variables: - variables["build_script"] = self.get_build_script() - if not isfile(variables["build_script"]): - raise exception.BuildScriptNotFound(variables["build_script"]) - - result = self._run_scons(variables, targets, jobs) - assert "returncode" in result - - return result - - def _report_non_sensitive_data(self, options, targets): - topts = options.copy() - topts["platform_packages"] = [ - dict(name=item["name"], version=item["version"]) - for item in self.dump_used_packages() - ] - topts["platform"] = {"name": self.name, "version": self.version} - if self.src_version: - topts["platform"]["src_version"] = self.src_version - telemetry.send_run_environment(topts, targets) - - def _run_scons(self, variables, targets, jobs): - args = [ - proc.get_pythonexe_path(), - join(get_core_package_dir("tool-scons"), "script", "scons"), - "-Q", - "--warn=no-no-parallel-support", - "--jobs", - str(jobs), - "--sconstruct", - join(fs.get_source_dir(), "builder", "main.py"), - ] - args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) - # pylint: disable=protected-access - args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) - args += targets - - # encode and append variables - for key, value in variables.items(): - args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - - proc.copy_pythonpath_to_osenv() - - if targets and "menuconfig" in targets: - return proc.exec_command( - args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin - ) - - if click._compat.isatty(sys.stdout): - - def _write_and_flush(stream, data): - try: - stream.write(data) - stream.flush() - except IOError: - pass - - return proc.exec_command( - args, - stdout=proc.BuildAsyncPipe( - line_callback=self._on_stdout_line, - data_callback=lambda data: _write_and_flush(sys.stdout, data), - ), - stderr=proc.BuildAsyncPipe( - line_callback=self._on_stderr_line, - data_callback=lambda data: _write_and_flush(sys.stderr, data), - ), - ) - - return proc.exec_command( - args, - stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), - stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), - ) - - def _on_stdout_line(self, line): - if "`buildprog' is up to date." in line: - return - self._echo_line(line, level=1) - - def _on_stderr_line(self, line): - is_error = self.LINE_ERROR_RE.search(line) is not None - self._echo_line(line, level=3 if is_error else 2) - - a_pos = line.find("fatal error:") - b_pos = line.rfind(": No such file or directory") - if a_pos == -1 or b_pos == -1: - return - self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) - - def _echo_line(self, line, level): - if line.startswith("scons: "): - line = line[7:] - assert 1 <= level <= 3 - if self.silent and (level < 2 or not line): - return - fg = (None, "yellow", "red")[level - 1] - if level == 1 and "is up to date" in line: - fg = "green" - click.secho(line, fg=fg, err=level > 1, nl=False) - - @staticmethod - def _echo_missed_dependency(filename): - if "/" in filename or not filename.endswith((".h", ".hpp")): - return - banner = """ -{dots} -* Looking for {filename_styled} dependency? Check our library registry! -* -* CLI > platformio lib search "header:{filename}" -* Web > {link} -* -{dots} -""".format( - filename=filename, - filename_styled=click.style(filename, fg="cyan"), - link=click.style( - "https://platformio.org/lib/search?query=header:%s" - % quote(filename, safe=""), - fg="blue", - ), - dots="*" * (56 + len(filename)), - ) - click.echo(banner, err=True) - - -class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): - - PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) - _BOARDS_CACHE = {} - - def __init__(self, manifest_path): - self.manifest_path = manifest_path - self.silent = False - self.verbose = False - - self._manifest = fs.load_json(manifest_path) - self._BOARDS_CACHE = {} - self._custom_packages = None - - self.config = ProjectConfig.get_instance() - self.pm = PackageManager( - self.config.get_optional_dir("packages"), self.package_repositories - ) - - self._src_manifest = None - src_manifest_path = self.pm.get_src_manifest_path(self.get_dir()) - if src_manifest_path: - self._src_manifest = fs.load_json(src_manifest_path) - - # if self.engines and "platformio" in self.engines: - # if self.PIO_VERSION not in semantic_version.SimpleSpec( - # self.engines['platformio']): - # raise exception.IncompatiblePlatform(self.name, - # str(self.PIO_VERSION)) - - @property - def name(self): - return self._manifest["name"] - - @property - def title(self): - return self._manifest["title"] - - @property - def description(self): - return self._manifest["description"] - - @property - def version(self): - return self._manifest["version"] - - @property - def src_version(self): - return self._src_manifest.get("version") if self._src_manifest else None - - @property - def src_url(self): - return self._src_manifest.get("url") if self._src_manifest else None - - @property - def homepage(self): - return self._manifest.get("homepage") - - @property - def vendor_url(self): - return self._manifest.get("url") - - @property - def docs_url(self): - return self._manifest.get("docs") - - @property - def repository_url(self): - return self._manifest.get("repository", {}).get("url") - - @property - def license(self): - return self._manifest.get("license") - - @property - def frameworks(self): - return self._manifest.get("frameworks") - - @property - def engines(self): - return self._manifest.get("engines") - - @property - def package_repositories(self): - return self._manifest.get("packageRepositories") - - @property - def manifest(self): - return self._manifest - - @property - def packages(self): - packages = self._manifest.get("packages", {}) - for item in self._custom_packages or []: - name = item - version = "*" - if "@" in item: - name, version = item.split("@", 2) - name = name.strip() - if name not in packages: - packages[name] = {} - packages[name].update({"version": version.strip(), "optional": False}) - return packages - - @property - def python_packages(self): - return self._manifest.get("pythonPackages") - - def get_dir(self): - return dirname(self.manifest_path) - - def get_build_script(self): - main_script = join(self.get_dir(), "builder", "main.py") - if isfile(main_script): - return main_script - raise NotImplementedError() - - def is_embedded(self): - for opts in self.packages.values(): - if opts.get("type") == "uploader": - return True - return False - - def get_boards(self, id_=None): - def _append_board(board_id, manifest_path): - config = PlatformBoardConfig(manifest_path) - if "platform" in config and config.get("platform") != self.name: - return - if "platforms" in config and self.name not in config.get("platforms"): - return - config.manifest["platform"] = self.name - self._BOARDS_CACHE[board_id] = config - - bdirs = [ - self.config.get_optional_dir("boards"), - join(self.config.get_optional_dir("core"), "boards"), - join(self.get_dir(), "boards"), - ] - - if id_ is None: - for boards_dir in bdirs: - if not isdir(boards_dir): - continue - for item in sorted(os.listdir(boards_dir)): - _id = item[:-5] - if not item.endswith(".json") or _id in self._BOARDS_CACHE: - continue - _append_board(_id, join(boards_dir, item)) - else: - if id_ not in self._BOARDS_CACHE: - for boards_dir in bdirs: - if not isdir(boards_dir): - continue - manifest_path = join(boards_dir, "%s.json" % id_) - if isfile(manifest_path): - _append_board(id_, manifest_path) - break - if id_ not in self._BOARDS_CACHE: - raise exception.UnknownBoard(id_) - return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE - - def board_config(self, id_): - return self.get_boards(id_) - - def get_package_type(self, name): - return self.packages[name].get("type") - - def configure_default_packages(self, options, targets): - # override user custom packages - self._custom_packages = options.get("platform_packages") - - # enable used frameworks - for framework in options.get("framework", []): - if not self.frameworks: - continue - framework = framework.lower().strip() - if not framework or framework not in self.frameworks: - continue - _pkg_name = self.frameworks[framework].get("package") - if _pkg_name: - self.packages[_pkg_name]["optional"] = False - - # enable upload tools for upload targets - if any(["upload" in t for t in targets] + ["program" in targets]): - for name, opts in self.packages.items(): - if opts.get("type") == "uploader": - self.packages[name]["optional"] = False - # skip all packages in "nobuild" mode - # allow only upload tools and frameworks - elif "nobuild" in targets and opts.get("type") != "framework": - self.packages[name]["optional"] = True - - def get_lib_storages(self): - storages = {} - for opts in (self.frameworks or {}).values(): - if "package" not in opts: - continue - pkg_dir = self.get_package_dir(opts["package"]) - if not pkg_dir or not isdir(join(pkg_dir, "libraries")): - continue - libs_dir = join(pkg_dir, "libraries") - storages[libs_dir] = opts["package"] - libcores_dir = join(libs_dir, "__cores__") - if not isdir(libcores_dir): - continue - for item in os.listdir(libcores_dir): - libcore_dir = join(libcores_dir, item) - if not isdir(libcore_dir): - continue - storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) - - return [dict(name=name, path=path) for path, name in storages.items()] - - def on_installed(self): - pass - - def on_uninstalled(self): - pass - - def install_python_packages(self): - if not self.python_packages: - return None - click.echo( - "Installing Python packages: %s" - % ", ".join(list(self.python_packages.keys())), - ) - args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"] - for name, requirements in self.python_packages.items(): - if any(c in requirements for c in ("<", ">", "=")): - args.append("%s%s" % (name, requirements)) - else: - args.append("%s==%s" % (name, requirements)) - try: - return subprocess.call(args) == 0 - except Exception as e: # pylint: disable=broad-except - click.secho( - "Could not install Python packages -> %s" % e, fg="red", err=True - ) - - def uninstall_python_packages(self): - if not self.python_packages: - return - click.echo("Uninstalling Python packages") - args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"] - args.extend(list(self.python_packages.keys())) - try: - subprocess.call(args) == 0 - except Exception as e: # pylint: disable=broad-except - click.secho( - "Could not install Python packages -> %s" % e, fg="red", err=True - ) - - -class PlatformBoardConfig(object): - def __init__(self, manifest_path): - self._id = basename(manifest_path)[:-5] - assert isfile(manifest_path) - self.manifest_path = manifest_path - try: - self._manifest = fs.load_json(manifest_path) - except ValueError: - raise exception.InvalidBoardManifest(manifest_path) - if not set(["name", "url", "vendor"]) <= set(self._manifest): - raise exception.PlatformioException( - "Please specify name, url and vendor fields for " + manifest_path - ) - - def get(self, path, default=None): - try: - value = self._manifest - for k in path.split("."): - value = value[k] - # pylint: disable=undefined-variable - if PY2 and isinstance(value, unicode): - # cast to plain string from unicode for PY2, resolves issue in - # dev/platform when BoardConfig.get() is used in pair with - # os.path.join(file_encoding, unicode_encoding) - try: - value = value.encode("utf-8") - except UnicodeEncodeError: - pass - return value - except KeyError: - if default is not None: - return default - raise KeyError("Invalid board option '%s'" % path) - - def update(self, path, value): - newdict = None - for key in path.split(".")[::-1]: - if newdict is None: - newdict = {key: value} - else: - newdict = {key: newdict} - util.merge_dicts(self._manifest, newdict) - - def __contains__(self, key): - try: - self.get(key) - return True - except KeyError: - return False - - @property - def id(self): - return self._id - - @property - def id_(self): - return self.id - - @property - def manifest(self): - return self._manifest - - def get_brief_data(self): - return { - "id": self.id, - "name": self._manifest["name"], - "platform": self._manifest.get("platform"), - "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), - "fcpu": int( - "".join( - [ - c - for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) - if c.isdigit() - ] - ) - ), - "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), - "rom": self._manifest.get("upload", {}).get("maximum_size", 0), - "connectivity": self._manifest.get("connectivity"), - "frameworks": self._manifest.get("frameworks"), - "debug": self.get_debug_data(), - "vendor": self._manifest["vendor"], - "url": self._manifest["url"], - } - - def get_debug_data(self): - if not self._manifest.get("debug", {}).get("tools"): - return None - tools = {} - for name, options in self._manifest["debug"]["tools"].items(): - tools[name] = {} - for key, value in options.items(): - if key in ("default", "onboard"): - tools[name][key] = value - return {"tools": tools} - - def get_debug_tool_name(self, custom=None): - debug_tools = self._manifest.get("debug", {}).get("tools") - tool_name = custom - if tool_name == "custom": - return tool_name - if not debug_tools: - telemetry.send_event("Debug", "Request", self.id) - raise DebugSupportError(self._manifest["name"]) - if tool_name: - if tool_name in debug_tools: - return tool_name - raise DebugInvalidOptionsError( - "Unknown debug tool `%s`. Please use one of `%s` or `custom`" - % (tool_name, ", ".join(sorted(list(debug_tools)))) - ) - - # automatically select best tool - data = {"default": [], "onboard": [], "external": []} - for key, value in debug_tools.items(): - if value.get("default"): - data["default"].append(key) - elif value.get("onboard"): - data["onboard"].append(key) - data["external"].append(key) - - for key, value in data.items(): - if not value: - continue - return sorted(value)[0] - - assert any(item for item in data) +# Backward compatibility with legacy dev-platforms +from platformio.platform.base import PlatformBase # pylint: disable=unused-import diff --git a/platformio/downloader.py b/platformio/package/download.py similarity index 64% rename from platformio/downloader.py rename to platformio/package/download.py index 21f5477bd8..bd425ac630 100644 --- a/platformio/downloader.py +++ b/platformio/package/download.py @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib import io import math -import sys from email.utils import parsedate_tz from os.path import getsize, join from time import mktime @@ -23,12 +21,8 @@ import click import requests -from platformio import app, util -from platformio.exception import ( - FDSHASumMismatch, - FDSizeMismatch, - FDUnrecognizedStatusCode, -) +from platformio import __default_requests_timeout__, app, fs +from platformio.package.exception import PackageException class FileDownloader(object): @@ -39,10 +33,14 @@ def __init__(self, url, dest_dir=None): url, stream=True, headers={"User-Agent": app.get_user_agent()}, - verify=sys.version_info >= (2, 7, 9), + timeout=__default_requests_timeout__, ) if self._request.status_code != 200: - raise FDUnrecognizedStatusCode(self._request.status_code, url) + raise PackageException( + "Got the unrecognized status code '{0}' when downloaded {1}".format( + self._request.status_code, url + ) + ) disposition = self._request.headers.get("content-disposition") if disposition and "filename=" in disposition: @@ -75,21 +73,21 @@ def get_size(self): def start(self, with_progress=True, silent=False): label = "Downloading" itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) - f = open(self._destination, "wb") + fp = open(self._destination, "wb") try: if not with_progress or self.get_size() == -1: if not silent: click.echo("%s..." % label) for chunk in itercontent: if chunk: - f.write(chunk) + fp.write(chunk) else: chunks = int(math.ceil(self.get_size() / float(io.DEFAULT_BUFFER_SIZE))) with click.progressbar(length=chunks, label=label) as pb: for _ in pb: - f.write(next(itercontent)) + fp.write(next(itercontent)) finally: - f.close() + fp.close() self._request.close() if self.get_lmtime(): @@ -97,29 +95,46 @@ def start(self, with_progress=True, silent=False): return True - def verify(self, sha1=None): + def verify(self, checksum=None): _dlsize = getsize(self._destination) if self.get_size() != -1 and _dlsize != self.get_size(): - raise FDSizeMismatch(_dlsize, self._fname, self.get_size()) - if not sha1: - return None - - checksum = hashlib.sha1() - with io.open(self._destination, "rb", buffering=0) as fp: - while True: - chunk = fp.read(io.DEFAULT_BUFFER_SIZE) - if not chunk: - break - checksum.update(chunk) - - if sha1.lower() != checksum.hexdigest().lower(): - raise FDSHASumMismatch(checksum.hexdigest(), self._fname, sha1) + raise PackageException( + ( + "The size ({0:d} bytes) of downloaded file '{1}' " + "is not equal to remote size ({2:d} bytes)" + ).format(_dlsize, self._fname, self.get_size()) + ) + if not checksum: + return True + + checksum_len = len(checksum) + hash_algo = None + if checksum_len == 32: + hash_algo = "md5" + elif checksum_len == 40: + hash_algo = "sha1" + elif checksum_len == 64: + hash_algo = "sha256" + + if not hash_algo: + raise PackageException( + "Could not determine checksum algorithm by %s" % checksum + ) + + dl_checksum = fs.calculate_file_hashsum(hash_algo, self._destination) + if checksum.lower() != dl_checksum.lower(): + raise PackageException( + "The checksum '{0}' of the downloaded file '{1}' " + "does not match to the remote '{2}'".format( + dl_checksum, self._fname, checksum + ) + ) return True def _preserve_filemtime(self, lmdate): timedata = parsedate_tz(lmdate) lmtime = mktime(timedata[:9]) - util.change_filemtime(self._destination, lmtime) + fs.change_filemtime(self._destination, lmtime) def __del__(self): if self._request: diff --git a/platformio/package/exception.py b/platformio/package/exception.py index adadc0889f..5d63649e8d 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio.exception import PlatformioException +from platformio import util +from platformio.exception import PlatformioException, UserSideException class PackageException(PlatformioException): @@ -44,3 +45,27 @@ def __str__(self): "https://docs.platformio.org/page/librarymanager/config.html" % self.messages ) + + +class MissingPackageManifestError(ManifestException): + + MESSAGE = "Could not find one of '{0}' manifest files in the package" + + +class UnknownPackageError(UserSideException): + + MESSAGE = ( + "Could not find the package with '{0}' requirements for your system '%s'" + % util.get_systype() + ) + + +class NotGlobalLibDir(UserSideException): + + MESSAGE = ( + "The `{0}` is not a PlatformIO project.\n\n" + "To manage libraries in global storage `{1}`,\n" + "please use `platformio lib --global {2}` or specify custom storage " + "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" + "Check `platformio lib --help` for details." + ) diff --git a/platformio/lockfile.py b/platformio/package/lockfile.py similarity index 94% rename from platformio/lockfile.py rename to platformio/package/lockfile.py index 44d2e4cf2f..db4b1d3ff9 100644 --- a/platformio/lockfile.py +++ b/platformio/package/lockfile.py @@ -15,7 +15,7 @@ import os from time import sleep, time -from platformio import exception +from platformio.exception import PlatformioException LOCKFILE_TIMEOUT = 3600 # in seconds, 1 hour LOCKFILE_DELAY = 0.2 @@ -36,7 +36,11 @@ LOCKFILE_CURRENT_INTERFACE = None -class LockFileExists(Exception): +class LockFileExists(PlatformioException): + pass + + +class LockFileTimeoutError(PlatformioException): pass @@ -88,7 +92,7 @@ def acquire(self): sleep(self.delay) elapsed += self.delay - raise exception.LockFileTimeoutError() + raise LockFileTimeoutError() def release(self): self._unlock() diff --git a/platformio/package/manager/__init__.py b/platformio/package/manager/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/package/manager/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py new file mode 100644 index 0000000000..4039568b61 --- /dev/null +++ b/platformio/package/manager/_download.py @@ -0,0 +1,95 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import tempfile +import time + +from platformio import app, compat +from platformio.package.download import FileDownloader +from platformio.package.lockfile import LockFile + + +class PackageManagerDownloadMixin(object): + + DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month + + def compute_download_path(self, *args): + request_hash = hashlib.new("sha1") + for arg in args: + request_hash.update(compat.hashlib_encode_data(arg)) + dl_path = os.path.join(self.get_download_dir(), request_hash.hexdigest()) + return dl_path + + def get_download_usagedb_path(self): + return os.path.join(self.get_download_dir(), "usage.db") + + def set_download_utime(self, path, utime=None): + with app.State(self.get_download_usagedb_path(), lock=True) as state: + state[os.path.basename(path)] = int(time.time() if not utime else utime) + + def cleanup_expired_downloads(self): + with app.State(self.get_download_usagedb_path(), lock=True) as state: + # remove outdated + for fname in list(state.keys()): + if state[fname] > (time.time() - self.DOWNLOAD_CACHE_EXPIRE): + continue + del state[fname] + dl_path = os.path.join(self.get_download_dir(), fname) + if os.path.isfile(dl_path): + os.remove(dl_path) + + def download(self, url, checksum=None, silent=False): + dl_path = self.compute_download_path(url, checksum or "") + if os.path.isfile(dl_path): + self.set_download_utime(dl_path) + return dl_path + + with_progress = not silent and not app.is_disabled_progressbar() + tmp_fd, tmp_path = tempfile.mkstemp(dir=self.get_download_dir()) + try: + with LockFile(dl_path): + try: + fd = FileDownloader(url) + fd.set_destination(tmp_path) + fd.start(with_progress=with_progress, silent=silent) + except IOError as e: + raise_error = not with_progress + if with_progress: + try: + fd = FileDownloader(url) + fd.set_destination(tmp_path) + fd.start(with_progress=False, silent=silent) + except IOError: + raise_error = True + if raise_error: + self.print_message( + "Error: Please read http://bit.ly/package-manager-ioerror", + fg="red", + err=True, + ) + raise e + if checksum: + fd.verify(checksum) + os.close(tmp_fd) + os.rename(tmp_path, dl_path) + finally: + if os.path.isfile(tmp_path): + os.close(tmp_fd) + os.remove(tmp_path) + + assert os.path.isfile(dl_path) + self.set_download_utime(dl_path) + return dl_path diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py new file mode 100644 index 0000000000..9d82d6fefd --- /dev/null +++ b/platformio/package/manager/_install.py @@ -0,0 +1,242 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os +import shutil +import tempfile + +import click + +from platformio import app, compat, fs, util +from platformio.package.exception import PackageException +from platformio.package.meta import PackageItem +from platformio.package.unpack import FileUnpacker +from platformio.package.vcsclient import VCSClientFactory + + +class PackageManagerInstallMixin(object): + + _INSTALL_HISTORY = None # avoid circle dependencies + + @staticmethod + def unpack(src, dst): + with_progress = not app.is_disabled_progressbar() + try: + with FileUnpacker(src) as fu: + return fu.unpack(dst, with_progress=with_progress) + except IOError as e: + if not with_progress: + raise e + with FileUnpacker(src) as fu: + return fu.unpack(dst, with_progress=False) + + def install(self, spec, silent=False, skip_dependencies=False, force=False): + try: + self.lock() + pkg = self._install( + spec, silent=silent, skip_dependencies=skip_dependencies, force=force + ) + self.memcache_reset() + self.cleanup_expired_downloads() + return pkg + finally: + self.unlock() + + def _install( # pylint: disable=too-many-arguments + self, + spec, + search_filters=None, + silent=False, + skip_dependencies=False, + force=False, + ): + spec = self.ensure_spec(spec) + + # avoid circle dependencies + if not self._INSTALL_HISTORY: + self._INSTALL_HISTORY = {} + if spec in self._INSTALL_HISTORY: + return self._INSTALL_HISTORY[spec] + + # check if package is already installed + pkg = self.get_package(spec) + + # if a forced installation + if pkg and force: + self.uninstall(pkg, silent=silent) + pkg = None + + if pkg: + if not silent: + self.print_message( + "{name} @ {version} is already installed".format( + **pkg.metadata.as_dict() + ), + fg="yellow", + ) + return pkg + + if not silent: + self.print_message( + "Installing %s" % click.style(spec.humanize(), fg="cyan") + ) + + if spec.external: + pkg = self.install_from_url(spec.url, spec, silent=silent) + else: + pkg = self.install_from_registry(spec, search_filters, silent=silent) + + if not pkg or not pkg.metadata: + raise PackageException( + "Could not install package '%s' for '%s' system" + % (spec.humanize(), util.get_systype()) + ) + + if not silent: + self.print_message( + "{name} @ {version} has been installed!".format( + **pkg.metadata.as_dict() + ), + fg="green", + ) + + self.memcache_reset() + if not skip_dependencies: + self.install_dependencies(pkg, silent) + self._INSTALL_HISTORY[spec] = pkg + return pkg + + def install_dependencies(self, pkg, silent=False): + pass + + def install_from_url(self, url, spec, checksum=None, silent=False): + spec = self.ensure_spec(spec) + tmp_dir = tempfile.mkdtemp(prefix="pkg-installing-", dir=self.get_tmp_dir()) + vcs = None + try: + if url.startswith("file://"): + _url = url[7:] + if os.path.isfile(_url): + self.unpack(_url, tmp_dir) + else: + fs.rmtree(tmp_dir) + shutil.copytree(_url, tmp_dir, symlinks=True) + elif url.startswith(("http://", "https://")): + dl_path = self.download(url, checksum, silent=silent) + assert os.path.isfile(dl_path) + self.unpack(dl_path, tmp_dir) + else: + vcs = VCSClientFactory.new(tmp_dir, url) + assert vcs.export() + + root_dir = self.find_pkg_root(tmp_dir, spec) + pkg_item = PackageItem( + root_dir, + self.build_metadata( + root_dir, spec, vcs.get_current_revision() if vcs else None + ), + ) + pkg_item.dump_meta() + return self._install_tmp_pkg(pkg_item) + finally: + if os.path.isdir(tmp_dir): + fs.rmtree(tmp_dir) + + def _install_tmp_pkg(self, tmp_pkg): + assert isinstance(tmp_pkg, PackageItem) + # validate package version and declared requirements + if ( + tmp_pkg.metadata.spec.requirements + and tmp_pkg.metadata.version not in tmp_pkg.metadata.spec.requirements + ): + raise PackageException( + "Package version %s doesn't satisfy requirements %s based on %s" + % ( + tmp_pkg.metadata.version, + tmp_pkg.metadata.spec.requirements, + tmp_pkg.metadata, + ) + ) + dst_pkg = PackageItem( + os.path.join(self.package_dir, tmp_pkg.get_safe_dirname()) + ) + + # what to do with existing package? + action = "overwrite" + if tmp_pkg.metadata.spec.has_custom_name(): + action = "overwrite" + dst_pkg = PackageItem( + os.path.join(self.package_dir, tmp_pkg.metadata.spec.name) + ) + elif dst_pkg.metadata: + if dst_pkg.metadata.spec.external: + if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: + action = "detach-existing" + elif ( + dst_pkg.metadata.version != tmp_pkg.metadata.version + or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner + ): + action = ( + "detach-existing" + if tmp_pkg.metadata.version > dst_pkg.metadata.version + else "detach-new" + ) + + def _cleanup_dir(path): + if os.path.isdir(path): + fs.rmtree(path) + + if action == "detach-existing": + target_dirname = "%s@%s" % ( + tmp_pkg.get_safe_dirname(), + dst_pkg.metadata.version, + ) + if dst_pkg.metadata.spec.url: + target_dirname = "%s@src-%s" % ( + tmp_pkg.get_safe_dirname(), + hashlib.md5( + compat.hashlib_encode_data(dst_pkg.metadata.spec.url) + ).hexdigest(), + ) + # move existing into the new place + pkg_dir = os.path.join(self.package_dir, target_dirname) + _cleanup_dir(pkg_dir) + shutil.move(dst_pkg.path, pkg_dir) + # move new source to the destination location + _cleanup_dir(dst_pkg.path) + shutil.move(tmp_pkg.path, dst_pkg.path) + return PackageItem(dst_pkg.path) + + if action == "detach-new": + target_dirname = "%s@%s" % ( + tmp_pkg.get_safe_dirname(), + tmp_pkg.metadata.version, + ) + if tmp_pkg.metadata.spec.external: + target_dirname = "%s@src-%s" % ( + tmp_pkg.get_safe_dirname(), + hashlib.md5( + compat.hashlib_encode_data(tmp_pkg.metadata.spec.url) + ).hexdigest(), + ) + pkg_dir = os.path.join(self.package_dir, target_dirname) + _cleanup_dir(pkg_dir) + shutil.move(tmp_pkg.path, pkg_dir) + return PackageItem(pkg_dir) + + # otherwise, overwrite existing + _cleanup_dir(dst_pkg.path) + shutil.move(tmp_pkg.path, dst_pkg.path) + return PackageItem(dst_pkg.path) diff --git a/platformio/package/manager/_legacy.py b/platformio/package/manager/_legacy.py new file mode 100644 index 0000000000..5c35ebeb12 --- /dev/null +++ b/platformio/package/manager/_legacy.py @@ -0,0 +1,61 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import fs +from platformio.package.meta import PackageItem, PackageSpec + + +class PackageManagerLegacyMixin(object): + def build_legacy_spec(self, pkg_dir): + # find src manifest + src_manifest_name = ".piopkgmanager.json" + src_manifest_path = None + for name in os.listdir(pkg_dir): + if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)): + continue + src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name) + break + + if src_manifest_path: + src_manifest = fs.load_json(src_manifest_path) + return PackageSpec( + name=src_manifest.get("name"), + url=src_manifest.get("url"), + requirements=src_manifest.get("requirements"), + ) + + # fall back to a package manifest + manifest = self.load_manifest(pkg_dir) + return PackageSpec(name=manifest.get("name")) + + def legacy_load_manifest(self, pkg): + if not isinstance(pkg, PackageItem): + assert os.path.isdir(pkg) + pkg = PackageItem(pkg) + manifest = self.load_manifest(pkg) + manifest["__pkg_dir"] = pkg.path + for key in ("name", "version"): + if not manifest.get(key): + manifest[key] = str(getattr(pkg.metadata, key)) + if pkg.metadata and pkg.metadata.spec and pkg.metadata.spec.external: + manifest["__src_url"] = pkg.metadata.spec.url + manifest["version"] = str(pkg.metadata.version) + if pkg.metadata and pkg.metadata.spec.owner: + manifest["ownername"] = pkg.metadata.spec.owner + return manifest + + def legacy_get_installed(self): + return [self.legacy_load_manifest(pkg) for pkg in self.get_installed()] diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py new file mode 100644 index 0000000000..6d243d7623 --- /dev/null +++ b/platformio/package/manager/_registry.py @@ -0,0 +1,229 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import click + +from platformio.clients.http import HTTPClient +from platformio.clients.registry import RegistryClient +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageSpec +from platformio.package.version import cast_version_to_semver + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +class RegistryFileMirrorIterator(object): + + HTTP_CLIENT_INSTANCES = {} + + def __init__(self, download_url): + self.download_url = download_url + self._url_parts = urlparse(download_url) + self._mirror = "%s://%s" % (self._url_parts.scheme, self._url_parts.netloc) + self._visited_mirrors = [] + + def __iter__(self): # pylint: disable=non-iterator-returned + return self + + def next(self): + """ For Python 2 compatibility """ + return self.__next__() + + def __next__(self): + http = self.get_http_client() + response = http.send_request( + "head", + self._url_parts.path, + allow_redirects=False, + params=dict(bypass=",".join(self._visited_mirrors)) + if self._visited_mirrors + else None, + ) + stop_conditions = [ + response.status_code not in (302, 307), + not response.headers.get("Location"), + not response.headers.get("X-PIO-Mirror"), + response.headers.get("X-PIO-Mirror") in self._visited_mirrors, + ] + if any(stop_conditions): + raise StopIteration + self._visited_mirrors.append(response.headers.get("X-PIO-Mirror")) + return ( + response.headers.get("Location"), + response.headers.get("X-PIO-Content-SHA256"), + ) + + def get_http_client(self): + if self._mirror not in RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES: + RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] = HTTPClient( + self._mirror + ) + return RegistryFileMirrorIterator.HTTP_CLIENT_INSTANCES[self._mirror] + + +class PackageManageRegistryMixin(object): + def install_from_registry(self, spec, search_filters=None, silent=False): + if spec.owner and spec.name and not search_filters: + package = self.fetch_registry_package(spec) + if not package: + raise UnknownPackageError(spec.humanize()) + version = self.pick_best_registry_version(package["versions"], spec) + else: + packages = self.search_registry_packages(spec, search_filters) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1 and not silent: + self.print_multi_package_issue(packages, spec) + package, version = self.find_best_registry_version(packages, spec) + + if not package or not version: + raise UnknownPackageError(spec.humanize()) + + pkgfile = self._pick_compatible_pkg_file(version["files"]) if version else None + if not pkgfile: + raise UnknownPackageError(spec.humanize()) + + for url, checksum in RegistryFileMirrorIterator(pkgfile["download_url"]): + try: + return self.install_from_url( + url, + PackageSpec( + owner=package["owner"]["username"], + id=package["id"], + name=package["name"], + ), + checksum or pkgfile["checksum"]["sha256"], + silent=silent, + ) + except Exception as e: # pylint: disable=broad-except + self.print_message("Warning! Package Mirror: %s" % e, fg="yellow") + self.print_message("Looking for another mirror...", fg="yellow") + + return None + + def get_registry_client_instance(self): + if not self._registry_client: + self._registry_client = RegistryClient() + return self._registry_client + + def search_registry_packages(self, spec, filters=None): + assert isinstance(spec, PackageSpec) + filters = filters or {} + if spec.id: + filters["ids"] = str(spec.id) + else: + filters["types"] = self.pkg_type + filters["names"] = spec.name.lower() + if spec.owner: + filters["owners"] = spec.owner.lower() + return self.get_registry_client_instance().list_packages(filters=filters)[ + "items" + ] + + def fetch_registry_package(self, spec): + assert isinstance(spec, PackageSpec) + result = None + regclient = self.get_registry_client_instance() + if spec.owner and spec.name: + result = regclient.get_package(self.pkg_type, spec.owner, spec.name) + if not result and (spec.id or (spec.name and not spec.owner)): + packages = self.search_registry_packages(spec) + if packages: + result = regclient.get_package( + self.pkg_type, packages[0]["owner"]["username"], packages[0]["name"] + ) + if not result: + raise UnknownPackageError(spec.humanize()) + return result + + def reveal_registry_package_id(self, spec, silent=False): + spec = self.ensure_spec(spec) + if spec.id: + return spec.id + packages = self.search_registry_packages(spec) + if not packages: + raise UnknownPackageError(spec.humanize()) + if len(packages) > 1 and not silent: + self.print_multi_package_issue(packages, spec) + click.echo("") + return packages[0]["id"] + + def print_multi_package_issue(self, packages, spec): + self.print_message( + "Warning! More than one package has been found by ", fg="yellow", nl=False + ) + click.secho(spec.humanize(), fg="cyan", nl=False) + click.secho(" requirements:", fg="yellow") + for item in packages: + click.echo( + " - {owner}/{name} @ {version}".format( + owner=click.style(item["owner"]["username"], fg="cyan"), + name=item["name"], + version=item["version"]["name"], + ) + ) + self.print_message( + "Please specify detailed REQUIREMENTS using package owner and version " + "(showed above) to avoid name conflicts", + fg="yellow", + ) + + def find_best_registry_version(self, packages, spec): + for package in packages: + # find compatible version within the latest package versions + version = self.pick_best_registry_version([package["version"]], spec) + if version: + return (package, version) + + # if the custom version requirements, check ALL package versions + version = self.pick_best_registry_version( + self.fetch_registry_package( + PackageSpec( + id=package["id"], + owner=package["owner"]["username"], + name=package["name"], + ) + ).get("versions"), + spec, + ) + if version: + return (package, version) + time.sleep(1) + return (None, None) + + def pick_best_registry_version(self, versions, spec=None): + assert not spec or isinstance(spec, PackageSpec) + best = None + for version in versions: + semver = cast_version_to_semver(version["name"]) + if spec and spec.requirements and semver not in spec.requirements: + continue + if not any( + self.is_system_compatible(f.get("system")) for f in version["files"] + ): + continue + if not best or (semver > cast_version_to_semver(best["name"])): + best = version + return best + + def _pick_compatible_pkg_file(self, version_files): + for item in version_files: + if self.is_system_compatible(item.get("system")): + return item + return None diff --git a/platformio/package/manager/_uninstall.py b/platformio/package/manager/_uninstall.py new file mode 100644 index 0000000000..68f7a30015 --- /dev/null +++ b/platformio/package/manager/_uninstall.py @@ -0,0 +1,78 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil + +import click + +from platformio import fs +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageSpec + + +class PackageManagerUninstallMixin(object): + def uninstall(self, spec, silent=False, skip_dependencies=False): + try: + self.lock() + return self._uninstall(spec, silent, skip_dependencies) + finally: + self.unlock() + + def _uninstall(self, spec, silent=False, skip_dependencies=False): + pkg = self.get_package(spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(spec) + + if not silent: + self.print_message( + "Removing %s @ %s" + % (click.style(pkg.metadata.name, fg="cyan"), pkg.metadata.version), + ) + + # firstly, remove dependencies + if not skip_dependencies: + self.uninstall_dependencies(pkg, silent) + + if os.path.islink(pkg.path): + os.unlink(pkg.path) + else: + fs.rmtree(pkg.path) + self.memcache_reset() + + # unfix detached-package with the same name + detached_pkg = self.get_package(PackageSpec(name=pkg.metadata.name)) + if ( + detached_pkg + and "@" in detached_pkg.path + and not os.path.isdir( + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()) + ) + ): + shutil.move( + detached_pkg.path, + os.path.join(self.package_dir, detached_pkg.get_safe_dirname()), + ) + self.memcache_reset() + + if not silent: + self.print_message( + "{name} @ {version} has been removed!".format(**pkg.metadata.as_dict()), + fg="green", + ) + + return pkg + + def uninstall_dependencies(self, pkg, silent=False): + pass diff --git a/platformio/package/manager/_update.py b/platformio/package/manager/_update.py new file mode 100644 index 0000000000..3b6dd2d44d --- /dev/null +++ b/platformio/package/manager/_update.py @@ -0,0 +1,169 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click + +from platformio.clients.http import ensure_internet_on +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageItem, PackageOutdatedResult, PackageSpec +from platformio.package.vcsclient import VCSBaseException, VCSClientFactory + + +class PackageManagerUpdateMixin(object): + def outdated(self, pkg, spec=None): + assert isinstance(pkg, PackageItem) + assert not spec or isinstance(spec, PackageSpec) + assert os.path.isdir(pkg.path) and pkg.metadata + + # skip detached package to a specific version + detached_conditions = [ + "@" in pkg.path, + pkg.metadata.spec and not pkg.metadata.spec.external, + not spec, + ] + if all(detached_conditions): + return PackageOutdatedResult(current=pkg.metadata.version, detached=True) + + latest = None + wanted = None + if pkg.metadata.spec.external: + latest = self._fetch_vcs_latest_version(pkg) + else: + try: + reg_pkg = self.fetch_registry_package(pkg.metadata.spec) + latest = ( + self.pick_best_registry_version(reg_pkg["versions"]) or {} + ).get("name") + if spec: + wanted = ( + self.pick_best_registry_version(reg_pkg["versions"], spec) or {} + ).get("name") + if not wanted: # wrong library + latest = None + except UnknownPackageError: + pass + + return PackageOutdatedResult( + current=pkg.metadata.version, latest=latest, wanted=wanted + ) + + def _fetch_vcs_latest_version(self, pkg): + vcs = None + try: + vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url, silent=True) + except VCSBaseException: + return None + if not vcs.can_be_updated: + return None + return str( + self.build_metadata( + pkg.path, pkg.metadata.spec, vcs_revision=vcs.get_latest_revision() + ).version + ) + + def update( # pylint: disable=too-many-arguments + self, + from_spec, + to_spec=None, + only_check=False, + silent=False, + show_incompatible=True, + ): + pkg = self.get_package(from_spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(from_spec) + + if not silent: + click.echo( + "{} {:<45} {:<35}".format( + "Checking" if only_check else "Updating", + click.style(pkg.metadata.spec.humanize(), fg="cyan"), + "%s @ %s" % (pkg.metadata.version, to_spec.requirements) + if to_spec and to_spec.requirements + else str(pkg.metadata.version), + ), + nl=False, + ) + if not ensure_internet_on(): + if not silent: + click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) + return pkg + + outdated = self.outdated(pkg, to_spec) + if not silent: + self.print_outdated_state(outdated, show_incompatible) + + if only_check or not outdated.is_outdated(allow_incompatible=False): + return pkg + + try: + self.lock() + return self._update(pkg, outdated, silent=silent) + finally: + self.unlock() + + @staticmethod + def print_outdated_state(outdated, show_incompatible=True): + if outdated.detached: + return click.echo("[%s]" % (click.style("Detached", fg="yellow"))) + if ( + not outdated.latest + or outdated.current == outdated.latest + or (not show_incompatible and outdated.current == outdated.wanted) + ): + return click.echo("[%s]" % (click.style("Up-to-date", fg="green"))) + if outdated.wanted and outdated.current == outdated.wanted: + return click.echo( + "[%s]" % (click.style("Incompatible %s" % outdated.latest, fg="yellow")) + ) + return click.echo( + "[%s]" + % ( + click.style( + "Outdated %s" % str(outdated.wanted or outdated.latest), fg="red" + ) + ) + ) + + def _update(self, pkg, outdated, silent=False): + if pkg.metadata.spec.external: + vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url) + assert vcs.update() + pkg.metadata.version = self._fetch_vcs_latest_version(pkg) + pkg.dump_meta() + return pkg + + new_pkg = self.install( + PackageSpec( + id=pkg.metadata.spec.id, + owner=pkg.metadata.spec.owner, + name=pkg.metadata.spec.name, + requirements=outdated.wanted or outdated.latest, + ), + silent=silent, + ) + if new_pkg: + old_pkg = self.get_package( + PackageSpec( + id=pkg.metadata.spec.id, + owner=pkg.metadata.spec.owner, + name=pkg.metadata.name, + requirements=pkg.metadata.version, + ) + ) + if old_pkg: + self.uninstall(old_pkg, silent=silent, skip_dependencies=True) + return new_pkg diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py new file mode 100644 index 0000000000..6cb609a3c8 --- /dev/null +++ b/platformio/package/manager/base.py @@ -0,0 +1,268 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from datetime import datetime + +import click +import semantic_version + +from platformio import util +from platformio.commands import PlatformioCLI +from platformio.compat import ci_strings_are_equal +from platformio.package.exception import ManifestException, MissingPackageManifestError +from platformio.package.lockfile import LockFile +from platformio.package.manager._download import PackageManagerDownloadMixin +from platformio.package.manager._install import PackageManagerInstallMixin +from platformio.package.manager._legacy import PackageManagerLegacyMixin +from platformio.package.manager._registry import PackageManageRegistryMixin +from platformio.package.manager._uninstall import PackageManagerUninstallMixin +from platformio.package.manager._update import PackageManagerUpdateMixin +from platformio.package.manifest.parser import ManifestParserFactory +from platformio.package.meta import ( + PackageItem, + PackageMetaData, + PackageSpec, + PackageType, +) +from platformio.project.helpers import get_project_cache_dir + + +class BasePackageManager( # pylint: disable=too-many-public-methods + PackageManagerDownloadMixin, + PackageManageRegistryMixin, + PackageManagerInstallMixin, + PackageManagerUninstallMixin, + PackageManagerUpdateMixin, + PackageManagerLegacyMixin, +): + _MEMORY_CACHE = {} + + def __init__(self, pkg_type, package_dir): + self.pkg_type = pkg_type + self.package_dir = self.ensure_dir_exists(package_dir) + self._MEMORY_CACHE = {} + + self._lockfile = None + self._download_dir = None + self._tmp_dir = None + self._registry_client = None + + def lock(self): + if self._lockfile: + return + self._lockfile = LockFile(self.package_dir) + self._lockfile.acquire() + + def unlock(self): + if hasattr(self, "_lockfile") and self._lockfile: + self._lockfile.release() + self._lockfile = None + + def __del__(self): + self.unlock() + + def memcache_get(self, key, default=None): + return self._MEMORY_CACHE.get(key, default) + + def memcache_set(self, key, value): + self._MEMORY_CACHE[key] = value + + def memcache_reset(self): + self._MEMORY_CACHE.clear() + + @staticmethod + def is_system_compatible(value): + if not value or "*" in value: + return True + return util.items_in_list(value, util.get_systype()) + + @staticmethod + def ensure_dir_exists(path): + if not os.path.isdir(path): + os.makedirs(path) + assert os.path.isdir(path) + return path + + @staticmethod + def ensure_spec(spec): + return spec if isinstance(spec, PackageSpec) else PackageSpec(spec) + + @property + def manifest_names(self): + raise NotImplementedError + + def print_message(self, message, **kwargs): + click.echo( + "%s: " % str(self.__class__.__name__).replace("Package", " "), nl=False + ) + click.secho(message, **kwargs) + + def get_download_dir(self): + if not self._download_dir: + self._download_dir = self.ensure_dir_exists( + os.path.join(get_project_cache_dir(), "downloads") + ) + return self._download_dir + + def get_tmp_dir(self): + if not self._tmp_dir: + self._tmp_dir = self.ensure_dir_exists( + os.path.join(get_project_cache_dir(), "tmp") + ) + return self._tmp_dir + + def find_pkg_root(self, path, spec): # pylint: disable=unused-argument + if self.manifest_exists(path): + return path + for root, _, _ in os.walk(path): + if self.manifest_exists(root): + return root + raise MissingPackageManifestError(", ".join(self.manifest_names)) + + def get_manifest_path(self, pkg_dir): + if not os.path.isdir(pkg_dir): + return None + for name in self.manifest_names: + manifest_path = os.path.join(pkg_dir, name) + if os.path.isfile(manifest_path): + return manifest_path + return None + + def manifest_exists(self, pkg_dir): + return self.get_manifest_path(pkg_dir) + + def load_manifest(self, src): + path = src.path if isinstance(src, PackageItem) else src + cache_key = "load_manifest-%s" % path + result = self.memcache_get(cache_key) + if result: + return result + candidates = ( + [os.path.join(path, name) for name in self.manifest_names] + if os.path.isdir(path) + else [path] + ) + for item in candidates: + if not os.path.isfile(item): + continue + try: + result = ManifestParserFactory.new_from_file(item).as_dict() + self.memcache_set(cache_key, result) + return result + except ManifestException as e: + if not PlatformioCLI.in_silence(): + self.print_message(str(e), fg="yellow") + raise MissingPackageManifestError(", ".join(self.manifest_names)) + + @staticmethod + def generate_rand_version(): + return datetime.now().strftime("0.0.0+%Y%m%d%H%M%S") + + def build_metadata(self, pkg_dir, spec, vcs_revision=None): + manifest = self.load_manifest(pkg_dir) + metadata = PackageMetaData( + type=self.pkg_type, + name=manifest.get("name"), + version=manifest.get("version"), + spec=spec, + ) + if not metadata.name or spec.has_custom_name(): + metadata.name = spec.name + if vcs_revision: + metadata.version = "%s+sha.%s" % ( + metadata.version if metadata.version else "0.0.0", + vcs_revision, + ) + if not metadata.version: + metadata.version = self.generate_rand_version() + return metadata + + def get_installed(self): + cache_key = "get_installed" + if self.memcache_get(cache_key): + return self.memcache_get(cache_key) + + result = [] + for name in sorted(os.listdir(self.package_dir)): + if name.startswith("_tmp_installing"): # legacy tmp folder + continue + pkg_dir = os.path.join(self.package_dir, name) + if not os.path.isdir(pkg_dir): + continue + pkg = PackageItem(pkg_dir) + if not pkg.metadata: + try: + spec = self.build_legacy_spec(pkg_dir) + pkg.metadata = self.build_metadata(pkg_dir, spec) + except MissingPackageManifestError: + pass + if not pkg.metadata: + continue + if self.pkg_type == PackageType.TOOL: + try: + if not self.is_system_compatible( + self.load_manifest(pkg).get("system") + ): + continue + except MissingPackageManifestError: + pass + result.append(pkg) + + self.memcache_set(cache_key, result) + return result + + def get_package(self, spec): + if isinstance(spec, PackageItem): + return spec + spec = self.ensure_spec(spec) + best = None + for pkg in self.get_installed(): + if not self.test_pkg_spec(pkg, spec): + continue + assert isinstance(pkg.metadata.version, semantic_version.Version) + if spec.requirements and pkg.metadata.version not in spec.requirements: + continue + if not best or (pkg.metadata.version > best.metadata.version): + best = pkg + return best + + @staticmethod + def test_pkg_spec(pkg, spec): + # "id" mismatch + if spec.id and spec.id != pkg.metadata.spec.id: + return False + + # external "URL" mismatch + if spec.external: + # local folder mismatch + if os.path.realpath(spec.url) == os.path.realpath(pkg.path) or ( + spec.url.startswith("file://") + and os.path.realpath(pkg.path) == os.path.realpath(spec.url[7:]) + ): + return True + if spec.url != pkg.metadata.spec.url: + return False + + # "owner" mismatch + elif spec.owner and not ci_strings_are_equal( + spec.owner, pkg.metadata.spec.owner + ): + return False + + # "name" mismatch + elif not spec.id and not ci_strings_are_equal(spec.name, pkg.metadata.name): + return False + + return True diff --git a/platformio/managers/core.py b/platformio/package/manager/core.py similarity index 55% rename from platformio/managers/core.py rename to platformio/package/manager/core.py index 53f435fd91..a11217e94a 100644 --- a/platformio/managers/core.py +++ b/platformio/package/manager/core.py @@ -17,89 +17,58 @@ import subprocess import sys -from platformio import exception, util +from platformio import __core_packages__, exception, fs, util from platformio.compat import PY2 -from platformio.managers.package import PackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec from platformio.proc import get_pythonexe_path -from platformio.project.config import ProjectConfig - -CORE_PACKAGES = { - "contrib-piohome": "~3.2.1", - "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), - "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0", - "tool-cppcheck": "~1.190.0", - "tool-clangtidy": "~1.100000.0", - "tool-pvs-studio": "~7.7.0", -} - -# pylint: disable=arguments-differ,signature-differs - - -class CorePackageManager(PackageManager): - def __init__(self): - config = ProjectConfig.get_instance() - packages_dir = config.get_optional_dir("packages") - super(CorePackageManager, self).__init__( - packages_dir, - [ - "https://dl.bintray.com/platformio/dl-packages/manifest.json", - "http%s://dl.platformio.org/packages/manifest.json" - % ("" if sys.version_info < (2, 7, 9) else "s"), - ], - ) - - def install( # pylint: disable=keyword-arg-before-vararg - self, name, requirements=None, *args, **kwargs - ): - PackageManager.install(self, name, requirements, *args, **kwargs) - self.cleanup_packages() - return self.get_package_dir(name, requirements) - - def update(self, *args, **kwargs): - result = PackageManager.update(self, *args, **kwargs) - self.cleanup_packages() - return result - - def cleanup_packages(self): - self.cache_reset() - best_pkg_versions = {} - for name, requirements in CORE_PACKAGES.items(): - pkg_dir = self.get_package_dir(name, requirements) - if not pkg_dir: - continue - best_pkg_versions[name] = self.load_manifest(pkg_dir)["version"] - for manifest in self.get_installed(): - if manifest["name"] not in best_pkg_versions: - continue - if manifest["version"] != best_pkg_versions[manifest["name"]]: - self.uninstall(manifest["__pkg_dir"], after_update=True) - self.cache_reset() - return True def get_core_package_dir(name): - if name not in CORE_PACKAGES: - raise exception.PlatformioException("Please upgrade PIO Core") - requirements = CORE_PACKAGES[name] - pm = CorePackageManager() - pkg_dir = pm.get_package_dir(name, requirements) - if pkg_dir: - return pkg_dir - return pm.install(name, requirements) + if name not in __core_packages__: + raise exception.PlatformioException("Please upgrade PlatformIO Core") + pm = ToolPackageManager() + spec = PackageSpec( + owner="platformio", name=name, requirements=__core_packages__[name] + ) + pkg = pm.get_package(spec) + if pkg: + return pkg.path + assert pm.install(spec) + _remove_unnecessary_packages() + return pm.get_package(spec).path def update_core_packages(only_check=False, silent=False): - pm = CorePackageManager() - for name, requirements in CORE_PACKAGES.items(): - pkg_dir = pm.get_package_dir(name) - if not pkg_dir: + pm = ToolPackageManager() + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if not pkg: continue - if not silent or pm.outdated(pkg_dir, requirements): - pm.update(name, requirements, only_check=only_check) + if not silent or pm.outdated(pkg, spec).is_outdated(): + pm.update(pkg, spec, only_check=only_check) + if not only_check: + _remove_unnecessary_packages() return True +def _remove_unnecessary_packages(): + pm = ToolPackageManager() + best_pkg_versions = {} + for name, requirements in __core_packages__.items(): + spec = PackageSpec(owner="platformio", name=name, requirements=requirements) + pkg = pm.get_package(spec) + if not pkg: + continue + best_pkg_versions[pkg.metadata.name] = pkg.metadata.version + for pkg in pm.get_installed(): + if pkg.metadata.name not in best_pkg_versions: + continue + if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]: + pm.uninstall(pkg) + + def inject_contrib_pysite(verify_openssl=False): # pylint: disable=import-outside-toplevel from site import addsitedir @@ -124,7 +93,7 @@ def inject_contrib_pysite(verify_openssl=False): def build_contrib_pysite_deps(target_dir): if os.path.isdir(target_dir): - util.rmtree_(target_dir) + fs.rmtree(target_dir) os.makedirs(target_dir) with open(os.path.join(target_dir, "package.json"), "w") as fp: json.dump( diff --git a/platformio/package/manager/library.py b/platformio/package/manager/library.py new file mode 100644 index 0000000000..a0d1407f9c --- /dev/null +++ b/platformio/package/manager/library.py @@ -0,0 +1,107 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +from platformio.package.exception import MissingPackageManifestError +from platformio.package.manager.base import BasePackageManager +from platformio.package.meta import PackageItem, PackageSpec, PackageType +from platformio.project.helpers import get_project_global_lib_dir + + +class LibraryPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors + def __init__(self, package_dir=None): + super(LibraryPackageManager, self).__init__( + PackageType.LIBRARY, package_dir or get_project_global_lib_dir() + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.LIBRARY] + + def find_pkg_root(self, path, spec): + try: + return super(LibraryPackageManager, self).find_pkg_root(path, spec) + except MissingPackageManifestError: + pass + assert isinstance(spec, PackageSpec) + + root_dir = self.find_library_root(path) + + # automatically generate library manifest + with open(os.path.join(root_dir, "library.json"), "w") as fp: + json.dump( + dict(name=spec.name, version=self.generate_rand_version(),), + fp, + indent=2, + ) + + return root_dir + + @staticmethod + def find_library_root(path): + for root, dirs, files in os.walk(path): + if not files and len(dirs) == 1: + continue + for fname in files: + if not fname.endswith((".c", ".cpp", ".h", ".S")): + continue + if os.path.isdir(os.path.join(os.path.dirname(root), "src")): + return os.path.dirname(root) + return root + return path + + def install_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message("Installing dependencies...") + for dependency in manifest.get("dependencies"): + if not self._install_dependency(dependency, silent) and not silent: + self.print_message( + "Warning! Could not install dependency %s for package '%s'" + % (dependency, pkg.metadata.name), + fg="yellow", + ) + + def _install_dependency(self, dependency, silent=False): + spec = PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + search_filters = { + key: value + for key, value in dependency.items() + if key in ("authors", "platforms", "frameworks") + } + return self._install(spec, search_filters=search_filters or None, silent=silent) + + def uninstall_dependencies(self, pkg, silent=False): + assert isinstance(pkg, PackageItem) + manifest = self.load_manifest(pkg) + if not manifest.get("dependencies"): + return + if not silent: + self.print_message("Removing dependencies...", fg="yellow") + for dependency in manifest.get("dependencies"): + pkg = self.get_package( + PackageSpec( + name=dependency.get("name"), requirements=dependency.get("version") + ) + ) + if not pkg: + continue + self._uninstall(pkg, silent=silent) diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py new file mode 100644 index 0000000000..71e8c5fbfe --- /dev/null +++ b/platformio/package/manager/platform.py @@ -0,0 +1,195 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio import util +from platformio.clients.http import HTTPClientError, InternetIsOffline +from platformio.package.exception import UnknownPackageError +from platformio.package.manager.base import BasePackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageType +from platformio.platform.exception import IncompatiblePlatform, UnknownBoard +from platformio.platform.factory import PlatformFactory +from platformio.project.config import ProjectConfig + + +class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors + def __init__(self, package_dir=None): + self.config = ProjectConfig.get_instance() + super(PlatformPackageManager, self).__init__( + PackageType.PLATFORM, + package_dir or self.config.get_optional_dir("platforms"), + ) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.PLATFORM] + + def install( # pylint: disable=arguments-differ, too-many-arguments + self, + spec, + with_packages=None, + without_packages=None, + skip_default_package=False, + with_all_packages=False, + silent=False, + force=False, + ): + pkg = super(PlatformPackageManager, self).install( + spec, silent=silent, force=force, skip_dependencies=True + ) + try: + p = PlatformFactory.new(pkg) + p.ensure_engine_compatible() + except IncompatiblePlatform as e: + super(PlatformPackageManager, self).uninstall( + pkg, silent=silent, skip_dependencies=True + ) + raise e + + if with_all_packages: + with_packages = list(p.packages) + + p.install_packages( + with_packages, + without_packages, + skip_default_package, + silent=silent, + force=force, + ) + p.install_python_packages() + p.on_installed() + self.cleanup_packages(list(p.packages)) + return pkg + + def uninstall(self, spec, silent=False, skip_dependencies=False): + pkg = self.get_package(spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(spec) + p = PlatformFactory.new(pkg) + assert super(PlatformPackageManager, self).uninstall( + pkg, silent=silent, skip_dependencies=True + ) + if not skip_dependencies: + p.uninstall_python_packages() + p.on_uninstalled() + self.cleanup_packages(list(p.packages)) + return pkg + + def update( # pylint: disable=arguments-differ, too-many-arguments + self, + from_spec, + to_spec=None, + only_check=False, + silent=False, + show_incompatible=True, + only_packages=False, + ): + pkg = self.get_package(from_spec) + if not pkg or not pkg.metadata: + raise UnknownPackageError(from_spec) + p = PlatformFactory.new(pkg) + pkgs_before = [item.metadata.name for item in p.get_installed_packages()] + + new_pkg = None + missed_pkgs = set() + if not only_packages: + new_pkg = super(PlatformPackageManager, self).update( + from_spec, + to_spec, + only_check=only_check, + silent=silent, + show_incompatible=show_incompatible, + ) + p = PlatformFactory.new(new_pkg) + missed_pkgs = set(pkgs_before) & set(p.packages) + missed_pkgs -= set( + item.metadata.name for item in p.get_installed_packages() + ) + + p.update_packages(only_check) + self.cleanup_packages(list(p.packages)) + + if missed_pkgs: + p.install_packages( + with_packages=list(missed_pkgs), skip_default_package=True + ) + + return new_pkg or pkg + + def cleanup_packages(self, names): + self.memcache_reset() + deppkgs = {} + for platform in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(platform) + for pkg in p.get_installed_packages(): + if pkg.metadata.name not in deppkgs: + deppkgs[pkg.metadata.name] = set() + deppkgs[pkg.metadata.name].add(pkg.metadata.version) + + pm = ToolPackageManager() + for pkg in pm.get_installed(): + if pkg.metadata.name not in names: + continue + if ( + pkg.metadata.name not in deppkgs + or pkg.metadata.version not in deppkgs[pkg.metadata.name] + ): + try: + pm.uninstall(pkg.metadata.spec) + except UnknownPackageError: + pass + + self.memcache_reset() + return True + + @util.memoized(expire="5s") + def get_installed_boards(self): + boards = [] + for pkg in self.get_installed(): + p = PlatformFactory.new(pkg) + for config in p.get_boards().values(): + board = config.get_brief_data() + if board not in boards: + boards.append(board) + return boards + + def get_registered_boards(self): + return self.get_registry_client_instance().fetch_json_data( + "get", "/v2/boards", cache_valid="1d" + ) + + def get_all_boards(self): + boards = self.get_installed_boards() + know_boards = ["%s:%s" % (b["platform"], b["id"]) for b in boards] + try: + for board in self.get_registered_boards(): + key = "%s:%s" % (board["platform"], board["id"]) + if key not in know_boards: + boards.append(board) + except (HTTPClientError, InternetIsOffline): + pass + return sorted(boards, key=lambda b: b["name"]) + + def board_config(self, id_, platform=None): + for manifest in self.get_installed_boards(): + if manifest["id"] == id_ and ( + not platform or manifest["platform"] == platform + ): + return manifest + for manifest in self.get_registered_boards(): + if manifest["id"] == id_ and ( + not platform or manifest["platform"] == platform + ): + return manifest + raise UnknownBoard(id_) diff --git a/platformio/package/manager/tool.py b/platformio/package/manager/tool.py new file mode 100644 index 0000000000..60aededd45 --- /dev/null +++ b/platformio/package/manager/tool.py @@ -0,0 +1,28 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.manager.base import BasePackageManager +from platformio.package.meta import PackageType +from platformio.project.config import ProjectConfig + + +class ToolPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors + def __init__(self, package_dir=None): + if not package_dir: + package_dir = ProjectConfig.get_instance().get_optional_dir("packages") + super(ToolPackageManager, self).__init__(PackageType.TOOL, package_dir) + + @property + def manifest_names(self): + return PackageType.get_manifest_map()[PackageType.TOOL] diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index e8ec592969..8949f43e53 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -17,10 +17,10 @@ import json import os import re - -import requests +import tarfile from platformio import util +from platformio.clients.http import fetch_remote_content from platformio.compat import get_object_members, string_types from platformio.package.exception import ManifestParserError, UnknownManifestError from platformio.project.helpers import is_platformio_project @@ -40,7 +40,7 @@ class ManifestFileType(object): @classmethod def items(cls): - return get_object_members(ManifestFileType) + return get_object_members(cls) @classmethod def from_uri(cls, uri): @@ -60,8 +60,14 @@ def from_dir(cls, path): class ManifestParserFactory(object): @staticmethod def read_manifest_contents(path): - with io.open(path, encoding="utf-8") as fp: - return fp.read() + last_err = None + for encoding in ("utf-8", "latin-1"): + try: + with io.open(path, encoding=encoding) as fp: + return fp.read() + except UnicodeDecodeError as e: + last_err = e + raise last_err # pylint: disable=raising-bad-type @classmethod def new_from_file(cls, path, remote_url=False): @@ -101,14 +107,26 @@ def new_from_dir(cls, path, remote_url=None): @staticmethod def new_from_url(remote_url): - r = requests.get(remote_url) - r.raise_for_status() + content = fetch_remote_content(remote_url) return ManifestParserFactory.new( - r.text, + content, ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON, remote_url, ) + @staticmethod + def new_from_archive(path): + assert path.endswith("tar.gz") + with tarfile.open(path, mode="r:gz") as tf: + for t in sorted(ManifestFileType.items().values()): + try: + return ManifestParserFactory.new( + tf.extractfile(t).read().decode(), t + ) + except KeyError: + pass + raise UnknownManifestError("Unknown manifest file type in %s archive" % path) + @staticmethod def new( # pylint: disable=redefined-builtin contents, type, remote_url=None, package_dir=None @@ -148,10 +166,27 @@ def as_dict(self): return self._data @staticmethod - def normalize_author(author): + def str_to_list(value, sep=",", lowercase=True): + if isinstance(value, string_types): + value = value.split(sep) + assert isinstance(value, list) + result = [] + for item in value: + item = item.strip() + if not item: + continue + if lowercase: + item = item.lower() + result.append(item) + return result + + @staticmethod + def cleanup_author(author): assert isinstance(author, dict) if author.get("email"): author["email"] = re.sub(r"\s+[aA][tT]\s+", "@", author["email"]) + if "@" not in author["email"]: + author["email"] = None for key in list(author.keys()): if author[key] is None: del author[key] @@ -163,10 +198,13 @@ def parse_author_name_and_email(raw): return (None, None) name = raw email = None - for ldel, rdel in [("<", ">"), ("(", ")")]: - if ldel in raw and rdel in raw: - name = raw[: raw.index(ldel)] - email = raw[raw.index(ldel) + 1 : raw.index(rdel)] + ldel = "<" + rdel = ">" + if ldel in raw and rdel in raw: + name = raw[: raw.index(ldel)] + email = raw[raw.index(ldel) + 1 : raw.index(rdel)] + if "(" in name: + name = name.split("(")[0] return (name.strip(), email.strip() if email else None) @staticmethod @@ -284,7 +322,7 @@ def parse(self, contents): # normalize Union[str, list] fields for k in ("keywords", "platforms", "frameworks"): if k in data: - data[k] = self._str_to_list(data[k], sep=",") + data[k] = self.str_to_list(data[k], sep=",") if "authors" in data: data["authors"] = self._parse_authors(data["authors"]) @@ -297,21 +335,6 @@ def parse(self, contents): return data - @staticmethod - def _str_to_list(value, sep=",", lowercase=True): - if isinstance(value, string_types): - value = value.split(sep) - assert isinstance(value, list) - result = [] - for item in value: - item = item.strip() - if not item: - continue - if lowercase: - item = item.lower() - result.append(item) - return result - @staticmethod def _process_renamed_fields(data): if "url" in data: @@ -334,7 +357,7 @@ def _parse_authors(self, raw): # normalize Union[dict, list] fields if not isinstance(raw, list): raw = [raw] - return [self.normalize_author(author) for author in raw] + return [self.cleanup_author(author) for author in raw] @staticmethod def _parse_platforms(raw): @@ -372,8 +395,6 @@ def _parse_dependencies(raw): for k, v in dependency.items(): if k not in ("platforms", "frameworks", "authors"): continue - if "*" in v: - del raw[i][k] raw[i][k] = util.items_to_list(v) else: raw[i] = {"name": dependency} @@ -399,6 +420,8 @@ def parse(self, contents): del data["licenses"] if "dependencies" in data: data["dependencies"] = self._parse_dependencies(data["dependencies"]) + if "keywords" in data: + data["keywords"] = self.str_to_list(data["keywords"], sep=",") return data def _parse_authors(self, raw): @@ -409,7 +432,7 @@ def _parse_authors(self, raw): name, email = self.parse_author_name_and_email(author) if not name: continue - result.append(self.normalize_author(dict(name=name, email=email))) + result.append(self.cleanup_author(dict(name=name, email=email))) return result @staticmethod @@ -450,7 +473,9 @@ def parse(self, contents): ) if "author" in data: data["authors"] = self._parse_authors(data) - del data["author"] + for key in ("author", "maintainer"): + if key in data: + del data[key] if "depends" in data: data["dependencies"] = self._parse_dependencies(data["depends"]) return data @@ -466,6 +491,8 @@ def _parse_properties(contents): if line.startswith("#"): continue key, value = line.split("=", 1) + if not value.strip(): + continue data[key.strip()] = value.strip() return data @@ -521,7 +548,7 @@ def _parse_authors(self, properties): name, email = self.parse_author_name_and_email(author) if not name: continue - authors.append(self.normalize_author(dict(name=name, email=email))) + authors.append(self.cleanup_author(dict(name=name, email=email))) for author in properties.get("maintainer", "").split(","): name, email = self.parse_author_name_and_email(author) if not name: @@ -532,11 +559,11 @@ def _parse_authors(self, properties): continue found = True item["maintainer"] = True - if not item.get("email") and email: + if not item.get("email") and email and "@" in email: item["email"] = email if not found: authors.append( - self.normalize_author(dict(name=name, email=email, maintainer=True)) + self.cleanup_author(dict(name=name, email=email, maintainer=True)) ) return authors @@ -605,6 +632,8 @@ class PlatformJsonManifestParser(BaseManifestParser): def parse(self, contents): data = json.loads(contents) + if "keywords" in data: + data["keywords"] = self.str_to_list(data["keywords"], sep=",") if "frameworks" in data: data["frameworks"] = self._parse_frameworks(data["frameworks"]) if "packages" in data: @@ -629,8 +658,11 @@ class PackageJsonManifestParser(BaseManifestParser): def parse(self, contents): data = json.loads(contents) + if "keywords" in data: + data["keywords"] = self.str_to_list(data["keywords"], sep=",") data = self._parse_system(data) data = self._parse_homepage(data) + data = self._parse_repository(data) return data @staticmethod @@ -651,3 +683,14 @@ def _parse_homepage(data): data["homepage"] = data["url"] del data["url"] return data + + @staticmethod + def _parse_repository(data): + if isinstance(data.get("repository", {}), dict): + return data + data["repository"] = dict(type="git", url=str(data["repository"])) + if data["repository"]["url"].startswith(("github:", "gitlab:", "bitbucket:")): + data["repository"]["url"] = "https://{0}.com/{1}".format( + *(data["repository"]["url"].split(":", 1)) + ) + return data diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 11d3f902f4..39327f4a55 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -14,11 +14,14 @@ # pylint: disable=too-many-ancestors +import json + import marshmallow import requests import semantic_version from marshmallow import Schema, ValidationError, fields, validate, validates +from platformio.clients.http import fetch_remote_content from platformio.package.exception import ManifestValidationError from platformio.util import memoized @@ -84,7 +87,7 @@ def _deserialize( # pylint: disable=arguments-differ class AuthorSchema(StrictSchema): - name = fields.Str(required=True, validate=validate.Length(min=1, max=50)) + name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) email = fields.Email(validate=validate.Length(min=1, max=50)) maintainer = fields.Bool(default=False) url = fields.Url(validate=validate.Length(min=1, max=255)) @@ -149,7 +152,15 @@ class ExampleSchema(StrictSchema): class ManifestSchema(BaseSchema): # Required fields - name = fields.Str(required=True, validate=validate.Length(min=1, max=100)) + name = fields.Str( + required=True, + validate=[ + validate.Length(min=1, max=100), + validate.Regexp( + r"^[^:;/,@\<\>]+$", error="The next chars [:;/,@<>] are not allowed" + ), + ], + ) version = fields.Str(required=True, validate=validate.Length(min=1, max=50)) # Optional fields @@ -240,9 +251,9 @@ def validate_license(self, value): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - r = requests.get( + version = "3.10" + spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data" - "/v3.9/json/licenses.json" + "/v%s/json/licenses.json" % version ) - r.raise_for_status() - return r.json() + return json.loads(fetch_remote_content(spdx_data_url)) diff --git a/platformio/package/meta.py b/platformio/package/meta.py new file mode 100644 index 0000000000..147a1faf2d --- /dev/null +++ b/platformio/package/meta.py @@ -0,0 +1,433 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import tarfile +from binascii import crc32 + +import semantic_version + +from platformio.compat import get_object_members, hashlib_encode_data, string_types +from platformio.package.manifest.parser import ManifestFileType +from platformio.package.version import cast_version_to_semver + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +class PackageType(object): + LIBRARY = "library" + PLATFORM = "platform" + TOOL = "tool" + + @classmethod + def items(cls): + return get_object_members(cls) + + @classmethod + def get_manifest_map(cls): + return { + cls.PLATFORM: (ManifestFileType.PLATFORM_JSON,), + cls.LIBRARY: ( + ManifestFileType.LIBRARY_JSON, + ManifestFileType.LIBRARY_PROPERTIES, + ManifestFileType.MODULE_JSON, + ), + cls.TOOL: (ManifestFileType.PACKAGE_JSON,), + } + + @classmethod + def from_archive(cls, path): + assert path.endswith("tar.gz") + manifest_map = cls.get_manifest_map() + with tarfile.open(path, mode="r:gz") as tf: + for t in sorted(cls.items().values()): + for manifest in manifest_map[t]: + try: + if tf.getmember(manifest): + return t + except KeyError: + pass + return None + + +class PackageOutdatedResult(object): + def __init__(self, current, latest=None, wanted=None, detached=False): + self.current = current + self.latest = latest + self.wanted = wanted + self.detached = detached + + def __repr__(self): + return ( + "PackageOutdatedResult ".format( + current=self.current, + latest=self.latest, + wanted=self.wanted, + detached=self.detached, + ) + ) + + def __setattr__(self, name, value): + if ( + value + and name in ("current", "latest", "wanted") + and not isinstance(value, semantic_version.Version) + ): + value = cast_version_to_semver(str(value)) + return super(PackageOutdatedResult, self).__setattr__(name, value) + + def is_outdated(self, allow_incompatible=False): + if self.detached or not self.latest or self.current == self.latest: + return False + if allow_incompatible: + return self.current != self.latest + if self.wanted: + return self.current != self.wanted + return True + + +class PackageSpec(object): # pylint: disable=too-many-instance-attributes + def __init__( # pylint: disable=redefined-builtin,too-many-arguments + self, raw=None, owner=None, id=None, name=None, requirements=None, url=None + ): + self.owner = owner + self.id = id + self.name = name + self._requirements = None + self.url = url + self.raw = raw + if requirements: + self.requirements = requirements + self._name_is_custom = False + self._parse(raw) + + def __eq__(self, other): + return all( + [ + self.owner == other.owner, + self.id == other.id, + self.name == other.name, + self.requirements == other.requirements, + self.url == other.url, + ] + ) + + def __hash__(self): + return crc32( + hashlib_encode_data( + "%s-%s-%s-%s-%s" + % (self.owner, self.id, self.name, self.requirements, self.url) + ) + ) + + def __repr__(self): + return ( + "PackageSpec ".format(**self.as_dict()) + ) + + @property + def external(self): + return bool(self.url) + + @property + def requirements(self): + return self._requirements + + @requirements.setter + def requirements(self, value): + if not value: + self._requirements = None + return + self._requirements = ( + value + if isinstance(value, semantic_version.SimpleSpec) + else semantic_version.SimpleSpec(str(value)) + ) + + def humanize(self): + result = "" + if self.url: + result = self.url + elif self.name: + if self.owner: + result = self.owner + "/" + result += self.name + elif self.id: + result = "id:%d" % self.id + if self.requirements: + result += " @ " + str(self.requirements) + return result + + def has_custom_name(self): + return self._name_is_custom + + def as_dict(self): + return dict( + owner=self.owner, + id=self.id, + name=self.name, + requirements=str(self.requirements) if self.requirements else None, + url=self.url, + ) + + def as_dependency(self): + if self.url: + return self.raw or self.url + result = "" + if self.name: + result = "%s/%s" % (self.owner, self.name) if self.owner else self.name + elif self.id: + result = str(self.id) + assert result + if self.requirements: + result = "%s@%s" % (result, self.requirements) + return result + + def _parse(self, raw): + if raw is None: + return + if not isinstance(raw, string_types): + raw = str(raw) + raw = raw.strip() + + parsers = ( + self._parse_requirements, + self._parse_custom_name, + self._parse_id, + self._parse_owner, + self._parse_url, + ) + for parser in parsers: + if raw is None: + break + raw = parser(raw) + + # if name is not custom, parse it from URL + if not self.name and self.url: + self.name = self._parse_name_from_url(self.url) + elif raw: + # the leftover is a package name + self.name = raw + + def _parse_requirements(self, raw): + if "@" not in raw: + return raw + if raw.startswith("file://") and os.path.exists(raw[7:]): + return raw + tokens = raw.rsplit("@", 1) + if any(s in tokens[1] for s in (":", "/")): + return raw + self.requirements = tokens[1].strip() + return tokens[0].strip() + + def _parse_custom_name(self, raw): + if "=" not in raw or raw.startswith("id="): + return raw + tokens = raw.split("=", 1) + if "/" in tokens[0]: + return raw + self.name = tokens[0].strip() + self._name_is_custom = True + return tokens[1].strip() + + def _parse_id(self, raw): + if raw.isdigit(): + self.id = int(raw) + return None + if raw.startswith("id="): + return self._parse_id(raw[3:]) + return raw + + def _parse_owner(self, raw): + if raw.count("/") != 1 or "@" in raw: + return raw + tokens = raw.split("/", 1) + self.owner = tokens[0].strip() + self.name = tokens[1].strip() + return None + + def _parse_url(self, raw): + if not any(s in raw for s in ("@", ":", "/")): + return raw + self.url = raw.strip() + parts = urlparse(self.url) + + # if local file or valid URL with scheme vcs+protocol:// + if parts.scheme == "file" or "+" in parts.scheme or self.url.startswith("git+"): + return None + + # parse VCS + git_conditions = [ + parts.path.endswith(".git"), + # Handle GitHub URL (https://github.com/user/package) + parts.netloc in ("github.com", "gitlab.com", "bitbucket.com") + and not parts.path.endswith((".zip", ".tar.gz")), + ] + hg_conditions = [ + # Handle Developer Mbed URL + # (https://developer.mbed.org/users/user/code/package/) + # (https://os.mbed.com/users/user/code/package/) + parts.netloc + in ("mbed.com", "os.mbed.com", "developer.mbed.org") + ] + if any(git_conditions): + self.url = "git+" + self.url + elif any(hg_conditions): + self.url = "hg+" + self.url + + return None + + @staticmethod + def _parse_name_from_url(url): + if url.endswith("/"): + url = url[:-1] + stop_chars = ["#", "?"] + if url.startswith("file://"): + stop_chars.append("@") # detached path + for c in stop_chars: + if c in url: + url = url[: url.index(c)] + + # parse real repository name from Github + parts = urlparse(url) + if parts.netloc == "github.com" and parts.path.count("/") > 2: + return parts.path.split("/")[2] + + name = os.path.basename(url) + if "." in name: + return name.split(".", 1)[0].strip() + return name + + +class PackageMetaData(object): + def __init__( # pylint: disable=redefined-builtin + self, type, name, version, spec=None + ): + # assert type in PackageType.items().values() + if spec: + assert isinstance(spec, PackageSpec) + self.type = type + self.name = name + self._version = None + self.version = version + self.spec = spec + + def __repr__(self): + return ( + "PackageMetaData http://bit.ly/faq-package-manager" + ) + + +class BaseArchiver(object): def __init__(self, arhfileobj): self._afo = arhfileobj @@ -46,9 +55,9 @@ def close(self): self._afo.close() -class TARArchive(ArchiveBase): +class TARArchiver(BaseArchiver): def __init__(self, archpath): - super(TARArchive, self).__init__(tarfile_open(archpath)) + super(TARArchiver, self).__init__(tarfile_open(archpath)) def get_items(self): return self._afo.getmembers() @@ -79,7 +88,7 @@ def extract_item(self, item, dest_dir): self.is_link(item) and self.is_bad_link(item, dest_dir), ] if not any(bad_conds): - super(TARArchive, self).extract_item(item, dest_dir) + super(TARArchiver, self).extract_item(item, dest_dir) else: click.secho( "Blocked insecure item `%s` from TAR archive" % item.name, @@ -88,9 +97,9 @@ def extract_item(self, item, dest_dir): ) -class ZIPArchive(ArchiveBase): +class ZIPArchiver(BaseArchiver): def __init__(self, archpath): - super(ZIPArchive, self).__init__(ZipFile(archpath)) + super(ZIPArchiver, self).__init__(ZipFile(archpath)) @staticmethod def preserve_permissions(item, dest_dir): @@ -100,7 +109,7 @@ def preserve_permissions(item, dest_dir): @staticmethod def preserve_mtime(item, dest_dir): - util.change_filemtime( + fs.change_filemtime( os.path.join(dest_dir, item.filename), mktime(tuple(item.date_time) + tuple([0, 0, 0])), ) @@ -121,48 +130,59 @@ def after_extract(self, item, dest_dir): class FileUnpacker(object): - def __init__(self, archpath): - self.archpath = archpath - self._unpacker = None + def __init__(self, path): + self.path = path + self._archiver = None + + def _init_archiver(self): + magic_map = { + b"\x1f\x8b\x08": TARArchiver, + b"\x42\x5a\x68": TARArchiver, + b"\x50\x4b\x03\x04": ZIPArchiver, + } + magic_len = max(len(k) for k in magic_map) + with open(self.path, "rb") as fp: + data = fp.read(magic_len) + for magic, archiver in magic_map.items(): + if data.startswith(magic): + return archiver(self.path) + raise PackageException("Unknown archive type '%s'" % self.path) def __enter__(self): - if self.archpath.lower().endswith((".gz", ".bz2", ".tar")): - self._unpacker = TARArchive(self.archpath) - elif self.archpath.lower().endswith(".zip"): - self._unpacker = ZIPArchive(self.archpath) - if not self._unpacker: - raise exception.UnsupportedArchiveType(self.archpath) + self._archiver = self._init_archiver() return self def __exit__(self, *args): - if self._unpacker: - self._unpacker.close() + if self._archiver: + self._archiver.close() def unpack( - self, dest_dir=".", with_progress=True, check_unpacked=True, silent=False + self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False ): - assert self._unpacker + assert self._archiver + if not dest_dir: + dest_dir = os.getcwd() if not with_progress or silent: if not silent: click.echo("Unpacking...") - for item in self._unpacker.get_items(): - self._unpacker.extract_item(item, dest_dir) + for item in self._archiver.get_items(): + self._archiver.extract_item(item, dest_dir) else: - items = self._unpacker.get_items() + items = self._archiver.get_items() with click.progressbar(items, label="Unpacking") as pb: for item in pb: - self._unpacker.extract_item(item, dest_dir) + self._archiver.extract_item(item, dest_dir) if not check_unpacked: return True # check on disk - for item in self._unpacker.get_items(): - filename = self._unpacker.get_item_filename(item) + for item in self._archiver.get_items(): + filename = self._archiver.get_item_filename(item) item_path = os.path.join(dest_dir, filename) try: - if not self._unpacker.is_link(item) and not os.path.exists(item_path): - raise exception.ExtractArchiveItemError(filename, dest_dir) + if not self._archiver.is_link(item) and not os.path.exists(item_path): + raise ExtractArchiveItemError(filename, dest_dir) except NotImplementedError: pass return True diff --git a/platformio/vcsclient.py b/platformio/package/vcsclient.py similarity index 89% rename from platformio/vcsclient.py rename to platformio/package/vcsclient.py index 5629196628..2e9bb23889 100644 --- a/platformio/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -17,7 +17,11 @@ from subprocess import CalledProcessError, check_call from sys import modules -from platformio.exception import PlatformioException, UserSideException +from platformio.package.exception import ( + PackageException, + PlatformioException, + UserSideException, +) from platformio.proc import exec_command try: @@ -26,9 +30,13 @@ from urlparse import urlparse +class VCSBaseException(PackageException): + pass + + class VCSClientFactory(object): @staticmethod - def newClient(src_dir, remote_url, silent=False): + def new(src_dir, remote_url, silent=False): result = urlparse(remote_url) type_ = result.scheme tag = None @@ -41,12 +49,15 @@ def newClient(src_dir, remote_url, silent=False): if "#" in remote_url: remote_url, tag = remote_url.rsplit("#", 1) if not type_: - raise PlatformioException("VCS: Unknown repository type %s" % remote_url) - obj = getattr(modules[__name__], "%sClient" % type_.title())( - src_dir, remote_url, tag, silent - ) - assert isinstance(obj, VCSClientBase) - return obj + raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) + try: + obj = getattr(modules[__name__], "%sClient" % type_.title())( + src_dir, remote_url, tag, silent + ) + assert isinstance(obj, VCSClientBase) + return obj + except (AttributeError, AssertionError): + raise VCSBaseException("VCS: Unknown repository type %s" % remote_url) class VCSClientBase(object): @@ -101,7 +112,7 @@ def run_cmd(self, args, **kwargs): check_call(args, **kwargs) return True except CalledProcessError as e: - raise PlatformioException("VCS: Could not process command %s" % e.cmd) + raise VCSBaseException("VCS: Could not process command %s" % e.cmd) def get_cmd_output(self, args, **kwargs): args = [self.command] + args @@ -110,7 +121,7 @@ def get_cmd_output(self, args, **kwargs): result = exec_command(args, **kwargs) if result["returncode"] == 0: return result["out"].strip() - raise PlatformioException( + raise VCSBaseException( "VCS: Could not receive an output from `%s` command (%s)" % (args, result) ) @@ -227,7 +238,6 @@ def export(self): return self.run_cmd(args) def update(self): - args = ["update"] return self.run_cmd(args) @@ -239,4 +249,4 @@ def get_current_revision(self): line = line.strip() if line.startswith("Revision:"): return line.split(":", 1)[1].strip() - raise PlatformioException("Could not detect current SVN revision") + raise VCSBaseException("Could not detect current SVN revision") diff --git a/platformio/package/version.py b/platformio/package/version.py new file mode 100644 index 0000000000..770be9e4b2 --- /dev/null +++ b/platformio/package/version.py @@ -0,0 +1,53 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import semantic_version + + +def cast_version_to_semver(value, force=True, raise_exception=False): + assert value + try: + return semantic_version.Version(value) + except ValueError: + pass + if force: + try: + return semantic_version.Version.coerce(value) + except ValueError: + pass + if raise_exception: + raise ValueError("Invalid SemVer version %s" % value) + # parse commit hash + if re.match(r"^[\da-f]+$", value, flags=re.I): + return semantic_version.Version("0.0.0+sha." + value) + return semantic_version.Version("0.0.0+" + value) + + +def pepver_to_semver(pepver): + return cast_version_to_semver( + re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) + ) + + +def get_original_version(version): + if version.count(".") != 2: + return None + _, raw = version.split(".")[:2] + if int(raw) <= 99: + return None + if int(raw) <= 9999: + return "%s.%s" % (raw[:-2], int(raw[-2:])) + return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) diff --git a/platformio/platform/__init__.py b/platformio/platform/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/platformio/platform/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py new file mode 100644 index 0000000000..ac495b4827 --- /dev/null +++ b/platformio/platform/_packages.py @@ -0,0 +1,137 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.exception import UnknownPackageError +from platformio.package.meta import PackageSpec + + +class PlatformPackagesMixin(object): + def get_package_spec(self, name): + version = self.packages[name].get("version", "") + if any(c in version for c in (":", "/", "@")): + return PackageSpec("%s=%s" % (name, version)) + return PackageSpec( + owner=self.packages[name].get("owner"), name=name, requirements=version + ) + + def get_package(self, name): + if not name: + return None + return self.pm.get_package(self.get_package_spec(name)) + + def get_package_dir(self, name): + pkg = self.get_package(name) + return pkg.path if pkg else None + + def get_package_version(self, name): + pkg = self.get_package(name) + return str(pkg.metadata.version) if pkg else None + + def get_installed_packages(self): + result = [] + for name in self.packages: + pkg = self.get_package(name) + if pkg: + result.append(pkg) + return result + + def dump_used_packages(self): + result = [] + for name, options in self.packages.items(): + if options.get("optional"): + continue + pkg = self.get_package(name) + if not pkg or not pkg.metadata: + continue + item = {"name": pkg.metadata.name, "version": str(pkg.metadata.version)} + if pkg.metadata.spec.external: + item["src_url"] = pkg.metadata.spec.url + result.append(item) + return result + + def autoinstall_runtime_packages(self): + for name, options in self.packages.items(): + if options.get("optional", False): + continue + if self.get_package(name): + continue + self.pm.install(self.get_package_spec(name)) + return True + + def install_packages( # pylint: disable=too-many-arguments + self, + with_packages=None, + without_packages=None, + skip_default_package=False, + silent=False, + force=False, + ): + with_packages = set(self._find_pkg_names(with_packages or [])) + without_packages = set(self._find_pkg_names(without_packages or [])) + + upkgs = with_packages | without_packages + ppkgs = set(self.packages) + if not upkgs.issubset(ppkgs): + raise UnknownPackageError(", ".join(upkgs - ppkgs)) + + for name, options in self.packages.items(): + if name in without_packages: + continue + if name in with_packages or not ( + skip_default_package or options.get("optional", False) + ): + self.pm.install(self.get_package_spec(name), silent=silent, force=force) + + return True + + def _find_pkg_names(self, candidates): + result = [] + for candidate in candidates: + found = False + + # lookup by package types + for _name, _opts in self.packages.items(): + if _opts.get("type") == candidate: + result.append(_name) + found = True + + if ( + self.frameworks + and candidate.startswith("framework-") + and candidate[10:] in self.frameworks + ): + result.append(self.frameworks[candidate[10:]]["package"]) + found = True + + if not found: + result.append(candidate) + + return result + + def update_packages(self, only_check=False): + for pkg in self.get_installed_packages(): + self.pm.update( + pkg, + to_spec=self.get_package_spec(pkg.metadata.name), + only_check=only_check, + show_incompatible=False, + ) + + def are_outdated_packages(self): + for pkg in self.get_installed_packages(): + if self.pm.outdated( + pkg, self.get_package_spec(pkg.metadata.name) + ).is_outdated(allow_incompatible=False): + return True + return False diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py new file mode 100644 index 0000000000..457983c49b --- /dev/null +++ b/platformio/platform/_run.py @@ -0,0 +1,199 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import re +import sys + +import click + +from platformio import app, fs, proc, telemetry +from platformio.compat import PY2, hashlib_encode_data, is_bytes +from platformio.package.manager.core import get_core_package_dir +from platformio.platform.exception import BuildScriptNotFound + +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + + +class PlatformRunMixin(object): + + LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) + + @staticmethod + def encode_scons_arg(value): + data = base64.urlsafe_b64encode(hashlib_encode_data(value)) + return data.decode() if is_bytes(data) else data + + @staticmethod + def decode_scons_arg(data): + value = base64.urlsafe_b64decode(data) + return value.decode() if is_bytes(value) else value + + def run( # pylint: disable=too-many-arguments + self, variables, targets, silent, verbose, jobs + ): + assert isinstance(variables, dict) + assert isinstance(targets, list) + + self.ensure_engine_compatible() + + options = self.config.items(env=variables["pioenv"], as_dict=True) + if "framework" in options: + # support PIO Core 3.0 dev/platforms + options["pioframework"] = options["framework"] + self.configure_default_packages(options, targets) + self.autoinstall_runtime_packages() + + self._report_non_sensitive_data(options, targets) + + self.silent = silent + self.verbose = verbose or app.get_setting("force_verbose") + + if "clean" in targets: + targets = ["-c", "."] + + variables["platform_manifest"] = self.manifest_path + + if "build_script" not in variables: + variables["build_script"] = self.get_build_script() + if not os.path.isfile(variables["build_script"]): + raise BuildScriptNotFound(variables["build_script"]) + + result = self._run_scons(variables, targets, jobs) + assert "returncode" in result + + return result + + def _report_non_sensitive_data(self, options, targets): + topts = options.copy() + topts["platform_packages"] = [ + dict(name=item["name"], version=item["version"]) + for item in self.dump_used_packages() + ] + topts["platform"] = {"name": self.name, "version": self.version} + telemetry.send_run_environment(topts, targets) + + def _run_scons(self, variables, targets, jobs): + scons_dir = get_core_package_dir("tool-scons") + script_path = ( + os.path.join(scons_dir, "script", "scons") + if PY2 + else os.path.join(scons_dir, "scons.py") + ) + args = [ + proc.get_pythonexe_path(), + script_path, + "-Q", + "--warn=no-no-parallel-support", + "--jobs", + str(jobs), + "--sconstruct", + os.path.join(fs.get_source_dir(), "builder", "main.py"), + ] + args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + # pylint: disable=protected-access + args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) + args += targets + + # encode and append variables + for key, value in variables.items(): + args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) + + proc.copy_pythonpath_to_osenv() + + if targets and "menuconfig" in targets: + return proc.exec_command( + args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin + ) + + if click._compat.isatty(sys.stdout): + + def _write_and_flush(stream, data): + try: + stream.write(data) + stream.flush() + except IOError: + pass + + return proc.exec_command( + args, + stdout=proc.BuildAsyncPipe( + line_callback=self._on_stdout_line, + data_callback=lambda data: _write_and_flush(sys.stdout, data), + ), + stderr=proc.BuildAsyncPipe( + line_callback=self._on_stderr_line, + data_callback=lambda data: _write_and_flush(sys.stderr, data), + ), + ) + + return proc.exec_command( + args, + stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), + stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), + ) + + def _on_stdout_line(self, line): + if "`buildprog' is up to date." in line: + return + self._echo_line(line, level=1) + + def _on_stderr_line(self, line): + is_error = self.LINE_ERROR_RE.search(line) is not None + self._echo_line(line, level=3 if is_error else 2) + + a_pos = line.find("fatal error:") + b_pos = line.rfind(": No such file or directory") + if a_pos == -1 or b_pos == -1: + return + self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) + + def _echo_line(self, line, level): + if line.startswith("scons: "): + line = line[7:] + assert 1 <= level <= 3 + if self.silent and (level < 2 or not line): + return + fg = (None, "yellow", "red")[level - 1] + if level == 1 and "is up to date" in line: + fg = "green" + click.secho(line, fg=fg, err=level > 1, nl=False) + + @staticmethod + def _echo_missed_dependency(filename): + if "/" in filename or not filename.endswith((".h", ".hpp")): + return + banner = """ +{dots} +* Looking for {filename_styled} dependency? Check our library registry! +* +* CLI > platformio lib search "header:{filename}" +* Web > {link} +* +{dots} +""".format( + filename=filename, + filename_styled=click.style(filename, fg="cyan"), + link=click.style( + "https://platformio.org/lib/search?query=header:%s" + % quote(filename, safe=""), + fg="blue", + ), + dots="*" * (56 + len(filename)), + ) + click.echo(banner, err=True) diff --git a/platformio/platform/base.py b/platformio/platform/base.py new file mode 100644 index 0000000000..b29a9d7b82 --- /dev/null +++ b/platformio/platform/base.py @@ -0,0 +1,264 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +import click +import semantic_version + +from platformio import __version__, fs, proc +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.version import pepver_to_semver +from platformio.platform._packages import PlatformPackagesMixin +from platformio.platform._run import PlatformRunMixin +from platformio.platform.board import PlatformBoardConfig +from platformio.platform.exception import IncompatiblePlatform, UnknownBoard +from platformio.project.config import ProjectConfig + + +class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods + PlatformPackagesMixin, PlatformRunMixin +): + + CORE_SEMVER = pepver_to_semver(__version__) + _BOARDS_CACHE = {} + + def __init__(self, manifest_path): + self.manifest_path = manifest_path + self.silent = False + self.verbose = False + + self._manifest = fs.load_json(manifest_path) + self._BOARDS_CACHE = {} + self._custom_packages = None + + self.config = ProjectConfig.get_instance() + self.pm = ToolPackageManager(self.config.get_optional_dir("packages")) + + @property + def name(self): + return self._manifest["name"] + + @property + def title(self): + return self._manifest["title"] + + @property + def description(self): + return self._manifest["description"] + + @property + def version(self): + return self._manifest["version"] + + @property + def homepage(self): + return self._manifest.get("homepage") + + @property + def repository_url(self): + return self._manifest.get("repository", {}).get("url") + + @property + def license(self): + return self._manifest.get("license") + + @property + def frameworks(self): + return self._manifest.get("frameworks") + + @property + def engines(self): + return self._manifest.get("engines") + + @property + def manifest(self): + return self._manifest + + @property + def packages(self): + packages = self._manifest.get("packages", {}) + for item in self._custom_packages or []: + name = item + version = "*" + if "@" in item: + name, version = item.split("@", 2) + spec = self.pm.ensure_spec(name) + options = {"version": version.strip(), "optional": False} + if spec.owner: + options["owner"] = spec.owner + if spec.name not in packages: + packages[spec.name] = {} + packages[spec.name].update(**options) + return packages + + @property + def python_packages(self): + return self._manifest.get("pythonPackages") + + def ensure_engine_compatible(self): + if not self.engines or "platformio" not in self.engines: + return True + core_spec = semantic_version.SimpleSpec(self.engines["platformio"]) + if self.CORE_SEMVER in core_spec: + return True + # PIO Core 5 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0 + if any(semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3, 4)): + return True + raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER), str(core_spec)) + + def get_dir(self): + return os.path.dirname(self.manifest_path) + + def get_build_script(self): + main_script = os.path.join(self.get_dir(), "builder", "main.py") + if os.path.isfile(main_script): + return main_script + raise NotImplementedError() + + def is_embedded(self): + for opts in self.packages.values(): + if opts.get("type") == "uploader": + return True + return False + + def get_boards(self, id_=None): + def _append_board(board_id, manifest_path): + config = PlatformBoardConfig(manifest_path) + if "platform" in config and config.get("platform") != self.name: + return + if "platforms" in config and self.name not in config.get("platforms"): + return + config.manifest["platform"] = self.name + self._BOARDS_CACHE[board_id] = config + + bdirs = [ + self.config.get_optional_dir("boards"), + os.path.join(self.config.get_optional_dir("core"), "boards"), + os.path.join(self.get_dir(), "boards"), + ] + + if id_ is None: + for boards_dir in bdirs: + if not os.path.isdir(boards_dir): + continue + for item in sorted(os.listdir(boards_dir)): + _id = item[:-5] + if not item.endswith(".json") or _id in self._BOARDS_CACHE: + continue + _append_board(_id, os.path.join(boards_dir, item)) + else: + if id_ not in self._BOARDS_CACHE: + for boards_dir in bdirs: + if not os.path.isdir(boards_dir): + continue + manifest_path = os.path.join(boards_dir, "%s.json" % id_) + if os.path.isfile(manifest_path): + _append_board(id_, manifest_path) + break + if id_ not in self._BOARDS_CACHE: + raise UnknownBoard(id_) + return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE + + def board_config(self, id_): + return self.get_boards(id_) + + def get_package_type(self, name): + return self.packages[name].get("type") + + def configure_default_packages(self, options, targets): + # override user custom packages + self._custom_packages = options.get("platform_packages") + + # enable used frameworks + for framework in options.get("framework", []): + if not self.frameworks: + continue + framework = framework.lower().strip() + if not framework or framework not in self.frameworks: + continue + _pkg_name = self.frameworks[framework].get("package") + if _pkg_name: + self.packages[_pkg_name]["optional"] = False + + # enable upload tools for upload targets + if any(["upload" in t for t in targets] + ["program" in targets]): + for name, opts in self.packages.items(): + if opts.get("type") == "uploader": + self.packages[name]["optional"] = False + # skip all packages in "nobuild" mode + # allow only upload tools and frameworks + elif "nobuild" in targets and opts.get("type") != "framework": + self.packages[name]["optional"] = True + + def get_lib_storages(self): + storages = {} + for opts in (self.frameworks or {}).values(): + if "package" not in opts: + continue + pkg = self.get_package(opts["package"]) + if not pkg or not os.path.isdir(os.path.join(pkg.path, "libraries")): + continue + libs_dir = os.path.join(pkg.path, "libraries") + storages[libs_dir] = opts["package"] + libcores_dir = os.path.join(libs_dir, "__cores__") + if not os.path.isdir(libcores_dir): + continue + for item in os.listdir(libcores_dir): + libcore_dir = os.path.join(libcores_dir, item) + if not os.path.isdir(libcore_dir): + continue + storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) + + return [dict(name=name, path=path) for path, name in storages.items()] + + def on_installed(self): + pass + + def on_uninstalled(self): + pass + + def install_python_packages(self): + if not self.python_packages: + return None + click.echo( + "Installing Python packages: %s" + % ", ".join(list(self.python_packages.keys())), + ) + args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"] + for name, requirements in self.python_packages.items(): + if any(c in requirements for c in ("<", ">", "=")): + args.append("%s%s" % (name, requirements)) + else: + args.append("%s==%s" % (name, requirements)) + try: + return subprocess.call(args) == 0 + except Exception as e: # pylint: disable=broad-except + click.secho( + "Could not install Python packages -> %s" % e, fg="red", err=True + ) + + def uninstall_python_packages(self): + if not self.python_packages: + return + click.echo("Uninstalling Python packages") + args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"] + args.extend(list(self.python_packages.keys())) + try: + subprocess.call(args) == 0 + except Exception as e: # pylint: disable=broad-except + click.secho( + "Could not install Python packages -> %s" % e, fg="red", err=True + ) diff --git a/platformio/platform/board.py b/platformio/platform/board.py new file mode 100644 index 0000000000..900892cdf3 --- /dev/null +++ b/platformio/platform/board.py @@ -0,0 +1,158 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import fs, telemetry, util +from platformio.commands.debug.exception import ( + DebugInvalidOptionsError, + DebugSupportError, +) +from platformio.compat import PY2 +from platformio.exception import UserSideException +from platformio.platform.exception import InvalidBoardManifest + + +class PlatformBoardConfig(object): + def __init__(self, manifest_path): + self._id = os.path.basename(manifest_path)[:-5] + assert os.path.isfile(manifest_path) + self.manifest_path = manifest_path + try: + self._manifest = fs.load_json(manifest_path) + except ValueError: + raise InvalidBoardManifest(manifest_path) + if not set(["name", "url", "vendor"]) <= set(self._manifest): + raise UserSideException( + "Please specify name, url and vendor fields for " + manifest_path + ) + + def get(self, path, default=None): + try: + value = self._manifest + for k in path.split("."): + value = value[k] + # pylint: disable=undefined-variable + if PY2 and isinstance(value, unicode): + # cast to plain string from unicode for PY2, resolves issue in + # dev/platform when BoardConfig.get() is used in pair with + # os.path.join(file_encoding, unicode_encoding) + try: + value = value.encode("utf-8") + except UnicodeEncodeError: + pass + return value + except KeyError: + if default is not None: + return default + raise KeyError("Invalid board option '%s'" % path) + + def update(self, path, value): + newdict = None + for key in path.split(".")[::-1]: + if newdict is None: + newdict = {key: value} + else: + newdict = {key: newdict} + util.merge_dicts(self._manifest, newdict) + + def __contains__(self, key): + try: + self.get(key) + return True + except KeyError: + return False + + @property + def id(self): + return self._id + + @property + def id_(self): + return self.id + + @property + def manifest(self): + return self._manifest + + def get_brief_data(self): + result = { + "id": self.id, + "name": self._manifest["name"], + "platform": self._manifest.get("platform"), + "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), + "fcpu": int( + "".join( + [ + c + for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) + if c.isdigit() + ] + ) + ), + "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), + "rom": self._manifest.get("upload", {}).get("maximum_size", 0), + "frameworks": self._manifest.get("frameworks"), + "vendor": self._manifest["vendor"], + "url": self._manifest["url"], + } + if self._manifest.get("connectivity"): + result["connectivity"] = self._manifest.get("connectivity") + debug = self.get_debug_data() + if debug: + result["debug"] = debug + return result + + def get_debug_data(self): + if not self._manifest.get("debug", {}).get("tools"): + return None + tools = {} + for name, options in self._manifest["debug"]["tools"].items(): + tools[name] = {} + for key, value in options.items(): + if key in ("default", "onboard") and value: + tools[name][key] = value + return {"tools": tools} + + def get_debug_tool_name(self, custom=None): + debug_tools = self._manifest.get("debug", {}).get("tools") + tool_name = custom + if tool_name == "custom": + return tool_name + if not debug_tools: + telemetry.send_event("Debug", "Request", self.id) + raise DebugSupportError(self._manifest["name"]) + if tool_name: + if tool_name in debug_tools: + return tool_name + raise DebugInvalidOptionsError( + "Unknown debug tool `%s`. Please use one of `%s` or `custom`" + % (tool_name, ", ".join(sorted(list(debug_tools)))) + ) + + # automatically select best tool + data = {"default": [], "onboard": [], "external": []} + for key, value in debug_tools.items(): + if value.get("default"): + data["default"].append(key) + elif value.get("onboard"): + data["onboard"].append(key) + data["external"].append(key) + + for key, value in data.items(): + if not value: + continue + return sorted(value)[0] + + assert any(item for item in data) diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py new file mode 100644 index 0000000000..604c322876 --- /dev/null +++ b/platformio/platform/exception.py @@ -0,0 +1,52 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException + + +class PlatformException(PlatformioException): + pass + + +class UnknownPlatform(PlatformException): + + MESSAGE = "Unknown development platform '{0}'" + + +class IncompatiblePlatform(PlatformException): + + MESSAGE = ( + "Development platform '{0}' is not compatible with PlatformIO Core v{1} and " + "depends on PlatformIO Core {2}.\n" + ) + + +class UnknownBoard(PlatformException): + + MESSAGE = "Unknown board ID '{0}'" + + +class InvalidBoardManifest(PlatformException): + + MESSAGE = "Invalid board JSON manifest '{0}'" + + +class UnknownFramework(PlatformException): + + MESSAGE = "Unknown framework '{0}'" + + +class BuildScriptNotFound(PlatformException): + + MESSAGE = "Invalid path '{0}' to build script" diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py new file mode 100644 index 0000000000..0f2bd15f34 --- /dev/null +++ b/platformio/platform/factory.py @@ -0,0 +1,83 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +from platformio import fs +from platformio.compat import load_python_module +from platformio.package.meta import PackageItem +from platformio.platform.base import PlatformBase +from platformio.platform.exception import UnknownPlatform + + +class PlatformFactory(object): + @staticmethod + def get_clsname(name): + name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) + return "%s%sPlatform" % (name.upper()[0], name.lower()[1:]) + + @staticmethod + def load_module(name, path): + try: + return load_python_module("platformio.platform.%s" % name, path) + except ImportError: + raise UnknownPlatform(name) + + @classmethod + def new(cls, pkg_or_spec): + platform_dir = None + platform_name = None + if isinstance(pkg_or_spec, PackageItem): + platform_dir = pkg_or_spec.path + platform_name = pkg_or_spec.metadata.name + elif os.path.isdir(pkg_or_spec): + platform_dir = pkg_or_spec + else: + from platformio.package.manager.platform import ( # pylint: disable=import-outside-toplevel + PlatformPackageManager, + ) + + pkg = PlatformPackageManager().get_package(pkg_or_spec) + if not pkg: + raise UnknownPlatform(pkg_or_spec) + platform_dir = pkg.path + platform_name = pkg.metadata.name + + if not platform_dir or not os.path.isfile( + os.path.join(platform_dir, "platform.json") + ): + raise UnknownPlatform(pkg_or_spec) + + if not platform_name: + platform_name = fs.load_json(os.path.join(platform_dir, "platform.json"))[ + "name" + ] + + platform_cls = None + if os.path.isfile(os.path.join(platform_dir, "platform.py")): + platform_cls = getattr( + cls.load_module( + platform_name, os.path.join(platform_dir, "platform.py") + ), + cls.get_clsname(platform_name), + ) + else: + platform_cls = type( + str(cls.get_clsname(platform_name)), (PlatformBase,), {} + ) + + _instance = platform_cls(os.path.join(platform_dir, "platform.json")) + assert isinstance(_instance, PlatformBase) + return _instance diff --git a/platformio/proc.py b/platformio/proc.py index 80e50201a0..82f5a9cf2e 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -15,7 +15,7 @@ import os import subprocess import sys -from os.path import isdir, isfile, join, normpath +from contextlib import contextmanager from threading import Thread from platformio import exception @@ -138,23 +138,32 @@ def exec_command(*args, **kwargs): return result +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr + + def is_ci(): return os.getenv("CI", "").lower() == "true" def is_container(): - if not isfile("/proc/1/cgroup"): + if os.path.exists("/.dockerenv"): + return True + if not os.path.isfile("/proc/1/cgroup"): return False with open("/proc/1/cgroup") as fp: - for line in fp: - line = line.strip() - if ":" in line and not line.endswith(":/"): - return True - return False + return ":/docker/" in fp.read() def get_pythonexe_path(): - return os.environ.get("PYTHONEXEPATH", normpath(sys.executable)) + return os.environ.get("PYTHONEXEPATH", os.path.normpath(sys.executable)) def copy_pythonpath_to_osenv(): @@ -164,7 +173,10 @@ def copy_pythonpath_to_osenv(): for p in os.sys.path: conditions = [p not in _PYTHONPATH] if not WINDOWS: - conditions.append(isdir(join(p, "click")) or isdir(join(p, "platformio"))) + conditions.append( + os.path.isdir(os.path.join(p, "click")) + or os.path.isdir(os.path.join(p, "platformio")) + ) if all(conditions): _PYTHONPATH.append(p) os.environ["PYTHONPATH"] = os.pathsep.join(_PYTHONPATH) @@ -178,16 +190,16 @@ def where_is_program(program, envpath=None): # try OS's built-in commands try: result = exec_command(["where" if WINDOWS else "which", program], env=env) - if result["returncode"] == 0 and isfile(result["out"].strip()): + if result["returncode"] == 0 and os.path.isfile(result["out"].strip()): return result["out"].strip() except OSError: pass # look up in $PATH for bin_dir in env.get("PATH", "").split(os.pathsep): - if isfile(join(bin_dir, program)): - return join(bin_dir, program) - if isfile(join(bin_dir, "%s.exe" % program)): - return join(bin_dir, "%s.exe" % program) + if os.path.isfile(os.path.join(bin_dir, program)): + return os.path.join(bin_dir, program) + if os.path.isfile(os.path.join(bin_dir, "%s.exe" % program)): + return os.path.join(bin_dir, "%s.exe" % program) return program diff --git a/platformio/project/config.py b/platformio/project/config.py index 23d089bfdf..2d841b396e 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob import json import os import re @@ -21,7 +20,13 @@ import click from platformio import fs -from platformio.compat import PY2, WINDOWS, hashlib_encode_data, string_types +from platformio.compat import ( + PY2, + WINDOWS, + glob_recursive, + hashlib_encode_data, + string_types, +) from platformio.project import exception from platformio.project.options import ProjectOptions @@ -117,7 +122,7 @@ def read(self, path, parse_extra=True): for pattern in self.get("platformio", "extra_configs", []): if pattern.startswith("~"): pattern = fs.expanduser(pattern) - for item in glob.glob(pattern): + for item in glob_recursive(pattern): self.read(item) def _maintain_renaimed_options(self): @@ -353,6 +358,12 @@ def validate(self, envs=None, silent=False): click.secho("Warning! %s" % warning, fg="yellow") return True + def remove_option(self, section, option): + return self._parser.remove_option(section, option) + + def remove_section(self, section): + return self._parser.remove_section(section) + class ProjectConfigDirsMixin(object): def _get_core_dir(self, exists=False): diff --git a/platformio/project/options.py b/platformio/project/options.py index 3f0cf76c30..b5eaf337d2 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -245,7 +245,7 @@ def ConfigEnvOption(*args, **kwargs): group="directory", name="test_dir", description=( - "A location where PIO Unit Testing engine looks for " + "A location where PlatformIO Unit Testing engine looks for " "test source files" ), sysenvvar="PLATFORMIO_TEST_DIR", @@ -262,8 +262,8 @@ def ConfigEnvOption(*args, **kwargs): group="directory", name="shared_dir", description=( - "A location which PIO Remote uses to synchronize extra files " - "between remote machines" + "A location which PlatformIO Remote Development service uses to " + "synchronize extra files between remote machines" ), sysenvvar="PLATFORMIO_SHARED_DIR", default=os.path.join("$PROJECT_DIR", "shared"), diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 83c53c7f1d..3fbcb74f5d 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -123,9 +123,7 @@ def _filter_args(items): caller_id = str(app.get_session_var("caller_id")) self["cd1"] = util.get_systype() - self["cd4"] = ( - 1 if (not util.is_ci() and (caller_id or not is_container())) else 0 - ) + self["cd4"] = 1 if (not is_ci() and (caller_id or not is_container())) else 0 if caller_id: self["cd5"] = caller_id.lower() @@ -146,7 +144,18 @@ def _first_arg_from_list(args_, list_): return cmd_path = args[:1] - if args[0] in ("account", "device", "platform", "project", "settings",): + if args[0] in ( + "access", + "account", + "device", + "org", + "package", + "platform", + "project", + "settings", + "system", + "team", + ): cmd_path = args[:2] if args[0] == "lib" and len(args) > 1: lib_subcmds = ( diff --git a/platformio/util.py b/platformio/util.py index 6a664c4990..6da4708b83 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,31 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import absolute_import + import json import math import os import platform import re -import socket import sys import time -from contextlib import contextmanager from functools import wraps from glob import glob import click -import requests -from platformio import __apiurl__, __version__, exception -from platformio.commands import PlatformioCLI +from platformio import __version__, exception, proc from platformio.compat import PY2, WINDOWS -from platformio.fs import cd # pylint: disable=unused-import -from platformio.fs import load_json # pylint: disable=unused-import -from platformio.fs import rmtree as rmtree_ # pylint: disable=unused-import +from platformio.fs import cd, load_json # pylint: disable=unused-import from platformio.proc import exec_command # pylint: disable=unused-import -from platformio.proc import is_ci # pylint: disable=unused-import - -# KEEP unused imports for backward compatibility with PIO Core 3.0 API class memoized(object): @@ -95,17 +88,6 @@ def get_instance(*args, **kwargs): return get_instance -@contextmanager -def capture_std_streams(stdout, stderr=None): - _stdout = sys.stdout - _stderr = sys.stderr - sys.stdout = stdout - sys.stderr = stderr or stdout - yield - sys.stdout = _stdout - sys.stderr = _stderr - - def get_systype(): type_ = platform.system().lower() arch = platform.machine().lower() @@ -114,16 +96,6 @@ def get_systype(): return "%s_%s" % (type_, arch) if arch else type_ -def pioversion_to_intstr(): - vermatch = re.match(r"^([\d\.]+)", __version__) - assert vermatch - return [int(i) for i in vermatch.group(1).split(".")[:3]] - - -def change_filemtime(path, mtime): - os.utime(path, (mtime, mtime)) - - def get_serial_ports(filter_hwid=False): try: # pylint: disable=import-outside-toplevel @@ -162,7 +134,7 @@ def get_logical_devices(): items = [] if WINDOWS: try: - result = exec_command( + result = proc.exec_command( ["wmic", "logicaldisk", "get", "name,VolumeName"] ).get("out", "") devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?") @@ -175,12 +147,12 @@ def get_logical_devices(): except WindowsError: # pylint: disable=undefined-variable pass # try "fsutil" - result = exec_command(["fsutil", "fsinfo", "drives"]).get("out", "") + result = proc.exec_command(["fsutil", "fsinfo", "drives"]).get("out", "") for device in re.findall(r"[A-Z]:\\", result): items.append({"path": device, "name": None}) return items - result = exec_command(["df"]).get("out") + result = proc.exec_command(["df"]).get("out") devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) for line in result.split("\n"): match = devicenamere.match(line.strip()) @@ -196,7 +168,7 @@ def get_mdns_services(): import zeroconf except ImportError: from site import addsitedir - from platformio.managers.core import get_core_package_dir + from platformio.package.manager.core import get_core_package_dir contrib_pysite_dir = get_core_package_dir("contrib-pysite") addsitedir(contrib_pysite_dir) @@ -270,132 +242,11 @@ def get_services(self): return items -@memoized(expire="60s") -def _api_request_session(): - return requests.Session() - - -@throttle(500) -def _get_api_result( - url, params=None, data=None, auth=None # pylint: disable=too-many-branches -): - # pylint: disable=import-outside-toplevel - from platformio.app import get_user_agent, get_setting - - result = {} - r = None - verify_ssl = sys.version_info >= (2, 7, 9) - - if not url.startswith("http"): - url = __apiurl__ + url - if not get_setting("strict_ssl"): - url = url.replace("https://", "http://") - - headers = {"User-Agent": get_user_agent()} - try: - if data: - r = _api_request_session().post( - url, - params=params, - data=data, - headers=headers, - auth=auth, - verify=verify_ssl, - ) - else: - r = _api_request_session().get( - url, params=params, headers=headers, auth=auth, verify=verify_ssl - ) - result = r.json() - r.raise_for_status() - return r.text - except requests.exceptions.HTTPError as e: - if result and "message" in result: - raise exception.APIRequestError(result["message"]) - if result and "errors" in result: - raise exception.APIRequestError(result["errors"][0]["title"]) - raise exception.APIRequestError(e) - except ValueError: - raise exception.APIRequestError("Invalid response: %s" % r.text.encode("utf-8")) - finally: - if r: - r.close() - return None - - -def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): - from platformio.app import ContentCache # pylint: disable=import-outside-toplevel - - total = 0 - max_retries = 5 - cache_key = ( - ContentCache.key_from_args(url, params, data, auth) if cache_valid else None - ) - while total < max_retries: - try: - with ContentCache() as cc: - if cache_key: - result = cc.get(cache_key) - if result is not None: - return json.loads(result) - - # check internet before and resolve issue with 60 seconds timeout - internet_on(raise_exception=True) - - result = _get_api_result(url, params, data) - if cache_valid: - with ContentCache() as cc: - cc.set(cache_key, result, cache_valid) - return json.loads(result) - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - total += 1 - if not PlatformioCLI.in_silence(): - click.secho( - "[API] ConnectionError: {0} (incremented retry: max={1}, " - "total={2})".format(e, max_retries, total), - fg="yellow", - ) - time.sleep(2 * total) - - raise exception.APIRequestError( - "Could not connect to PlatformIO API Service. Please try later." - ) - - -PING_REMOTE_HOSTS = [ - "140.82.118.3", # Github.com - "35.231.145.151", # Gitlab.com - "88.198.170.159", # platformio.org - "github.com", - "platformio.org", -] - - -@memoized(expire="5s") -def _internet_on(): - timeout = 2 - socket.setdefaulttimeout(timeout) - for host in PING_REMOTE_HOSTS: - try: - if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): - requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) - else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, 80)) - return True - except: # pylint: disable=bare-except - pass - return False - - -def internet_on(raise_exception=False): - result = _internet_on() - if raise_exception and not result: - raise exception.InternetIsOffline() - return result - - -def pepver_to_semver(pepver): - return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) +def pioversion_to_intstr(): + """ Legacy for framework-zephyr/scripts/platformio/platformio-build-pre.py""" + vermatch = re.match(r"^([\d\.]+)", __version__) + assert vermatch + return [int(i) for i in vermatch.group(1).split(".")[:3]] def items_to_list(items): @@ -446,14 +297,3 @@ def humanize_duration_time(duration): tokens.append(int(round(duration) if multiplier == 1 else fraction)) duration -= fraction * multiplier return "{:02d}:{:02d}:{:02d}.{:03d}".format(*tokens) - - -def get_original_version(version): - if version.count(".") != 2: - return None - _, raw = version.split(".")[:2] - if int(raw) <= 99: - return None - if int(raw) <= 9999: - return "%s.%s" % (raw[:-2], int(raw[-2:])) - return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 3628f6092d..3698aa0bc7 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -22,7 +22,8 @@ import click from platformio import fs, util -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory try: from urlparse import ParseResult, urlparse, urlunparse @@ -41,15 +42,16 @@ limitations under the License. """ -API_PACKAGES = util.get_api_result("/packages") -API_FRAMEWORKS = util.get_api_result("/frameworks") -BOARDS = PlatformManager().get_installed_boards() -PLATFORM_MANIFESTS = PlatformManager().get_installed() +REGCLIENT = regclient = PlatformPackageManager().get_registry_client_instance() +API_PACKAGES = regclient.fetch_json_data("get", "/v2/packages") +API_FRAMEWORKS = regclient.fetch_json_data("get", "/v2/frameworks") +BOARDS = PlatformPackageManager().get_installed_boards() +PLATFORM_MANIFESTS = PlatformPackageManager().legacy_get_installed() DOCS_ROOT_DIR = realpath(join(dirname(realpath(__file__)), "..", "docs")) def is_compat_platform_and_framework(platform, framework): - p = PlatformFactory.newPlatform(platform) + p = PlatformFactory.new(platform) return framework in (p.frameworks or {}).keys() @@ -60,8 +62,10 @@ def campaign_url(url, source="platformio.org", medium="docs"): query += "&" query += "utm_source=%s&utm_medium=%s" % (source, medium) return urlunparse( - ParseResult(data.scheme, data.netloc, data.path, data.params, query, - data.fragment)) + ParseResult( + data.scheme, data.netloc, data.path, data.params, query, data.fragment + ) + ) def generate_boards_table(boards, skip_columns=None): @@ -75,10 +79,12 @@ def generate_boards_table(boards, skip_columns=None): ("RAM", "{ram}"), ] lines = [] - lines.append(""" + lines.append( + """ .. list-table:: :header-rows: 1 -""") +""" + ) # add header for (name, template) in columns: @@ -87,23 +93,26 @@ def generate_boards_table(boards, skip_columns=None): prefix = " * - " if name == "Name" else " - " lines.append(prefix + name) - for data in sorted(boards, key=lambda item: item['name']): - has_onboard_debug = (data['debug'] and any( - t.get("onboard") for (_, t) in data['debug']['tools'].items())) + for data in sorted(boards, key=lambda item: item["name"]): + has_onboard_debug = data.get("debug") and any( + t.get("onboard") for (_, t) in data["debug"]["tools"].items() + ) debug = "No" if has_onboard_debug: debug = "On-board" - elif data['debug']: + elif data.get("debug"): debug = "External" - variables = dict(id=data['id'], - name=data['name'], - platform=data['platform'], - debug=debug, - mcu=data['mcu'].upper(), - f_cpu=int(data['fcpu'] / 1000000.0), - ram=fs.format_filesize(data['ram']), - rom=fs.format_filesize(data['rom'])) + variables = dict( + id=data["id"], + name=data["name"], + platform=data["platform"], + debug=debug, + mcu=data["mcu"].upper(), + f_cpu=int(data["fcpu"] / 1000000.0), + ram=fs.humanize_file_size(data["ram"]), + rom=fs.humanize_file_size(data["rom"]), + ) for (name, template) in columns: if skip_columns and name in skip_columns: @@ -121,25 +130,30 @@ def generate_frameworks_contents(frameworks): if not frameworks: return [] lines = [] - lines.append(""" + lines.append( + """ Frameworks ---------- .. list-table:: :header-rows: 1 * - Name - - Description""") + - Description""" + ) known = set() for framework in API_FRAMEWORKS: - known.add(framework['name']) - if framework['name'] not in frameworks: + known.add(framework["name"]) + if framework["name"] not in frameworks: continue - lines.append(""" + lines.append( + """ * - :ref:`framework_{name}` - - {description}""".format(**framework)) + - {description}""".format( + **framework + ) + ) if set(frameworks) - known: - click.secho("Unknown frameworks %s " % ( - set(frameworks) - known), fg="red") + click.secho("Unknown frameworks %s " % (set(frameworks) - known), fg="red") return lines @@ -147,20 +161,26 @@ def generate_platforms_contents(platforms): if not platforms: return [] lines = [] - lines.append(""" + lines.append( + """ Platforms --------- .. list-table:: :header-rows: 1 * - Name - - Description""") + - Description""" + ) for name in sorted(platforms): - p = PlatformFactory.newPlatform(name) - lines.append(""" + p = PlatformFactory.new(name) + lines.append( + """ * - :ref:`platform_{name}` - - {description}""".format(name=p.name, description=p.description)) + - {description}""".format( + name=p.name, description=p.description + ) + ) return lines @@ -170,16 +190,17 @@ def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): skip_board_columns.append("Debug") lines = [] onboard_debug = [ - b for b in boards if b['debug'] and any( - t.get("onboard") for (_, t) in b['debug']['tools'].items()) - ] - external_debug = [ - b for b in boards if b['debug'] and b not in onboard_debug + b + for b in boards + if b.get("debug") + and any(t.get("onboard") for (_, t) in b["debug"]["tools"].items()) ] + external_debug = [b for b in boards if b.get("debug") and b not in onboard_debug] if not onboard_debug and not external_debug: return lines - lines.append(""" + lines.append( + """ Debugging --------- @@ -187,11 +208,13 @@ def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): .. contents:: :local: -""") +""" + ) if extra_rst: lines.append(".. include:: %s" % extra_rst) - lines.append(""" + lines.append( + """ Tools & Debug Probes ~~~~~~~~~~~~~~~~~~~~ @@ -203,31 +226,36 @@ def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): .. warning:: You will need to install debug tool drivers depending on your system. Please click on compatible debug tool below for the further instructions. -""") +""" + ) if onboard_debug: - lines.append(""" + lines.append( + """ On-Board Debug Tools ^^^^^^^^^^^^^^^^^^^^ Boards listed below have on-board debug probe and **ARE READY** for debugging! You do not need to use/buy external debug probe. -""") +""" + ) lines.extend( - generate_boards_table(onboard_debug, - skip_columns=skip_board_columns)) + generate_boards_table(onboard_debug, skip_columns=skip_board_columns) + ) if external_debug: - lines.append(""" + lines.append( + """ External Debug Tools ^^^^^^^^^^^^^^^^^^^^ Boards listed below are compatible with :ref:`piodebug` but **DEPEND ON** external debug probe. They **ARE NOT READY** for debugging. Please click on board name for the further details. -""") +""" + ) lines.extend( - generate_boards_table(external_debug, - skip_columns=skip_board_columns)) + generate_boards_table(external_debug, skip_columns=skip_board_columns) + ) return lines @@ -235,41 +263,56 @@ def generate_packages(platform, packagenames, is_embedded): if not packagenames: return lines = [] - lines.append(""" + lines.append( + """ Packages -------- -""") - lines.append(""".. list-table:: +""" + ) + lines.append( + """.. list-table:: :header-rows: 1 * - Name - - Description""") + - Description""" + ) for name in sorted(packagenames): if name not in API_PACKAGES: click.secho("Unknown package `%s`" % name, fg="red") - lines.append(""" + lines.append( + """ * - {name} - - """.format(name=name)) + """.format( + name=name + ) + ) else: - lines.append(""" + lines.append( + """ * - `{name} <{url}>`__ - - {description}""".format(name=name, - url=campaign_url(API_PACKAGES[name]['url']), - description=API_PACKAGES[name]['description'])) + - {description}""".format( + name=name, + url=campaign_url(API_PACKAGES[name]["url"]), + description=API_PACKAGES[name]["description"], + ) + ) if is_embedded: - lines.append(""" + lines.append( + """ .. warning:: **Linux Users**: * Install "udev" rules :ref:`faq_udev_rules` * Raspberry Pi users, please read this article `Enable serial port on Raspberry Pi `__. -""") +""" + ) if platform == "teensy": - lines.append(""" + lines.append( + """ **Windows Users:** Teensy programming uses only Windows built-in HID @@ -278,14 +321,17 @@ def generate_packages(platform, packagenames, is_embedded): `_ is needed to access the COM port your program uses. No special driver installation is necessary on Windows 10. -""") +""" + ) else: - lines.append(""" + lines.append( + """ **Windows Users:** Please check that you have a correctly installed USB driver from board manufacturer -""") +""" + ) return "\n".join(lines) @@ -293,14 +339,12 @@ def generate_packages(platform, packagenames, is_embedded): def generate_platform(name, rst_dir): print("Processing platform: %s" % name) - compatible_boards = [ - board for board in BOARDS if name == board['platform'] - ] + compatible_boards = [board for board in BOARDS if name == board["platform"]] lines = [] lines.append(RST_COPYRIGHT) - p = PlatformFactory.newPlatform(name) + p = PlatformFactory.new(name) assert p.repository_url.endswith(".git") github_url = p.repository_url[:-4] @@ -314,14 +358,18 @@ def generate_platform(name, rst_dir): lines.append(" :ref:`projectconf_env_platform` = ``%s``" % p.name) lines.append("") lines.append(p.description) - lines.append(""" -For more detailed information please visit `vendor site <%s>`_.""" % - campaign_url(p.vendor_url)) - lines.append(""" + lines.append( + """ +For more detailed information please visit `vendor site <%s>`_.""" + % campaign_url(p.homepage) + ) + lines.append( + """ .. contents:: Contents :local: :depth: 1 -""") +""" + ) # # Extra @@ -332,12 +380,15 @@ def generate_platform(name, rst_dir): # # Examples # - lines.append(""" + lines.append( + """ Examples -------- Examples are listed from `%s development platform repository <%s>`_: -""" % (p.title, campaign_url("%s/tree/master/examples" % github_url))) +""" + % (p.title, campaign_url("%s/tree/master/examples" % github_url)) + ) examples_dir = join(p.get_dir(), "examples") if isdir(examples_dir): for eitem in os.listdir(examples_dir): @@ -355,14 +406,17 @@ def generate_platform(name, rst_dir): generate_debug_contents( compatible_boards, skip_board_columns=["Platform"], - extra_rst="%s_debug.rst" % - name if isfile(join(rst_dir, "%s_debug.rst" % - name)) else None)) + extra_rst="%s_debug.rst" % name + if isfile(join(rst_dir, "%s_debug.rst" % name)) + else None, + ) + ) # # Development version of dev/platform # - lines.append(""" + lines.append( + """ Stable and upstream versions ---------------------------- @@ -393,13 +447,15 @@ def generate_platform(name, rst_dir): [env:upstream_develop] platform = {github_url}.git board = ... -""".format(name=p.name, title=p.title, github_url=github_url)) +""".format( + name=p.name, title=p.title, github_url=github_url + ) + ) # # Packages # - _packages_content = generate_packages(name, p.packages.keys(), - p.is_embedded()) + _packages_content = generate_packages(name, p.packages.keys(), p.is_embedded()) if _packages_content: lines.append(_packages_content) @@ -408,8 +464,8 @@ def generate_platform(name, rst_dir): # compatible_frameworks = [] for framework in API_FRAMEWORKS: - if is_compat_platform_and_framework(name, framework['name']): - compatible_frameworks.append(framework['name']) + if is_compat_platform_and_framework(name, framework["name"]): + compatible_frameworks.append(framework["name"]) lines.extend(generate_frameworks_contents(compatible_frameworks)) # @@ -418,11 +474,12 @@ def generate_platform(name, rst_dir): if compatible_boards: vendors = {} for board in compatible_boards: - if board['vendor'] not in vendors: - vendors[board['vendor']] = [] - vendors[board['vendor']].append(board) + if board["vendor"] not in vendors: + vendors[board["vendor"]] = [] + vendors[board["vendor"]].append(board) - lines.append(""" + lines.append( + """ Boards ------ @@ -431,20 +488,20 @@ def generate_platform(name, rst_dir): `PlatformIO Boards Explorer `_ * For more detailed ``board`` information please scroll the tables below by horizontally. -""") +""" + ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) - lines.extend( - generate_boards_table(boards, skip_columns=["Platform"])) + lines.extend(generate_boards_table(boards, skip_columns=["Platform"])) return "\n".join(lines) def update_platform_docs(): for manifest in PLATFORM_MANIFESTS: - name = manifest['name'] + name = manifest["name"] platforms_dir = join(DOCS_ROOT_DIR, "platforms") rst_path = join(platforms_dir, "%s.rst" % name) with open(rst_path, "w") as f: @@ -455,12 +512,11 @@ def generate_framework(type_, data, rst_dir=None): print("Processing framework: %s" % type_) compatible_platforms = [ - m for m in PLATFORM_MANIFESTS - if is_compat_platform_and_framework(m['name'], type_) - ] - compatible_boards = [ - board for board in BOARDS if type_ in board['frameworks'] + m + for m in PLATFORM_MANIFESTS + if is_compat_platform_and_framework(m["name"], type_) ] + compatible_boards = [board for board in BOARDS if type_ in board["frameworks"]] lines = [] @@ -468,21 +524,26 @@ def generate_framework(type_, data, rst_dir=None): lines.append(".. _framework_%s:" % type_) lines.append("") - lines.append(data['title']) - lines.append("=" * len(data['title'])) + lines.append(data["title"]) + lines.append("=" * len(data["title"])) lines.append("") lines.append(":Configuration:") lines.append(" :ref:`projectconf_env_framework` = ``%s``" % type_) lines.append("") - lines.append(data['description']) - lines.append(""" + lines.append(data["description"]) + lines.append( + """ For more detailed information please visit `vendor site <%s>`_. -""" % campaign_url(data['url'])) +""" + % campaign_url(data["url"]) + ) - lines.append(""" + lines.append( + """ .. contents:: Contents :local: - :depth: 1""") + :depth: 1""" + ) # Extra if isfile(join(rst_dir, "%s_extra.rst" % type_)): @@ -495,27 +556,37 @@ def generate_framework(type_, data, rst_dir=None): lines.extend( generate_debug_contents( compatible_boards, - extra_rst="%s_debug.rst" % - type_ if isfile(join(rst_dir, "%s_debug.rst" % - type_)) else None)) + extra_rst="%s_debug.rst" % type_ + if isfile(join(rst_dir, "%s_debug.rst" % type_)) + else None, + ) + ) if compatible_platforms: # examples - lines.append(""" + lines.append( + """ Examples -------- -""") +""" + ) for manifest in compatible_platforms: - p = PlatformFactory.newPlatform(manifest['name']) - lines.append("* `%s for %s <%s>`_" % - (data['title'], manifest['title'], - campaign_url("%s/tree/master/examples" % - p.repository_url[:-4]))) + p = PlatformFactory.new(manifest["name"]) + lines.append( + "* `%s for %s <%s>`_" + % ( + data["title"], + manifest["title"], + campaign_url("%s/tree/master/examples" % p.repository_url[:-4]), + ) + ) # Platforms lines.extend( generate_platforms_contents( - [manifest['name'] for manifest in compatible_platforms])) + [manifest["name"] for manifest in compatible_platforms] + ) + ) # # Boards @@ -523,10 +594,11 @@ def generate_framework(type_, data, rst_dir=None): if compatible_boards: vendors = {} for board in compatible_boards: - if board['vendor'] not in vendors: - vendors[board['vendor']] = [] - vendors[board['vendor']].append(board) - lines.append(""" + if board["vendor"] not in vendors: + vendors[board["vendor"]] = [] + vendors[board["vendor"]].append(board) + lines.append( + """ Boards ------ @@ -534,7 +606,8 @@ def generate_framework(type_, data, rst_dir=None): * You can list pre-configured boards by :ref:`cmd_boards` command or `PlatformIO Boards Explorer `_ * For more detailed ``board`` information please scroll the tables below by horizontally. -""") +""" + ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) @@ -544,7 +617,7 @@ def generate_framework(type_, data, rst_dir=None): def update_framework_docs(): for framework in API_FRAMEWORKS: - name = framework['name'] + name = framework["name"] frameworks_dir = join(DOCS_ROOT_DIR, "frameworks") rst_path = join(frameworks_dir, "%s.rst" % name) with open(rst_path, "w") as f: @@ -561,7 +634,8 @@ def update_boards(): lines.append("Boards") lines.append("======") - lines.append(""" + lines.append( + """ Rapid Embedded Development, Continuous and IDE integration in a few steps with PlatformIO thanks to built-in project generator for the most popular embedded boards and IDE. @@ -570,25 +644,28 @@ def update_boards(): * You can list pre-configured boards by :ref:`cmd_boards` command or `PlatformIO Boards Explorer `_ * For more detailed ``board`` information please scroll tables below by horizontal. -""") +""" + ) platforms = {} for data in BOARDS: - platform = data['platform'] + platform = data["platform"] if platform in platforms: platforms[platform].append(data) else: platforms[platform] = [data] for platform, boards in sorted(platforms.items()): - p = PlatformFactory.newPlatform(platform) + p = PlatformFactory.new(platform) lines.append(p.title) lines.append("-" * len(p.title)) - lines.append(""" + lines.append( + """ .. toctree:: :maxdepth: 1 - """) - for board in sorted(boards, key=lambda item: item['name']): + """ + ) + for board in sorted(boards, key=lambda item: item["name"]): lines.append(" %s/%s" % (platform, board["id"])) lines.append("") @@ -600,44 +677,48 @@ def update_boards(): for data in BOARDS: # if data['id'] != "m5stack-core-esp32": # continue - rst_path = join(DOCS_ROOT_DIR, "boards", data["platform"], - "%s.rst" % data["id"]) + rst_path = join( + DOCS_ROOT_DIR, "boards", data["platform"], "%s.rst" % data["id"] + ) if not isdir(dirname(rst_path)): os.makedirs(dirname(rst_path)) update_embedded_board(rst_path, data) def update_embedded_board(rst_path, board): - platform = PlatformFactory.newPlatform(board['platform']) - board_config = platform.board_config(board['id']) + platform = PlatformFactory.new(board["platform"]) + board_config = platform.board_config(board["id"]) board_manifest_url = platform.repository_url assert board_manifest_url if board_manifest_url.endswith(".git"): board_manifest_url = board_manifest_url[:-4] - board_manifest_url += "/blob/master/boards/%s.json" % board['id'] - - variables = dict(id=board['id'], - name=board['name'], - platform=board['platform'], - platform_description=platform.description, - url=campaign_url(board['url']), - mcu=board_config.get("build", {}).get("mcu", ""), - mcu_upper=board['mcu'].upper(), - f_cpu=board['fcpu'], - f_cpu_mhz=int(int(board['fcpu']) / 1000000), - ram=fs.format_filesize(board['ram']), - rom=fs.format_filesize(board['rom']), - vendor=board['vendor'], - board_manifest_url=board_manifest_url, - upload_protocol=board_config.get("upload.protocol", "")) + board_manifest_url += "/blob/master/boards/%s.json" % board["id"] + + variables = dict( + id=board["id"], + name=board["name"], + platform=board["platform"], + platform_description=platform.description, + url=campaign_url(board["url"]), + mcu=board_config.get("build", {}).get("mcu", ""), + mcu_upper=board["mcu"].upper(), + f_cpu=board["fcpu"], + f_cpu_mhz=int(int(board["fcpu"]) / 1000000), + ram=fs.humanize_file_size(board["ram"]), + rom=fs.humanize_file_size(board["rom"]), + vendor=board["vendor"], + board_manifest_url=board_manifest_url, + upload_protocol=board_config.get("upload.protocol", ""), + ) lines = [RST_COPYRIGHT] lines.append(".. _board_{platform}_{id}:".format(**variables)) lines.append("") - lines.append(board['name']) - lines.append("=" * len(board['name'])) - lines.append(""" + lines.append(board["name"]) + lines.append("=" * len(board["name"])) + lines.append( + """ .. contents:: Hardware @@ -657,12 +738,16 @@ def update_embedded_board(rst_path, board): - {ram} * - **Vendor** - `{vendor} <{url}>`__ -""".format(**variables)) +""".format( + **variables + ) + ) # # Configuration # - lines.append(""" + lines.append( + """ Configuration ------------- @@ -690,23 +775,33 @@ def update_embedded_board(rst_path, board): ; change MCU frequency board_build.f_cpu = {f_cpu}L -""".format(**variables)) +""".format( + **variables + ) + ) # # Uploading # upload_protocols = board_config.get("upload.protocols", []) if len(upload_protocols) > 1: - lines.append(""" + lines.append( + """ Uploading --------- %s supports the next uploading protocols: -""" % board['name']) +""" + % board["name"] + ) for protocol in sorted(upload_protocols): lines.append("* ``%s``" % protocol) - lines.append(""" -Default protocol is ``%s``""" % variables['upload_protocol']) - lines.append(""" + lines.append( + """ +Default protocol is ``%s``""" + % variables["upload_protocol"] + ) + lines.append( + """ You can change upload protocol using :ref:`projectconf_upload_protocol` option: .. code-block:: ini @@ -716,22 +811,29 @@ def update_embedded_board(rst_path, board): board = {id} upload_protocol = {upload_protocol} -""".format(**variables)) +""".format( + **variables + ) + ) # # Debugging # lines.append("Debugging") lines.append("---------") - if not board['debug']: + if not board.get("debug"): lines.append( ":ref:`piodebug` currently does not support {name} board.".format( - **variables)) + **variables + ) + ) else: default_debug_tool = board_config.get_debug_tool_name() has_onboard_debug = any( - t.get("onboard") for (_, t) in board['debug']['tools'].items()) - lines.append(""" + t.get("onboard") for (_, t) in board["debug"]["tools"].items() + ) + lines.append( + """ :ref:`piodebug` - "1-click" solution for debugging with a zero configuration. .. warning:: @@ -741,34 +843,43 @@ def update_embedded_board(rst_path, board): You can switch between debugging :ref:`debugging_tools` using :ref:`projectconf_debug_tool` option in :ref:`projectconf`. -""") +""" + ) if has_onboard_debug: lines.append( "{name} has on-board debug probe and **IS READY** for " - "debugging. You don't need to use/buy external debug probe.". - format(**variables)) + "debugging. You don't need to use/buy external debug probe.".format( + **variables + ) + ) else: lines.append( "{name} does not have on-board debug probe and **IS NOT " "READY** for debugging. You will need to use/buy one of " - "external probe listed below.".format(**variables)) - lines.append(""" + "external probe listed below.".format(**variables) + ) + lines.append( + """ .. list-table:: :header-rows: 1 * - Compatible Tools - On-board - - Default""") - for (tool_name, tool_data) in sorted(board['debug']['tools'].items()): - lines.append(""" * - :ref:`debugging_tool_{name}` + - Default""" + ) + for (tool_name, tool_data) in sorted(board["debug"]["tools"].items()): + lines.append( + """ * - :ref:`debugging_tool_{name}` - {onboard} - {default}""".format( - name=tool_name, - onboard="Yes" if tool_data.get("onboard") else "", - default="Yes" if tool_name == default_debug_tool else "")) + name=tool_name, + onboard="Yes" if tool_data.get("onboard") else "", + default="Yes" if tool_name == default_debug_tool else "", + ) + ) - if board['frameworks']: - lines.extend(generate_frameworks_contents(board['frameworks'])) + if board["frameworks"]: + lines.extend(generate_frameworks_contents(board["frameworks"])) with open(rst_path, "w") as f: f.write("\n".join(lines)) @@ -781,21 +892,21 @@ def update_debugging(): platforms = [] frameworks = [] for data in BOARDS: - if not data['debug']: + if not data.get("debug"): continue - for tool in data['debug']['tools']: + for tool in data["debug"]["tools"]: tool = str(tool) if tool not in tool_to_platforms: tool_to_platforms[tool] = [] - tool_to_platforms[tool].append(data['platform']) + tool_to_platforms[tool].append(data["platform"]) if tool not in tool_to_boards: tool_to_boards[tool] = [] - tool_to_boards[tool].append(data['id']) + tool_to_boards[tool].append(data["id"]) - platforms.append(data['platform']) - frameworks.extend(data['frameworks']) - vendor = data['vendor'] + platforms.append(data["platform"]) + frameworks.extend(data["frameworks"]) + vendor = data["vendor"] if vendor in vendors: vendors[vendor].append(data) else: @@ -809,26 +920,30 @@ def update_debugging(): lines.extend(generate_frameworks_contents(frameworks)) # Boards - lines.append(""" + lines.append( + """ Boards ------ .. note:: For more detailed ``board`` information please scroll tables below by horizontal. -""") +""" + ) for vendor, boards in sorted(vendors.items()): lines.append(str(vendor)) lines.append("~" * len(vendor)) lines.extend(generate_boards_table(boards)) # save - with open(join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), - "r+") as fp: + with open( + join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), "r+" + ) as fp: content = fp.read() fp.seek(0) fp.truncate() - fp.write(content[:content.index(".. _debugging_platforms:")] + - "\n".join(lines)) + fp.write( + content[: content.index(".. _debugging_platforms:")] + "\n".join(lines) + ) # Debug tools for tool, platforms in tool_to_platforms.items(): @@ -847,24 +962,27 @@ def update_debugging(): tool_frameworks.append(framework) lines.extend(generate_frameworks_contents(tool_frameworks)) - lines.append(""" + lines.append( + """ Boards ------ .. note:: For more detailed ``board`` information please scroll tables below by horizontal. -""") +""" + ) lines.extend( generate_boards_table( - [b for b in BOARDS if b['id'] in tool_to_boards[tool]], - skip_columns=None)) + [b for b in BOARDS if b["id"] in tool_to_boards[tool]], + skip_columns=None, + ) + ) with open(tool_path, "r+") as fp: content = fp.read() fp.seek(0) fp.truncate() - fp.write(content[:content.index(".. begin_platforms")] + - "\n".join(lines)) + fp.write(content[: content.index(".. begin_platforms")] + "\n".join(lines)) def update_project_examples(): @@ -899,7 +1017,7 @@ def update_project_examples(): desktop = [] for manifest in PLATFORM_MANIFESTS: - p = PlatformFactory.newPlatform(manifest['name']) + p = PlatformFactory.new(manifest["name"]) github_url = p.repository_url[:-4] # Platform README @@ -922,19 +1040,21 @@ def update_project_examples(): name=p.name, title=p.title, description=p.description, - examples="\n".join(examples_md_lines))) + examples="\n".join(examples_md_lines), + ) + ) # Framework README for framework in API_FRAMEWORKS: - if not is_compat_platform_and_framework(p.name, framework['name']): + if not is_compat_platform_and_framework(p.name, framework["name"]): continue - if framework['name'] not in framework_examples_md_lines: - framework_examples_md_lines[framework['name']] = [] + if framework["name"] not in framework_examples_md_lines: + framework_examples_md_lines[framework["name"]] = [] lines = [] lines.append("- [%s](%s)" % (p.title, github_url)) lines.extend(" %s" % l for l in examples_md_lines) lines.append("") - framework_examples_md_lines[framework['name']].extend(lines) + framework_examples_md_lines[framework["name"]].extend(lines) # Root README line = "* [%s](%s)" % (p.title, "%s/tree/master/examples" % github_url) @@ -946,27 +1066,29 @@ def update_project_examples(): # Frameworks frameworks = [] for framework in API_FRAMEWORKS: - readme_dir = join(project_examples_dir, "frameworks", - framework['name']) + readme_dir = join(project_examples_dir, "frameworks", framework["name"]) if not isdir(readme_dir): os.makedirs(readme_dir) with open(join(readme_dir, "README.md"), "w") as fp: fp.write( framework_readme_tpl.format( - name=framework['name'], - title=framework['title'], - description=framework['description'], - examples="\n".join( - framework_examples_md_lines[framework['name']]))) + name=framework["name"], + title=framework["title"], + description=framework["description"], + examples="\n".join(framework_examples_md_lines[framework["name"]]), + ) + ) url = campaign_url( "https://docs.platformio.org/en/latest/frameworks/%s.html#examples" - % framework['name'], + % framework["name"], source="github", - medium="examples") - frameworks.append("* [%s](%s)" % (framework['title'], url)) + medium="examples", + ) + frameworks.append("* [%s](%s)" % (framework["title"], url)) with open(join(project_examples_dir, "README.md"), "w") as fp: - fp.write("""# PlatformIO Project Examples + fp.write( + """# PlatformIO Project Examples - [Development platforms](#development-platforms): - [Embedded](#embedded) @@ -986,7 +1108,9 @@ def update_project_examples(): ## Frameworks %s -""" % ("\n".join(embedded), "\n".join(desktop), "\n".join(frameworks))) +""" + % ("\n".join(embedded), "\n".join(desktop), "\n".join(frameworks)) + ) def main(): diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/tests/commands/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/commands/test_account.py b/tests/commands/test_account.py deleted file mode 100644 index ef7ffbad66..0000000000 --- a/tests/commands/test_account.py +++ /dev/null @@ -1,605 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import time - -import pytest - -from platformio.commands.account.command import cli as cmd_account - -pytestmark = pytest.mark.skipif( - not ( - os.environ.get("PLATFORMIO_TEST_ACCOUNT_LOGIN") - and os.environ.get("PLATFORMIO_TEST_ACCOUNT_PASSWORD") - ), - reason="requires PLATFORMIO_TEST_ACCOUNT_LOGIN, PLATFORMIO_TEST_ACCOUNT_PASSWORD environ variables", -) - - -@pytest.fixture(scope="session") -def credentials(): - return { - "login": os.environ["PLATFORMIO_TEST_ACCOUNT_LOGIN"], - "password": os.environ["PLATFORMIO_TEST_ACCOUNT_PASSWORD"], - } - - -def test_account_register_with_already_exists_username( - clirunner, credentials, isolated_pio_home -): - username = credentials["login"] - email = "test@test.com" - if "@" in credentials["login"]: - username = "Testusername" - email = credentials["login"] - result = clirunner.invoke( - cmd_account, - [ - "register", - "-u", - username, - "-e", - email, - "-p", - credentials["password"], - "--firstname", - "First", - "--lastname", - "Last", - ], - ) - assert result.exit_code > 0 - assert result.exception - assert "User with same username already exists" in str( - result.exception - ) or "User with same email already exists" in str(result.exception) - - -@pytest.mark.skip_ci -def test_account_login_with_invalid_creds(clirunner, credentials, isolated_pio_home): - result = clirunner.invoke(cmd_account, ["login", "-u", "123", "-p", "123"]) - assert result.exit_code > 0 - assert result.exception - assert "Invalid user credentials" in str(result.exception) - - -def test_account_login(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - assert "Successfully logged in!" in result.output - - with open(str(isolated_pio_home.join("appstate.json"))) as fp: - appstate = json.load(fp) - assert appstate.get("account") - assert appstate.get("account").get("email") - assert appstate.get("account").get("username") - assert appstate.get("account").get("auth") - assert appstate.get("account").get("auth").get("access_token") - assert appstate.get("account").get("auth").get("access_token_expire") - assert appstate.get("account").get("auth").get("refresh_token") - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are already authorized with" in str(result.exception) - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_logout(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_account, ["logout"]) - validate_cliresult(result) - assert "Successfully logged out" in result.output - - result = clirunner.invoke(cmd_account, ["logout"]) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_password_change_with_invalid_old_password( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - ["password", "--old-password", "test", "--new-password", "test"], - ) - assert result.exit_code > 0 - assert result.exception - assert "Invalid user password" in str(result.exception) - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_password_change_with_invalid_new_password_format( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - credentials["password"], - "--new-password", - "test", - ], - ) - assert result.exit_code > 0 - assert result.exception - assert ( - "Invalid password format. Password must contain at" - " least 8 characters including a number and a lowercase letter" - in str(result.exception) - ) - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_password_change( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - credentials["password"], - "--new-password", - "Testpassword123", - ], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - credentials["password"], - "--new-password", - "Testpassword123", - ], - ) - validate_cliresult(result) - assert "Password successfully changed!" in result.output - - result = clirunner.invoke(cmd_account, ["logout"]) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, ["login", "-u", credentials["login"], "-p", "Testpassword123"], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - [ - "password", - "--old-password", - "Testpassword123", - "--new-password", - credentials["password"], - ], - ) - validate_cliresult(result) - assert "Password successfully changed!" in result.output - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_token_with_invalid_password( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, ["token", "--password", credentials["password"],], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_account, ["token", "--password", "test",],) - assert result.exit_code > 0 - assert result.exception - assert "Invalid user password" in str(result.exception) - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_token(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, ["token", "--password", credentials["password"],], - ) - validate_cliresult(result) - assert "Personal Authentication Token:" in result.output - token = result.output.strip().split(": ")[-1] - - result = clirunner.invoke( - cmd_account, - ["token", "--password", credentials["password"], "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") == token - token = json_result.get("result") - - clirunner.invoke(cmd_account, ["logout"]) - - result = clirunner.invoke( - cmd_account, ["token", "--password", credentials["password"],], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - os.environ["PLATFORMIO_AUTH_TOKEN"] = token - - result = clirunner.invoke( - cmd_account, - ["token", "--password", credentials["password"], "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") == token - - os.environ.pop("PLATFORMIO_AUTH_TOKEN") - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_token_with_refreshing( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - ["token", "--password", credentials["password"], "--json-output"], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") - token = json_result.get("result") - - result = clirunner.invoke( - cmd_account, - [ - "token", - "--password", - credentials["password"], - "--json-output", - "--regenerate", - ], - ) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result - assert json_result.get("status") == "success" - assert json_result.get("result") - assert token != json_result.get("result") - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -def test_account_summary(clirunner, credentials, validate_cliresult, isolated_pio_home): - try: - result = clirunner.invoke(cmd_account, ["show"],) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert not json_result.get("user_id") - assert json_result.get("profile") - assert json_result.get("profile").get("username") - assert json_result.get("profile").get("email") - assert not json_result.get("packages") - assert not json_result.get("subscriptions") - - result = clirunner.invoke(cmd_account, ["show"]) - validate_cliresult(result) - assert credentials["login"] in result.output - assert "Community" in result.output - assert "100 Concurrent Remote Agents" in result.output - - result = clirunner.invoke(cmd_account, ["show", "--json-output"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result.get("user_id") - assert json_result.get("profile") - assert json_result.get("profile").get("username") - assert json_result.get("profile").get("email") - assert credentials["login"] == json_result.get("profile").get( - "username" - ) or credentials["login"] == json_result.get("profile").get("email") - assert json_result.get("profile").get("firstname") - assert json_result.get("profile").get("lastname") - assert json_result.get("packages") - assert json_result.get("packages")[0].get("name") - assert json_result.get("packages")[0].get("path") - assert json_result.get("subscriptions") is not None - - result = clirunner.invoke(cmd_account, ["show", "--json-output", "--offline"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result.get("user_id") - assert json_result.get("profile") - assert json_result.get("profile").get("username") - assert json_result.get("profile").get("email") - assert credentials["login"] == json_result.get("profile").get( - "username" - ) or credentials["login"] == json_result.get("profile").get("email") - assert json_result.get("profile").get("firstname") - assert json_result.get("profile").get("lastname") - assert json_result.get("packages") - assert json_result.get("packages")[0].get("name") - assert json_result.get("packages")[0].get("path") - assert json_result.get("subscriptions") is not None - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_profile_update_with_invalid_password( - clirunner, credentials, validate_cliresult -): - try: - result = clirunner.invoke( - cmd_account, ["update", "--current-password", credentials["password"]], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - firstname = "First " + str(int(time.time() * 1000)) - - result = clirunner.invoke( - cmd_account, - ["update", "--current-password", "test", "--firstname", firstname], - ) - assert result.exit_code > 0 - assert result.exception - assert "Invalid user password" in str(result.exception) - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_profile_update_only_firstname_and_lastname( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: - result = clirunner.invoke( - cmd_account, ["update", "--current-password", credentials["password"]], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - firstname = "First " + str(int(time.time() * 1000)) - lastname = "Last" + str(int(time.time() * 1000)) - - result = clirunner.invoke( - cmd_account, - [ - "update", - "--current-password", - credentials["password"], - "--firstname", - firstname, - "--lastname", - lastname, - ], - ) - validate_cliresult(result) - assert "Profile successfully updated!" in result.output - - result = clirunner.invoke(cmd_account, ["show", "--json-output"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - assert json_result.get("profile").get("firstname") == firstname - assert json_result.get("profile").get("lastname") == lastname - - finally: - clirunner.invoke(cmd_account, ["logout"]) - - -@pytest.mark.skip_ci -def test_account_profile_update( - clirunner, credentials, validate_cliresult, isolated_pio_home -): - try: - result = clirunner.invoke( - cmd_account, ["update", "--current-password", credentials["password"]], - ) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, - ["login", "-u", credentials["login"], "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke(cmd_account, ["show", "--json-output"]) - validate_cliresult(result) - json_result = json.loads(result.output.strip()) - - firstname = "First " + str(int(time.time() * 1000)) - lastname = "Last" + str(int(time.time() * 1000)) - - old_username = json_result.get("profile").get("username") - new_username = "username" + str(int(time.time() * 1000))[-5:] - - result = clirunner.invoke( - cmd_account, - [ - "update", - "--current-password", - credentials["password"], - "--firstname", - firstname, - "--lastname", - lastname, - "--username", - new_username, - ], - ) - validate_cliresult(result) - assert "Profile successfully updated!" in result.output - assert "Please re-login." in result.output - - result = clirunner.invoke(cmd_account, ["show"],) - assert result.exit_code > 0 - assert result.exception - assert "You are not authorized! Please log in to PIO Account" in str( - result.exception - ) - - result = clirunner.invoke( - cmd_account, ["login", "-u", new_username, "-p", credentials["password"]], - ) - validate_cliresult(result) - - result = clirunner.invoke( - cmd_account, - [ - "update", - "--current-password", - credentials["password"], - "--username", - old_username, - ], - ) - validate_cliresult(result) - assert "Profile successfully updated!" in result.output - assert "Please re-login." in result.output - - result = clirunner.invoke( - cmd_account, ["login", "-u", old_username, "-p", credentials["password"]], - ) - validate_cliresult(result) - finally: - clirunner.invoke(cmd_account, ["logout"]) diff --git a/tests/commands/test_account_org_team.py b/tests/commands/test_account_org_team.py new file mode 100644 index 0000000000..fc64db41ec --- /dev/null +++ b/tests/commands/test_account_org_team.py @@ -0,0 +1,491 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=global-statement,unused-argument + +import json +import os +import random + +import pytest +import requests + +from platformio.commands.account import cli as cmd_account +from platformio.commands.org import cli as cmd_org +from platformio.commands.team import cli as cmd_team + +pytestmark = pytest.mark.skipif( + not (os.environ.get("TEST_EMAIL_LOGIN") and os.environ.get("TEST_EMAIL_PASSWORD")), + reason="requires TEST_EMAIL_LOGIN, TEST_EMAIL_PASSWORD environ variables", +) + +username = None +email = None +splited_email = None +firstname = None +lastname = None +password = None + +orgname = None +display_name = None +second_username = None + +teamname = None +team_description = None + + +def test_prepare(): + global username, email, splited_email, firstname, lastname + global password, orgname, display_name, second_username, teamname, team_description + + username = "test-piocore-%s" % str(random.randint(0, 100000)) + splited_email = os.environ.get("TEST_EMAIL_LOGIN").split("@") + email = "%s+%s@%s" % (splited_email[0], username, splited_email[1]) + firstname = "Test" + lastname = "User" + password = "Qwerty123!" + + orgname = "testorg-piocore-%s" % str(random.randint(0, 100000)) + display_name = "Test Org for PIO Core" + second_username = "ivankravets" + + teamname = "test-" + str(random.randint(0, 100000)) + team_description = "team for CI test" + + +def test_account_register( + clirunner, validate_cliresult, receive_email, isolated_pio_core +): + result = clirunner.invoke( + cmd_account, + [ + "register", + "-u", + username, + "-e", + email, + "-p", + password, + "--firstname", + firstname, + "--lastname", + lastname, + ], + ) + validate_cliresult(result) + + # email verification + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split(' 0 + assert result.exception + assert "You are not authorized! Please log in to PIO Account" in str( + result.exception + ) + + os.environ["PLATFORMIO_AUTH_TOKEN"] = token + + result = clirunner.invoke( + cmd_account, ["token", "--password", password, "--json-output"], + ) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result + assert json_result.get("status") == "success" + assert json_result.get("result") == token + + os.environ.pop("PLATFORMIO_AUTH_TOKEN") + + result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) + validate_cliresult(result) + + +def test_account_change_password(clirunner, validate_cliresult, isolated_pio_core): + new_password = "Testpassword123" + result = clirunner.invoke( + cmd_account, + ["password", "--old-password", password, "--new-password", new_password,], + ) + validate_cliresult(result) + assert "Password successfully changed!" in result.output + + clirunner.invoke(cmd_account, ["logout"]) + + result = clirunner.invoke( + cmd_account, ["login", "-u", username, "-p", new_password], + ) + validate_cliresult(result) + + result = clirunner.invoke( + cmd_account, + ["password", "--old-password", new_password, "--new-password", password,], + ) + validate_cliresult(result) + + +def test_account_update( + clirunner, validate_cliresult, receive_email, isolated_pio_core +): + global username + global email + global firstname + global lastname + + firstname = "First " + str(random.randint(0, 100000)) + lastname = "Last" + str(random.randint(0, 100000)) + + username = "username" + str(random.randint(0, 100000)) + email = "%s+new-%s@%s" % (splited_email[0], username, splited_email[1]) + result = clirunner.invoke( + cmd_account, + [ + "update", + "--current-password", + password, + "--firstname", + firstname, + "--lastname", + lastname, + "--username", + username, + "--email", + email, + ], + ) + validate_cliresult(result) + assert "Profile successfully updated!" in result.output + assert ( + "Please check your mail to verify your new email address and re-login. " + in result.output + ) + + result = receive_email(email) + link = ( + result.split("Click on the link below to start this process.")[1] + .split("This link will expire within 12 hours.")[0] + .strip() + ) + session = requests.Session() + result = session.get(link).text + link = result.split(' 0 + assert result.exception + assert "You are not authorized! Please log in to PIO Account" in str( + result.exception + ) + + result = clirunner.invoke(cmd_account, ["login", "-u", username, "-p", password],) + validate_cliresult(result) + + +# def test_account_destroy_with_linked_resources( +# clirunner, validate_cliresult, receive_email, isolated_pio_core, tmpdir_factory +# ): +# package_url = "https://github.com/bblanchon/ArduinoJson/archive/v6.11.0.tar.gz" +# +# tmp_dir = tmpdir_factory.mktemp("package") +# fd = FileDownloader(package_url, str(tmp_dir)) +# pkg_dir = tmp_dir.mkdir("raw_package") +# fd.start(with_progress=False, silent=True) +# with FileUnpacker(fd.get_filepath()) as unpacker: +# unpacker.unpack(str(pkg_dir), with_progress=False, silent=True) +# +# result = clirunner.invoke(cmd_package, ["publish", str(pkg_dir)],) +# validate_cliresult(result) +# try: +# result = receive_email(email) +# assert "Congrats" in result +# assert "was published" in result +# except: # pylint:disable=bare-except +# pass +# +# result = clirunner.invoke(cmd_account, ["destroy"], "y") +# assert result.exit_code != 0 +# assert ( +# "We can not destroy the %s account due to 1 linked resources from registry" +# % username +# ) +# +# result = clirunner.invoke(cmd_package, ["unpublish", "ArduinoJson"],) +# validate_cliresult(result) + + +def test_org_create(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_org, ["create", "--email", email, "--displayname", display_name, orgname], + ) + validate_cliresult(result) + + +def test_org_list(clirunner, validate_cliresult, isolated_pio_core): + # pio org list + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result == [ + { + "orgname": orgname, + "displayname": display_name, + "email": email, + "owners": [ + {"username": username, "firstname": firstname, "lastname": lastname} + ], + } + ] + + +def test_org_add_owner(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke(cmd_org, ["add", orgname, second_username]) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + assert second_username in result.output + + +def test_org_remove_owner(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke(cmd_org, ["remove", orgname, second_username]) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + assert second_username not in result.output + + +def test_org_update(clirunner, validate_cliresult, isolated_pio_core): + new_orgname = "neworg-piocore-%s" % str(random.randint(0, 100000)) + new_display_name = "Test Org for PIO Core" + + result = clirunner.invoke( + cmd_org, + [ + "update", + orgname, + "--new-orgname", + new_orgname, + "--displayname", + new_display_name, + ], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_org, ["list", "--json-output"]) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + assert json_result == [ + { + "orgname": new_orgname, + "displayname": new_display_name, + "email": email, + "owners": [ + {"username": username, "firstname": firstname, "lastname": lastname} + ], + } + ] + + result = clirunner.invoke( + cmd_org, + [ + "update", + new_orgname, + "--new-orgname", + orgname, + "--displayname", + display_name, + ], + ) + validate_cliresult(result) + + +def test_team_create(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_team, + ["create", "%s:%s" % (orgname, teamname), "--description", team_description,], + ) + validate_cliresult(result) + + +def test_team_list(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + for item in json_result: + del item["id"] + assert json_result == [ + {"name": teamname, "description": team_description, "members": []} + ] + + +def test_team_add_member(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_team, ["add", "%s:%s" % (orgname, teamname), second_username], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + assert second_username in result.output + + +def test_team_remove(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_team, ["remove", "%s:%s" % (orgname, teamname), second_username], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + assert second_username not in result.output + + +def test_team_update(clirunner, validate_cliresult, receive_email, isolated_pio_core): + new_teamname = "new-" + str(random.randint(0, 100000)) + newteam_description = "Updated Description" + result = clirunner.invoke( + cmd_team, + [ + "update", + "%s:%s" % (orgname, teamname), + "--name", + new_teamname, + "--description", + newteam_description, + ], + ) + validate_cliresult(result) + + result = clirunner.invoke(cmd_team, ["list", "%s" % orgname, "--json-output"],) + validate_cliresult(result) + json_result = json.loads(result.output.strip()) + for item in json_result: + del item["id"] + assert json_result == [ + {"name": new_teamname, "description": newteam_description, "members": []} + ] + + result = clirunner.invoke( + cmd_team, + [ + "update", + "%s:%s" % (orgname, new_teamname), + "--name", + teamname, + "--description", + team_description, + ], + ) + validate_cliresult(result) + + +def test_cleanup(clirunner, validate_cliresult, receive_email, isolated_pio_core): + result = clirunner.invoke(cmd_team, ["destroy", "%s:%s" % (orgname, teamname)], "y") + validate_cliresult(result) + result = clirunner.invoke(cmd_org, ["destroy", orgname], "y") + validate_cliresult(result) + result = clirunner.invoke(cmd_account, ["destroy"], "y") + validate_cliresult(result) diff --git a/tests/commands/test_boards.py b/tests/commands/test_boards.py index bcb1f2804c..21142dd49a 100644 --- a/tests/commands/test_boards.py +++ b/tests/commands/test_boards.py @@ -40,7 +40,7 @@ def test_board_options(clirunner, validate_cliresult): validate_cliresult(result) search_result = json.loads(result.output) assert isinstance(search_result, list) - assert len(search_result) + assert search_result platforms = [item["name"] for item in search_result] result = clirunner.invoke(cmd_boards, ["mbed", "--json-output"]) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index c5125b1e95..fa33af6874 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=redefined-outer-name + import json import sys from os.path import isfile, join @@ -59,6 +61,12 @@ } """ + +PVS_STUDIO_FREE_LICENSE_HEADER = """ +// This is an open source non-commercial project. Dear PVS-Studio, please check it. +// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com +""" + EXPECTED_ERRORS = 4 EXPECTED_WARNINGS = 1 EXPECTED_STYLE = 1 @@ -85,19 +93,21 @@ def count_defects(output): return error, warning, style -def test_check_cli_output(clirunner, check_dir): +def test_check_cli_output(clirunner, validate_cliresult, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir)]) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors + warnings + style == EXPECTED_DEFECTS -def test_check_json_output(clirunner, check_dir): +def test_check_json_output(clirunner, validate_cliresult, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), "--json-output"] ) + validate_cliresult(result) + output = json.loads(result.stdout.strip()) assert isinstance(output, list) @@ -112,14 +122,24 @@ def test_check_tool_defines_passed(clirunner, check_dir): assert "__GNUC__" in output -def test_check_severity_threshold(clirunner, check_dir): +def test_check_language_standard_definition_passed(clirunner, tmpdir): + config = DEFAULT_CONFIG + "\nbuild_flags = -std=c++17" + tmpdir.join("platformio.ini").write(config) + tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + + assert "__cplusplus=201703L" in result.output + assert "--std=c++17" in result.output + + +def test_check_severity_threshold(clirunner, validate_cliresult, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), "--severity=high"] ) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == 0 assert style == 0 @@ -127,10 +147,9 @@ def test_check_severity_threshold(clirunner, check_dir): def test_check_includes_passed(clirunner, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) - output = result.output inc_count = 0 - for l in output.split("\n"): + for l in result.output.split("\n"): if l.startswith("Includes:"): inc_count = l.count("-I") @@ -138,18 +157,20 @@ def test_check_includes_passed(clirunner, check_dir): assert inc_count > 1 -def test_check_silent_mode(clirunner, check_dir): +def test_check_silent_mode(clirunner, validate_cliresult, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--silent"]) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == 0 assert style == 0 -def test_check_custom_pattern_absolute_path(clirunner, tmpdir_factory): +def test_check_custom_pattern_absolute_path( + clirunner, validate_cliresult, tmpdir_factory +): project_dir = tmpdir_factory.mktemp("project") project_dir.join("platformio.ini").write(DEFAULT_CONFIG) @@ -159,16 +180,18 @@ def test_check_custom_pattern_absolute_path(clirunner, tmpdir_factory): result = clirunner.invoke( cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] ) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors == EXPECTED_ERRORS assert warnings == EXPECTED_WARNINGS assert style == EXPECTED_STYLE -def test_check_custom_pattern_relative_path(clirunner, tmpdir_factory): +def test_check_custom_pattern_relative_path( + clirunner, validate_cliresult, tmpdir_factory +): tmpdir = tmpdir_factory.mktemp("project") tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) @@ -178,10 +201,10 @@ def test_check_custom_pattern_relative_path(clirunner, tmpdir_factory): result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--pattern=app", "--pattern=prj"] ) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) - assert result.exit_code == 0 assert errors + warnings + style == EXPECTED_DEFECTS * 2 @@ -212,7 +235,7 @@ def test_check_bad_flag_passed(clirunner, check_dir): assert style == 0 -def test_check_success_if_no_errors(clirunner, tmpdir): +def test_check_success_if_no_errors(clirunner, validate_cliresult, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src").join("main.c").write( """ @@ -230,26 +253,30 @@ def test_check_success_if_no_errors(clirunner, tmpdir): ) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) errors, warnings, style = count_defects(result.output) assert "[PASSED]" in result.output - assert result.exit_code == 0 assert errors == 0 assert warnings == 1 assert style == 1 -def test_check_individual_flags_passed(clirunner, tmpdir): +def test_check_individual_flags_passed(clirunner, validate_cliresult, tmpdir): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy, pvs-studio" config += """\ncheck_flags = cppcheck: --std=c++11 clangtidy: --fix-errors pvs-studio: --analysis-mode=4 """ + tmpdir.join("platformio.ini").write(config) - tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) + tmpdir.mkdir("src").join("main.cpp").write( + PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE + ) result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir), "-v"]) + validate_cliresult(result) clang_flags_found = cppcheck_flags_found = pvs_flags_found = False for l in result.output.split("\n"): @@ -267,7 +294,7 @@ def test_check_individual_flags_passed(clirunner, tmpdir): assert pvs_flags_found -def test_check_cppcheck_misra_addon(clirunner, check_dir): +def test_check_cppcheck_misra_addon(clirunner, validate_cliresult, check_dir): check_dir.join("misra.json").write( """ { @@ -307,12 +334,12 @@ def test_check_cppcheck_misra_addon(clirunner, check_dir): cmd_check, ["--project-dir", str(check_dir), "--flags=--addon=misra.json"] ) - assert result.exit_code == 0 + validate_cliresult(result) assert "R21.3 Found MISRA defect" in result.output assert not isfile(join(str(check_dir), "src", "main.cpp.dump")) -def test_check_fails_on_defects_only_with_flag(clirunner, tmpdir): +def test_check_fails_on_defects_only_with_flag(clirunner, validate_cliresult, tmpdir): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.cpp").write(TEST_CODE) @@ -323,11 +350,13 @@ def test_check_fails_on_defects_only_with_flag(clirunner, tmpdir): cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] ) - assert default_result.exit_code == 0 + validate_cliresult(default_result) assert result_with_flag.exit_code != 0 -def test_check_fails_on_defects_only_on_specified_level(clirunner, tmpdir): +def test_check_fails_on_defects_only_on_specified_level( + clirunner, validate_cliresult, tmpdir +): config = DEFAULT_CONFIG + "\ncheck_tool = cppcheck, clangtidy" tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write( @@ -348,12 +377,12 @@ def test_check_fails_on_defects_only_on_specified_level(clirunner, tmpdir): high_result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high"] ) + validate_cliresult(high_result) low_result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=low"] ) - assert high_result.exit_code == 0 assert low_result.exit_code != 0 @@ -365,15 +394,9 @@ def test_check_pvs_studio_free_license(clirunner, tmpdir): framework = arduino check_tool = pvs-studio """ - code = ( - """// This is an open source non-commercial project. Dear PVS-Studio, please check it. -// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com -""" - + TEST_CODE - ) tmpdir.join("platformio.ini").write(config) - tmpdir.mkdir("src").join("main.c").write(code) + tmpdir.mkdir("src").join("main.c").write(PVS_STUDIO_FREE_LICENSE_HEADER + TEST_CODE) result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--fail-on-defect=high", "-v"] @@ -387,7 +410,7 @@ def test_check_pvs_studio_free_license(clirunner, tmpdir): assert style == 0 -def test_check_embedded_platform_all_tools(clirunner, tmpdir): +def test_check_embedded_platform_all_tools(clirunner, validate_cliresult, tmpdir): config = """ [env:test] platform = ststm32 @@ -397,8 +420,8 @@ def test_check_embedded_platform_all_tools(clirunner, tmpdir): """ # tmpdir.join("platformio.ini").write(config) tmpdir.mkdir("src").join("main.c").write( - """// This is an open source non-commercial project. Dear PVS-Studio, please check it. -// PVS-Studio Static Code Analyzer for C, C++, C#, and Java: http://www.viva64.com + PVS_STUDIO_FREE_LICENSE_HEADER + + """ #include void unused_function(int val){ @@ -412,7 +435,7 @@ def test_check_embedded_platform_all_tools(clirunner, tmpdir): """ ) - frameworks = ["arduino", "mbed", "stm32cube"] + frameworks = ["arduino", "stm32cube"] if sys.version_info[0] == 3: # Zephyr only supports Python 3 frameworks.append("zephyr") @@ -420,18 +443,13 @@ def test_check_embedded_platform_all_tools(clirunner, tmpdir): for framework in frameworks: for tool in ("cppcheck", "clangtidy", "pvs-studio"): tmpdir.join("platformio.ini").write(config % (framework, tool)) - result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) - + validate_cliresult(result) defects = sum(count_defects(result.output)) - - assert result.exit_code == 0 and defects > 0, "Failed %s with %s" % ( - framework, - tool, - ) + assert defects > 0, "Failed %s with %s" % (framework, tool,) -def test_check_skip_includes_from_packages(clirunner, tmpdir): +def test_check_skip_includes_from_packages(clirunner, validate_cliresult, tmpdir): config = """ [env:test] platform = nordicnrf52 @@ -445,13 +463,42 @@ def test_check_skip_includes_from_packages(clirunner, tmpdir): result = clirunner.invoke( cmd_check, ["--project-dir", str(tmpdir), "--skip-packages", "-v"] ) - - output = result.output + validate_cliresult(result) project_path = fs.to_unix_path(str(tmpdir)) - for l in output.split("\n"): + for l in result.output.split("\n"): if not l.startswith("Includes:"): continue for inc in l.split(" "): if inc.startswith("-I") and project_path not in inc: pytest.fail("Detected an include path from packages: " + inc) + + +def test_check_multiline_error(clirunner, tmpdir_factory): + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write(DEFAULT_CONFIG) + + project_dir.mkdir("include").join("main.h").write( + """ +#error This is a multiline error message \\ +that should be correctly reported \\ +in both default and verbose modes. +""" + ) + + project_dir.mkdir("src").join("main.c").write( + """ +#include +#include "main.h" + +int main() {} +""" + ) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir)]) + errors, _, _ = count_defects(result.output) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(project_dir), "-v"]) + verbose_errors, _, _ = count_defects(result.output) + + assert verbose_errors == errors == 1 diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 0f7aceeda2..0ea22dd6a2 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -15,7 +15,7 @@ from os.path import isfile, join from platformio.commands.ci import cli as cmd_ci -from platformio.commands.lib import cli as cmd_lib +from platformio.commands.lib.command import cli as cmd_lib def test_ci_empty(clirunner): @@ -114,13 +114,13 @@ def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult): [ join( storage_dir, - "OneWire_ID1", + "OneWire", "examples", "DS2408_Switch", "DS2408_Switch.pde", ), "-l", - join(storage_dir, "OneWire_ID1"), + join(storage_dir, "OneWire"), "-b", "uno", ], diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index b874ead763..09bd8cf9b5 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -62,29 +62,50 @@ def test_init_ide_without_board(clirunner, tmpdir): assert isinstance(result.exception, ProjectEnvsNotAvailableError) -def test_init_ide_atom(clirunner, validate_cliresult, tmpdir): +def test_init_ide_vscode(clirunner, validate_cliresult, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke( - cmd_init, ["--ide", "atom", "-b", "uno", "-b", "teensy31"] + cmd_init, ["--ide", "vscode", "-b", "uno", "-b", "teensy31"] ) validate_cliresult(result) validate_pioproject(str(tmpdir)) assert all( - [tmpdir.join(f).check() for f in (".clang_complete", ".gcc-flags.json")] + [ + tmpdir.join(".vscode").join(f).check() + for f in ("c_cpp_properties.json", "launch.json") + ] + ) + assert ( + "framework-arduino-avr" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() ) - assert "framework-arduino" in tmpdir.join(".clang_complete").read() # switch to NodeMCU - result = clirunner.invoke(cmd_init, ["--ide", "atom", "-b", "nodemcuv2"]) + result = clirunner.invoke(cmd_init, ["--ide", "vscode", "-b", "nodemcuv2"]) + validate_cliresult(result) + validate_pioproject(str(tmpdir)) + assert ( + "framework-arduinoespressif8266" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) + + # switch to teensy31 via env name + result = clirunner.invoke(cmd_init, ["--ide", "vscode", "-e", "teensy31"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) - assert "arduinoespressif" in tmpdir.join(".clang_complete").read() + assert ( + "framework-arduinoteensy" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) # switch to the first board - result = clirunner.invoke(cmd_init, ["--ide", "atom"]) + result = clirunner.invoke(cmd_init, ["--ide", "vscode"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) - assert "framework-arduino" in tmpdir.join(".clang_complete").read() + assert ( + "framework-arduino-avr" + in tmpdir.join(".vscode").join("c_cpp_properties.json").read() + ) def test_init_ide_eclipse(clirunner, validate_cliresult): diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 752c2c30bc..b25418419d 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -12,338 +12,218 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -import re - -from platformio import exception -from platformio.commands import PlatformioCLI -from platformio.commands.lib import cli as cmd_lib - -PlatformioCLI.leftover_args = ["--json-output"] # hook for click - - -def test_search(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) - validate_cliresult(result) - match = re.search(r"Found\s+(\d+)\slibraries:", result.output) - assert int(match.group(1)) > 2 +# pylint: disable=unused-argument - result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) - validate_cliresult(result) - match = re.search(r"Found\s+(\d+)\slibraries:", result.output) - assert int(match.group(1)) > 1 - - -def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_home): +import json +import os + +import semantic_version + +from platformio.clients.registry import RegistryClient +from platformio.commands.lib.command import cli as cmd_lib +from platformio.package.meta import PackageType +from platformio.package.vcsclient import VCSClientFactory +from platformio.project.config import ProjectConfig + + +def test_saving_deps(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): + regclient = RegistryClient() + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write( + """ +[env] +lib_deps = ArduinoJson + +[env:one] +board = devkit + +[env:two] +framework = foo +lib_deps = + CustomLib + ArduinoJson @ 5.10.1 +""" + ) result = clirunner.invoke( cmd_lib, - [ - "-g", - "install", - "64", - "ArduinoJson@~5.10.0", - "547@2.2.4", - "AsyncMqttClient@<=0.8.2", - "Adafruit PN532@1.2.0", - ], + ["-d", str(project_dir), "install", "64", "knolleary/PubSubClient@~2.7"], ) validate_cliresult(result) - - # install unknown library - result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"]) - assert result.exit_code != 0 - assert isinstance(result.exception, exception.LibNotFound) - - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] - items2 = [ - "ArduinoJson_ID64", - "ArduinoJson_ID64@5.10.1", - "NeoPixelBus_ID547", - "AsyncMqttClient_ID346", - "ESPAsyncTCP_ID305", - "AsyncTCP_ID1826", - "Adafruit PN532_ID29", - "Adafruit BusIO_ID6214", - ] - assert set(items1) == set(items2) - - -def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_home): - result = clirunner.invoke( - cmd_lib, + aj_pkg_data = regclient.get_package(PackageType.LIBRARY, "bblanchon", "ArduinoJson") + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert sorted(config.get("env:one", "lib_deps")) == sorted( [ - "-g", - "install", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", - "SomeLib=http://dl.platformio.org/libraries/archives/0/9540.tar.gz", - "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", - ], + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + ] + ) + assert sorted(config.get("env:two", "lib_deps")) == sorted( + [ + "CustomLib", + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + ] ) - validate_cliresult(result) - # incorrect requirements + # ensure "build" version without NPM spec result = clirunner.invoke( cmd_lib, + ["-d", str(project_dir), "-e", "one", "install", "mbed-sam-grove/LinkedList"], + ) + validate_cliresult(result) + ll_pkg_data = regclient.get_package( + PackageType.LIBRARY, "mbed-sam-grove", "LinkedList" + ) + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert sorted(config.get("env:one", "lib_deps")) == sorted( [ - "-g", - "install", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3", - ], + "bblanchon/ArduinoJson@^%s" % aj_pkg_data["version"]["name"], + "knolleary/PubSubClient@~2.7", + "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], + ] ) - assert result.exit_code != 0 - - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] - items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire_ID1", "ESP32WebServer"] - assert set(items1) >= set(items2) - -def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_home): + # check external package via Git repo result = clirunner.invoke( cmd_lib, [ - "-g", + "-d", + str(project_dir), + "-e", + "one", "install", - "https://github.com/gioblu/PJON.git#3.0", - "https://github.com/gioblu/PJON.git#6.2", - "https://github.com/bblanchon/ArduinoJson.git", - "https://gitlab.com/ivankravets/rs485-nodeproto.git", - "https://github.com/platformio/platformio-libmirror.git", - # "https://developer.mbed.org/users/simon/code/TextLCD/", - "knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163", + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", ], ) validate_cliresult(result) - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] - items2 = [ - "PJON", - "PJON@src-79de467ebe19de18287becff0a1fb42d", - "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "rs485-nodeproto", - "platformio-libmirror", - "PubSubClient", - ] - assert set(items1) >= set(items2) - + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert len(config.get("env:one", "lib_deps")) == 4 + assert config.get("env:one", "lib_deps")[3] == ( + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3" + ) -def test_install_duplicates(clirunner, validate_cliresult, without_internet): - # registry + # test uninstalling + # from all envs result = clirunner.invoke( - cmd_lib, - ["-g", "install", "http://dl.platformio.org/libraries/archives/0/9540.tar.gz"], + cmd_lib, ["-d", str(project_dir), "uninstall", "ArduinoJson"] ) validate_cliresult(result) - assert "is already installed" in result.output - - # by ID - result = clirunner.invoke(cmd_lib, ["-g", "install", "29"]) - validate_cliresult(result) - assert "is already installed" in result.output - - # archive + # from "one" env result = clirunner.invoke( cmd_lib, [ - "-g", - "install", - "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + "-d", + str(project_dir), + "-e", + "one", + "uninstall", + "knolleary/PubSubClient@~2.7", ], ) validate_cliresult(result) - assert "is already installed" in result.output + config = ProjectConfig(os.path.join(str(project_dir), "platformio.ini")) + assert len(config.get("env:one", "lib_deps")) == 2 + assert len(config.get("env:two", "lib_deps")) == 2 + assert config.get("env:one", "lib_deps") == [ + "mbed-sam-grove/LinkedList@%s" % ll_pkg_data["version"]["name"], + "https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3 @ 0.8.3", + ] + assert config.get("env:two", "lib_deps") == [ + "CustomLib", + "knolleary/PubSubClient@~2.7", + ] - # repository + # test list + result = clirunner.invoke(cmd_lib, ["-d", str(project_dir), "list"]) + validate_cliresult(result) + assert "Version: 0.8.3+sha." in result.stdout + assert ( + "Source: git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3" + in result.stdout + ) result = clirunner.invoke( - cmd_lib, - ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"], + cmd_lib, ["-d", str(project_dir), "list", "--json-output"] ) validate_cliresult(result) - assert "is already installed" in result.output - - -def test_global_lib_list(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["-g", "list"]) - validate_cliresult(result) - assert all( - [ - n in result.output - for n in ( - "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", - "Version: 5.10.1", - "Source: git+https://github.com/gioblu/PJON.git#3.0", - "Version: 1fb26fd", - ) - ] + data = {} + for key, value in json.loads(result.stdout).items(): + data[os.path.basename(key)] = value + ame_lib = next( + item for item in data["one"] if item["name"] == "AsyncMqttClient-esphome" ) - - result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) - assert all( - [ - n in result.output - for n in ( - "__pkg_dir", - '"__src_url": "git+https://gitlab.com/ivankravets/rs485-nodeproto.git"', - '"version": "5.10.1"', - ) - ] + ame_vcs = VCSClientFactory.new(ame_lib["__pkg_dir"], ame_lib["__src_url"]) + assert len(data["two"]) == 1 + assert data["two"][0]["name"] == "PubSubClient" + assert "__pkg_dir" in data["one"][0] + assert ( + ame_lib["__src_url"] + == "git+https://github.com/OttoWinter/async-mqtt-client.git#v0.8.3" ) - items1 = [i["name"] for i in json.loads(result.output)] - items2 = [ - "ESP32WebServer", - "ArduinoJson", - "ArduinoJson", - "ArduinoJson", - "ArduinoJson", - "AsyncMqttClient", - "AsyncTCP", - "SomeLib", - "ESPAsyncTCP", - "NeoPixelBus", - "OneWire", - "PJON", - "PJON", - "PubSubClient", - "Adafruit PN532", - "Adafruit BusIO", - "platformio-libmirror", - "rs485-nodeproto", - ] - assert sorted(items1) == sorted(items2) - - versions1 = [ - "{name}@{version}".format(**item) for item in json.loads(result.output) - ] - versions2 = [ - "ArduinoJson@5.8.2", - "ArduinoJson@5.10.1", - "AsyncMqttClient@0.8.2", - "NeoPixelBus@2.2.4", - "PJON@07fe9aa", - "PJON@1fb26fd", - "PubSubClient@bef5814", - "Adafruit PN532@1.2.0", - ] - assert set(versions1) >= set(versions2) + assert ame_lib["version"] == ("0.8.3+sha.%s" % ame_vcs.get_current_revision()) -def test_global_lib_update_check(clirunner, validate_cliresult): +def test_update(clirunner, validate_cliresult, isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("test-updates") result = clirunner.invoke( - cmd_lib, ["-g", "update", "--only-check", "--json-output"] + cmd_lib, + ["-d", str(storage_dir), "install", "ArduinoJson @ 5.10.1", "Blynk @ ~0.5.0"], ) validate_cliresult(result) - output = json.loads(result.output) - assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([l["name"] for l in output]) - - -def test_global_lib_update(clirunner, validate_cliresult): - # update library using package directory result = clirunner.invoke( - cmd_lib, ["-g", "update", "NeoPixelBus", "--only-check", "--json-output"] + cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "--json-output"] ) validate_cliresult(result) - oudated = json.loads(result.output) - assert len(oudated) == 1 - assert "__pkg_dir" in oudated[0] - result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]]) - validate_cliresult(result) - assert "Uninstalling NeoPixelBus @ 2.2.4" in result.output - - # update rest libraries - result = clirunner.invoke(cmd_lib, ["-g", "update"]) - validate_cliresult(result) - assert result.output.count("[Detached]") == 5 - assert result.output.count("[Up-to-date]") == 12 - - # update unknown library - result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) - assert result.exit_code != 0 - assert isinstance(result.exception, exception.UnknownPackage) - - -def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): - # uninstall using package directory - result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) - validate_cliresult(result) - items = json.loads(result.output) - items = sorted(items, key=lambda item: item["__pkg_dir"]) - result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) - validate_cliresult(result) - assert ("Uninstalling %s" % items[0]["name"]) in result.output - - # uninstall the rest libraries + outdated = json.loads(result.stdout) + assert len(outdated) == 2 + # ArduinoJson + assert outdated[0]["version"] == "5.10.1" + assert outdated[0]["versionWanted"] is None + assert semantic_version.Version( + outdated[0]["versionLatest"] + ) > semantic_version.Version("6.16.0") + # Blynk + assert outdated[1]["version"] == "0.5.4" + assert outdated[1]["versionWanted"] is None + assert semantic_version.Version( + outdated[1]["versionLatest"] + ) > semantic_version.Version("0.6.0") + + # check with spec result = clirunner.invoke( cmd_lib, [ - "-g", - "uninstall", - "1", - "https://github.com/bblanchon/ArduinoJson.git", - "ArduinoJson@!=5.6.7", - "Adafruit PN532", + "-d", + str(storage_dir), + "update", + "--dry-run", + "--json-output", + "ArduinoJson @ ^5", ], ) validate_cliresult(result) - - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] - items2 = [ - "rs485-nodeproto", - "platformio-libmirror", - "PubSubClient", - "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "ESPAsyncTCP_ID305", - "ESP32WebServer", - "NeoPixelBus_ID547", - "PJON", - "AsyncMqttClient_ID346", - "ArduinoJson_ID64", - "SomeLib_ID54", - "PJON@src-79de467ebe19de18287becff0a1fb42d", - "AsyncTCP_ID1826", - ] - assert set(items1) == set(items2) - - # uninstall unknown library - result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"]) - assert result.exit_code != 0 - assert isinstance(result.exception, exception.UnknownPackage) - - -def test_lib_show(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["show", "64"]) - validate_cliresult(result) - assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")]) - result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) - validate_cliresult(result) - assert "OneWire" in result.output - - -def test_lib_builtin(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["builtin"]) + outdated = json.loads(result.stdout) + assert outdated[0]["version"] == "5.10.1" + assert outdated[0]["versionWanted"] == "5.13.4" + assert semantic_version.Version( + outdated[0]["versionLatest"] + ) > semantic_version.Version("6.16.0") + # update with spec + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "update", "--silent", "ArduinoJson @ ^5.10.1"] + ) validate_cliresult(result) - result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"]) + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "list", "--json-output"] + ) validate_cliresult(result) + items = json.loads(result.stdout) + assert len(items) == 2 + assert items[0]["version"] == "5.13.4" + assert items[1]["version"] == "0.5.4" - -def test_lib_stats(clirunner, validate_cliresult): - result = clirunner.invoke(cmd_lib, ["stats"]) - validate_cliresult(result) - assert all( - [ - s in result.output - for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show") - ] + # Check incompatible + result = clirunner.invoke( + cmd_lib, ["-d", str(storage_dir), "update", "--dry-run", "ArduinoJson @ ^5"] ) - - result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) validate_cliresult(result) - assert set( - [ - "dlweek", - "added", - "updated", - "topkeywords", - "dlmonth", - "dlday", - "lastkeywords", - ] - ) == set(json.loads(result.output).keys()) + assert "Incompatible" in result.stdout diff --git a/tests/commands/test_lib_complex.py b/tests/commands/test_lib_complex.py new file mode 100644 index 0000000000..442edee35f --- /dev/null +++ b/tests/commands/test_lib_complex.py @@ -0,0 +1,352 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long + +import json +import re + +from platformio.commands import PlatformioCLI +from platformio.commands.lib.command import cli as cmd_lib +from platformio.package.exception import UnknownPackageError + +PlatformioCLI.leftover_args = ["--json-output"] # hook for click + + +def test_search(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) + validate_cliresult(result) + match = re.search(r"Found\s+(\d+)\slibraries:", result.output) + assert int(match.group(1)) > 2 + + result = clirunner.invoke(cmd_lib, ["search", "DHT22", "--platform=timsp430"]) + validate_cliresult(result) + match = re.search(r"Found\s+(\d+)\slibraries:", result.output) + assert int(match.group(1)) > 1 + + +def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "64", + "ArduinoJson@~5.10.0", + "547@2.2.4", + "AsyncMqttClient@<=0.8.2", + "Adafruit PN532@1.2.0", + ], + ) + validate_cliresult(result) + + # install unknown library + result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "ArduinoJson@5.10.1", + "NeoPixelBus", + "AsyncMqttClient", + "ESPAsyncTCP", + "AsyncTCP", + "Adafruit PN532", + "Adafruit BusIO", + ] + assert set(items1) == set(items2) + + +def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", + "SomeLib=https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz", + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + ], + ) + validate_cliresult(result) + + # incorrect requirements + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3", + ], + ) + assert result.exit_code != 0 + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "SomeLib", + "OneWire", + "ESP32WebServer", + ] + assert set(items1) >= set(items2) + + +def test_global_install_repository(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/gioblu/PJON.git#3.0", + "https://github.com/gioblu/PJON.git#6.2", + "https://github.com/bblanchon/ArduinoJson.git", + "https://github.com/platformio/platformio-libmirror.git", + # "https://developer.mbed.org/users/simon/code/TextLCD/", + "https://github.com/knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163", + ], + ) + validate_cliresult(result) + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "PJON", + "PJON@src-79de467ebe19de18287becff0a1fb42d", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "platformio-libmirror", + "PubSubClient", + ] + assert set(items1) >= set(items2) + + +def test_install_duplicates( # pylint: disable=unused-argument + clirunner, validate_cliresult, without_internet +): + # registry + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://dl.registry.platformio.org/download/milesburton/library/DallasTemperature/3.8.1/DallasTemperature-3.8.1.tar.gz", + ], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + # archive + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "install", + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + ], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + # repository + result = clirunner.invoke( + cmd_lib, + ["-g", "install", "https://github.com/platformio/platformio-libmirror.git"], + ) + validate_cliresult(result) + assert "is already installed" in result.output + + +def test_global_lib_list(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["-g", "list"]) + validate_cliresult(result) + assert all( + [ + n in result.output + for n in ( + "Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", + "Version: 5.10.1", + "Source: git+https://github.com/gioblu/PJON.git#3.0", + "Version: 3.0.0+sha.1fb26fd", + ) + ] + ) + + result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) + assert all( + [ + n in result.output + for n in ( + "__pkg_dir", + '"__src_url": "git+https://github.com/gioblu/PJON.git#6.2"', + '"version": "5.10.1"', + ) + ] + ) + items1 = [i["name"] for i in json.loads(result.output)] + items2 = [ + "Adafruit BusIO", + "Adafruit PN532", + "ArduinoJson", + "ArduinoJson", + "ArduinoJson", + "ArduinoJson", + "AsyncMqttClient", + "AsyncTCP", + "DallasTemperature", + "ESP32WebServer", + "ESPAsyncTCP", + "NeoPixelBus", + "OneWire", + "PJON", + "PJON", + "platformio-libmirror", + "PubSubClient", + ] + assert sorted(items1) == sorted(items2) + + versions1 = [ + "{name}@{version}".format(**item) for item in json.loads(result.output) + ] + versions2 = [ + "ArduinoJson@5.8.2", + "ArduinoJson@5.10.1", + "AsyncMqttClient@0.8.2", + "NeoPixelBus@2.2.4", + "PJON@6.2.0+sha.07fe9aa", + "PJON@3.0.0+sha.1fb26fd", + "PubSubClient@2.6.0+sha.bef5814", + "Adafruit PN532@1.2.0", + ] + assert set(versions1) >= set(versions2) + + +def test_global_lib_update_check(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["-g", "update", "--dry-run", "--json-output"]) + validate_cliresult(result) + output = json.loads(result.output) + assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set(lib["name"] for lib in output) + + +def test_global_lib_update(clirunner, validate_cliresult): + # update library using package directory + result = clirunner.invoke( + cmd_lib, ["-g", "update", "NeoPixelBus", "--dry-run", "--json-output"] + ) + validate_cliresult(result) + oudated = json.loads(result.output) + assert len(oudated) == 1 + assert "__pkg_dir" in oudated[0] + result = clirunner.invoke(cmd_lib, ["-g", "update", oudated[0]["__pkg_dir"]]) + validate_cliresult(result) + assert "Removing NeoPixelBus @ 2.2.4" in result.output + + # update rest libraries + result = clirunner.invoke(cmd_lib, ["-g", "update"]) + validate_cliresult(result) + assert result.output.count("[Detached]") == 1 + assert result.output.count("[Up-to-date]") == 15 + + # update unknown library + result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + +def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_core): + # uninstall using package directory + result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) + validate_cliresult(result) + items = json.loads(result.output) + items = sorted(items, key=lambda item: item["__pkg_dir"]) + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) + validate_cliresult(result) + assert ("Removing %s" % items[0]["name"]) in result.output + + # uninstall the rest libraries + result = clirunner.invoke( + cmd_lib, + [ + "-g", + "uninstall", + "OneWire", + "https://github.com/bblanchon/ArduinoJson.git", + "ArduinoJson@!=5.6.7", + "Adafruit PN532", + ], + ) + validate_cliresult(result) + + items1 = [d.basename for d in isolated_pio_core.join("lib").listdir()] + items2 = [ + "ArduinoJson", + "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "AsyncMqttClient", + "AsyncTCP", + "ESP32WebServer", + "ESPAsyncTCP", + "NeoPixelBus", + "PJON", + "PJON@src-79de467ebe19de18287becff0a1fb42d", + "platformio-libmirror", + "PubSubClient", + "SomeLib", + ] + assert set(items1) == set(items2) + + # uninstall unknown library + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"]) + assert result.exit_code != 0 + assert isinstance(result.exception, UnknownPackageError) + + +def test_lib_show(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["show", "64"]) + validate_cliresult(result) + assert all([s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")]) + result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"]) + validate_cliresult(result) + assert "OneWire" in result.output + + +def test_lib_builtin(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["builtin"]) + validate_cliresult(result) + result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"]) + validate_cliresult(result) + + +def test_lib_stats(clirunner, validate_cliresult): + result = clirunner.invoke(cmd_lib, ["stats"]) + validate_cliresult(result) + assert all( + [ + s in result.output + for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show") + ] + ) + + result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) + validate_cliresult(result) + assert set( + [ + "dlweek", + "added", + "updated", + "topkeywords", + "dlmonth", + "dlday", + "lastkeywords", + ] + ) == set(json.loads(result.output).keys()) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index b2db5d83f9..cfb7fe3131 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import json -from platformio import exception from platformio.commands import platform as cli_platform +from platformio.package.exception import UnknownPackageError -def test_search_json_output(clirunner, validate_cliresult, isolated_pio_home): +def test_search_json_output(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_search, ["arduino", "--json-output"] ) @@ -39,37 +41,52 @@ def test_search_raw_output(clirunner, validate_cliresult): def test_install_unknown_version(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["atmelavr@99.99.99"]) assert result.exit_code != 0 - assert isinstance(result.exception, exception.UndefinedPackageVersion) + assert isinstance(result.exception, UnknownPackageError) def test_install_unknown_from_registry(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["unknown-platform"]) assert result.exit_code != 0 - assert isinstance(result.exception, exception.UnknownPackage) + assert isinstance(result.exception, UnknownPackageError) + + +# def test_install_incompatbile(clirunner, validate_cliresult, isolated_pio_core): +# result = clirunner.invoke( +# cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], +# ) +# assert result.exit_code != 0 +# assert isinstance(result.exception, IncompatiblePlatform) + + +def test_install_core_3_dev_platform(clirunner, validate_cliresult, isolated_pio_core): + result = clirunner.invoke( + cli_platform.platform_install, ["atmelavr@1.2.0", "--skip-default-package"], + ) + assert result.exit_code == 0 -def test_install_known_version(clirunner, validate_cliresult, isolated_pio_home): +def test_install_known_version(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, - ["atmelavr@1.2.0", "--skip-default-package", "--with-package", "tool-avrdude"], + ["atmelavr@2.0.0", "--skip-default-package", "--with-package", "tool-avrdude"], ) validate_cliresult(result) - assert "atmelavr @ 1.2.0" in result.output + assert "atmelavr @ 2.0.0" in result.output assert "Installing tool-avrdude @" in result.output - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 -def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home): +def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( cli_platform.platform_install, [ - "https://github.com/platformio/" "platform-espressif8266.git", + "https://github.com/platformio/platform-espressif8266.git", "--skip-default-package", ], ) validate_cliresult(result) assert "espressif8266" in result.output - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 def test_list_json_output(clirunner, validate_cliresult): @@ -88,28 +105,28 @@ def test_list_raw_output(clirunner, validate_cliresult): assert all([s in result.output for s in ("atmelavr", "espressif8266")]) -def test_update_check(clirunner, validate_cliresult, isolated_pio_home): +def test_update_check(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( - cli_platform.platform_update, ["--only-check", "--json-output"] + cli_platform.platform_update, ["--dry-run", "--json-output"] ) validate_cliresult(result) output = json.loads(result.output) assert len(output) == 1 assert output[0]["name"] == "atmelavr" - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert len(isolated_pio_core.join("packages").listdir()) == 1 -def test_update_raw(clirunner, validate_cliresult, isolated_pio_home): +def test_update_raw(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke(cli_platform.platform_update) validate_cliresult(result) - assert "Uninstalling atmelavr @ 1.2.0:" in result.output - assert "PlatformManager: Installing atmelavr @" in result.output - assert len(isolated_pio_home.join("packages").listdir()) == 1 + assert "Removing atmelavr @ 2.0.0" in result.output + assert "Platform Manager: Installing platformio/atmelavr @" in result.output + assert len(isolated_pio_core.join("packages").listdir()) == 2 -def test_uninstall(clirunner, validate_cliresult, isolated_pio_home): +def test_uninstall(clirunner, validate_cliresult, isolated_pio_core): result = clirunner.invoke( - cli_platform.platform_uninstall, ["atmelavr", "espressif8266"] + cli_platform.platform_uninstall, ["atmelavr@1.2.0", "atmelavr", "espressif8266"] ) validate_cliresult(result) - assert not isolated_pio_home.join("platforms").listdir() + assert not isolated_pio_core.join("platforms").listdir() diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index a201e7232e..e0a64a8cf0 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -12,26 +12,157 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import join +import os import pytest -from platformio import util +from platformio import proc +from platformio.commands.test.command import cli as cmd_test def test_local_env(): - result = util.exec_command( + result = proc.exec_command( [ "platformio", "test", "-d", - join("examples", "unit-testing", "calculator"), + os.path.join("examples", "unit-testing", "calculator"), "-e", "native", ] ) if result["returncode"] != 1: - pytest.fail(result) + pytest.fail(str(result)) + # pylint: disable=unsupported-membership-test assert all([s in result["err"] for s in ("PASSED", "IGNORED", "FAILED")]), result[ "out" ] + + +def test_multiple_env_build(clirunner, validate_cliresult, tmpdir): + + project_dir = tmpdir.mkdir("project") + project_dir.join("platformio.ini").write( + """ +[env:teensy31] +platform = teensy +framework = arduino +board = teensy31 + +[env:native] +platform = native + +[env:espressif8266] +platform = espressif8266 +framework = arduino +board = nodemcuv2 +""" + ) + + project_dir.mkdir("test").join("test_main.cpp").write( + """ +#include +#ifdef ARDUINO +void setup() +#else +int main() +#endif +{ + UNITY_BEGIN(); + UNITY_END(); + +} +void loop() {} +""" + ) + + result = clirunner.invoke( + cmd_test, ["-d", str(project_dir), "--without-testing", "--without-uploading"], + ) + + validate_cliresult(result) + assert "Multiple ways to build" not in result.output + + +def test_setup_teardown_are_compilable(clirunner, validate_cliresult, tmpdir): + + project_dir = tmpdir.mkdir("project") + project_dir.join("platformio.ini").write( + """ +[env:embedded] +platform = ststm32 +framework = stm32cube +board = nucleo_f401re +test_transport = custom + +[env:native] +platform = native + +""" + ) + + test_dir = project_dir.mkdir("test") + test_dir.join("test_main.c").write( + """ +#include +#include + +void setUp(){ + printf("setUp called"); +} +void tearDown(){ + printf("tearDown called"); +} + +void dummy_test(void) { + TEST_ASSERT_EQUAL(1, 1); +} + +int main() { + UNITY_BEGIN(); + RUN_TEST(dummy_test); + UNITY_END(); +} +""" + ) + + native_result = clirunner.invoke( + cmd_test, ["-d", str(project_dir), "-e", "native"], + ) + + test_dir.join("unittest_transport.h").write( + """ +#ifdef __cplusplus +extern "C" { +#endif + +void unittest_uart_begin(){} +void unittest_uart_putchar(char c){} +void unittest_uart_flush(){} +void unittest_uart_end(){} + +#ifdef __cplusplus +} +#endif +""" + ) + + embedded_result = clirunner.invoke( + cmd_test, + [ + "-d", + str(project_dir), + "--without-testing", + "--without-uploading", + "-e", + "embedded", + ], + ) + + validate_cliresult(native_result) + validate_cliresult(embedded_result) + + assert all(f in native_result.output for f in ("setUp called", "tearDown called")) + assert all( + "[FAILED]" not in out for out in (native_result.output, embedded_result.output) + ) diff --git a/tests/commands/test_update.py b/tests/commands/test_update.py index 9325e5017a..b9ecb5c186 100644 --- a/tests/commands/test_update.py +++ b/tests/commands/test_update.py @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + from platformio.commands.update import cli as cmd_update -def test_update(clirunner, validate_cliresult): - matches = ("Platform Manager", "Up-to-date", "Library Manager") +def test_update(clirunner, validate_cliresult, isolated_pio_core): + matches = ("Platform Manager", "Library Manager") result = clirunner.invoke(cmd_update, ["--only-check"]) validate_cliresult(result) assert all([m in result.output for m in matches]) diff --git a/tests/conftest.py b/tests/conftest.py index f0529146f2..d81f0e8ad4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,12 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import email +import imaplib import os +import time import pytest from click.testing import CliRunner -from platformio import util +from platformio.clients import http def pytest_configure(config): @@ -33,23 +36,85 @@ def decorator(result): return decorator -@pytest.fixture(scope="module") -def clirunner(): +@pytest.fixture(scope="session") +def clirunner(request): + backup_env_vars = { + "PLATFORMIO_WORKSPACE_DIR": {"new": None}, + } + for key, item in backup_env_vars.items(): + backup_env_vars[key]["old"] = os.environ.get(key) + if item["new"] is not None: + os.environ[key] = item["new"] + elif key in os.environ: + del os.environ[key] + + def fin(): + for key, item in backup_env_vars.items(): + if item["old"] is not None: + os.environ[key] = item["old"] + elif key in os.environ: + del os.environ[key] + + request.addfinalizer(fin) + return CliRunner() @pytest.fixture(scope="module") -def isolated_pio_home(request, tmpdir_factory): - home_dir = tmpdir_factory.mktemp(".platformio") - os.environ["PLATFORMIO_CORE_DIR"] = str(home_dir) +def isolated_pio_core(request, tmpdir_factory): + core_dir = tmpdir_factory.mktemp(".platformio") + os.environ["PLATFORMIO_CORE_DIR"] = str(core_dir) def fin(): del os.environ["PLATFORMIO_CORE_DIR"] request.addfinalizer(fin) - return home_dir + return core_dir @pytest.fixture(scope="function") def without_internet(monkeypatch): - monkeypatch.setattr(util, "_internet_on", lambda: False) + monkeypatch.setattr(http, "_internet_on", lambda: False) + + +@pytest.fixture +def receive_email(): # pylint:disable=redefined-outer-name, too-many-locals + def _receive_email(from_who): + test_email = os.environ.get("TEST_EMAIL_LOGIN") + test_password = os.environ.get("TEST_EMAIL_PASSWORD") + imap_server = os.environ.get("TEST_EMAIL_IMAP_SERVER") or "imap.gmail.com" + + def get_body(msg): + if msg.is_multipart(): + return get_body(msg.get_payload(0)) + return msg.get_payload(None, True) + + result = None + start_time = time.time() + while not result: + time.sleep(5) + server = imaplib.IMAP4_SSL(imap_server) + server.login(test_email, test_password) + server.select("INBOX") + _, mails = server.search(None, "ALL") + for index in mails[0].split(): + status, data = server.fetch(index, "(RFC822)") + if status != "OK" or not data or not isinstance(data[0], tuple): + continue + msg = email.message_from_string( + data[0][1].decode("ASCII", errors="surrogateescape") + ) + if from_who not in msg.get("To"): + continue + if "gmail" in imap_server: + server.store(index, "+X-GM-LABELS", "\\Trash") + server.store(index, "+FLAGS", "\\Deleted") + server.expunge() + result = get_body(msg).decode() + if time.time() - start_time > 120: + break + server.close() + server.logout() + return result + + return _receive_email diff --git a/tests/ino2cpp/__init__.py b/tests/ino2cpp/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/tests/ino2cpp/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/package/__init__.py b/tests/package/__init__.py new file mode 100644 index 0000000000..b051490361 --- /dev/null +++ b/tests/package/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py new file mode 100644 index 0000000000..299b0020ca --- /dev/null +++ b/tests/package/test_manager.py @@ -0,0 +1,453 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-argument + +import os +import time + +import pytest +import semantic_version + +from platformio import fs, util +from platformio.package.exception import ( + MissingPackageManifestError, + UnknownPackageError, +) +from platformio.package.manager.library import LibraryPackageManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.package.manager.tool import ToolPackageManager +from platformio.package.meta import PackageSpec +from platformio.package.pack import PackagePacker + + +def test_download(isolated_pio_core): + url = "https://github.com/platformio/platformio-core/archive/v4.3.4.zip" + checksum = "69d59642cb91e64344f2cdc1d3b98c5cd57679b5f6db7accc7707bd4c5d9664a" + lm = LibraryPackageManager() + archive_path = lm.download(url, checksum, silent=True) + assert fs.calculate_file_hashsum("sha256", archive_path) == checksum + lm.cleanup_expired_downloads() + assert os.path.isfile(archive_path) + # test outdated downloads + lm.set_download_utime(archive_path, time.time() - lm.DOWNLOAD_CACHE_EXPIRE - 1) + lm.cleanup_expired_downloads() + assert not os.path.isfile(archive_path) + # check that key is deleted from DB + with open(lm.get_download_usagedb_path()) as fp: + assert os.path.basename(archive_path) not in fp.read() + + +def test_find_pkg_root(isolated_pio_core, tmpdir_factory): + # has manifest + pkg_dir = tmpdir_factory.mktemp("package-has-manifest") + root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir() + root_dir.join("platform.json").write("") + pm = PlatformPackageManager() + found_dir = pm.find_pkg_root(str(pkg_dir), spec=None) + assert os.path.realpath(str(root_dir)) == os.path.realpath(found_dir) + + # does not have manifest + pkg_dir = tmpdir_factory.mktemp("package-does-not-have-manifest") + pkg_dir.join("nested").mkdir().join("folder").mkdir().join("readme.txt").write("") + pm = PlatformPackageManager() + with pytest.raises(MissingPackageManifestError): + pm.find_pkg_root(str(pkg_dir), spec=None) + + # library package without manifest, should find source root + pkg_dir = tmpdir_factory.mktemp("library-package-without-manifest") + root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir() + root_dir.join("src").mkdir().join("main.cpp").write("") + root_dir.join("include").mkdir().join("main.h").write("") + assert os.path.realpath(str(root_dir)) == os.path.realpath( + LibraryPackageManager.find_library_root(str(pkg_dir)) + ) + + # library manager should create "library.json" + lm = LibraryPackageManager() + spec = PackageSpec("custom-name@1.0.0") + pkg_root = lm.find_pkg_root(str(pkg_dir), spec) + manifest_path = os.path.join(pkg_root, "library.json") + assert os.path.realpath(str(root_dir)) == os.path.realpath(pkg_root) + assert os.path.isfile(manifest_path) + manifest = lm.load_manifest(pkg_root) + assert manifest["name"] == "custom-name" + assert "0.0.0" in str(manifest["version"]) + + +def test_build_legacy_spec(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + pm = PlatformPackageManager(str(storage_dir)) + # test src manifest + pkg1_dir = storage_dir.join("pkg-1").mkdir() + pkg1_dir.join(".pio").mkdir().join(".piopkgmanager.json").write( + """ +{ + "name": "StreamSpy-0.0.1.tar", + "url": "https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", + "requirements": null +} +""" + ) + assert pm.build_legacy_spec(str(pkg1_dir)) == PackageSpec( + name="StreamSpy-0.0.1.tar", + url="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz", + ) + + # without src manifest + pkg2_dir = storage_dir.join("pkg-2").mkdir() + pkg2_dir.join("main.cpp").write("") + with pytest.raises(MissingPackageManifestError): + pm.build_legacy_spec(str(pkg2_dir)) + + # with package manifest + pkg3_dir = storage_dir.join("pkg-3").mkdir() + pkg3_dir.join("platform.json").write('{"name": "pkg3", "version": "1.2.0"}') + assert pm.build_legacy_spec(str(pkg3_dir)) == PackageSpec(name="pkg3") + + +def test_build_metadata(isolated_pio_core, tmpdir_factory): + pm = PlatformPackageManager() + vcs_revision = "a2ebfd7c0f" + pkg_dir = tmpdir_factory.mktemp("package") + + # test package without manifest + with pytest.raises(MissingPackageManifestError): + pm.load_manifest(str(pkg_dir)) + with pytest.raises(MissingPackageManifestError): + pm.build_metadata(str(pkg_dir), PackageSpec("MyLib")) + + # with manifest + pkg_dir.join("platform.json").write( + '{"name": "Dev-Platform", "version": "1.2.3-alpha.1"}' + ) + metadata = pm.build_metadata(str(pkg_dir), PackageSpec("owner/platform-name")) + assert metadata.name == "Dev-Platform" + assert str(metadata.version) == "1.2.3-alpha.1" + + # with vcs + metadata = pm.build_metadata( + str(pkg_dir), PackageSpec("owner/platform-name"), vcs_revision + ) + assert str(metadata.version) == ("1.2.3-alpha.1+sha." + vcs_revision) + assert metadata.version.build[1] == vcs_revision + + +def test_install_from_url(isolated_pio_core, tmpdir_factory): + tmp_dir = tmpdir_factory.mktemp("tmp") + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + + # install from local directory + src_dir = tmp_dir.join("local-lib-dir").mkdir() + src_dir.join("main.cpp").write("") + spec = PackageSpec("file://%s" % src_dir) + pkg = lm.install(spec, silent=True) + assert os.path.isfile(os.path.join(pkg.path, "main.cpp")) + manifest = lm.load_manifest(pkg) + assert manifest["name"] == "local-lib-dir" + assert manifest["version"].startswith("0.0.0+") + assert spec == pkg.metadata.spec + + # install from local archive + src_dir = tmp_dir.join("archive-src").mkdir() + root_dir = src_dir.mkdir("root") + root_dir.mkdir("src").join("main.cpp").write("#include ") + root_dir.join("library.json").write( + '{"name": "manifest-lib-name", "version": "2.0.0"}' + ) + tarball_path = PackagePacker(str(src_dir)).pack(str(tmp_dir)) + spec = PackageSpec("file://%s" % tarball_path) + pkg = lm.install(spec, silent=True) + assert os.path.isfile(os.path.join(pkg.path, "src", "main.cpp")) + assert pkg == lm.get_package(spec) + assert spec == pkg.metadata.spec + + # install from registry + src_dir = tmp_dir.join("registry-1").mkdir() + src_dir.join("library.properties").write( + """ +name = wifilib +version = 5.2.7 +""" + ) + spec = PackageSpec("company/wifilib @ ^5") + pkg = lm.install_from_url("file://%s" % src_dir, spec) + assert str(pkg.metadata.version) == "5.2.7" + + # check package folder names + lm.memcache_reset() + assert ["local-lib-dir", "manifest-lib-name", "wifilib"] == [ + os.path.basename(pkg.path) for pkg in lm.get_installed() + ] + + +def test_install_from_registry(isolated_pio_core, tmpdir_factory): + # Libraries + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) + # library with dependencies + lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert len(lm.get_installed()) == 3 + pkg = lm.get_package("AsyncTCP-esphome") + assert pkg.metadata.spec.owner == "ottowinter" + assert not lm.get_package("non-existing-package") + # mbed library + assert lm.install("wolfSSL", silent=True) + assert len(lm.get_installed()) == 4 + # case sensitive author name + assert lm.install("DallasTemperature", silent=True) + assert lm.get_package("OneWire").metadata.version.major >= 2 + assert len(lm.get_installed()) == 6 + + # test conflicted names + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("conflicted-storage"))) + lm.install("4@2.6.1", silent=True) + lm.install("5357@2.6.1", silent=True) + assert len(lm.get_installed()) == 2 + + # Tools + tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) + pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True) + manifest = tm.load_manifest(pkg) + assert tm.is_system_compatible(manifest.get("system")) + assert util.get_systype() in manifest.get("system", []) + + # Test unknown + with pytest.raises(UnknownPackageError): + tm.install("unknown-package-tool @ 9.1.1", silent=True) + with pytest.raises(UnknownPackageError): + tm.install("owner/unknown-package-tool", silent=True) + + +def test_install_force(isolated_pio_core, tmpdir_factory): + lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) + # install #64 ArduinoJson + pkg = lm.install("64 @ ^5", silent=True) + assert pkg.metadata.version.major == 5 + # try install the latest without specification + pkg = lm.install("64", silent=True) + assert pkg.metadata.version.major == 5 + assert len(lm.get_installed()) == 1 + # re-install the latest + pkg = lm.install(64, silent=True, force=True) + assert len(lm.get_installed()) == 1 + assert pkg.metadata.version.major > 5 + + +def test_get_installed(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + pm = ToolPackageManager(str(storage_dir)) + + # VCS package + ( + storage_dir.join("pkg-vcs") + .mkdir() + .join(".git") + .mkdir() + .join(".piopm") + .write( + """ +{ + "name": "pkg-via-vcs", + "spec": { + "id": null, + "name": "pkg-via-vcs", + "owner": null, + "requirements": null, + "url": "git+https://github.com/username/repo.git" + }, + "type": "tool", + "version": "0.0.0+sha.1ea4d5e" +} +""" + ) + ) + + # package without metadata file + ( + storage_dir.join("foo@3.4.5") + .mkdir() + .join("package.json") + .write('{"name": "foo", "version": "3.4.5"}') + ) + + # package with metadata file + foo_dir = storage_dir.join("foo").mkdir() + foo_dir.join("package.json").write('{"name": "foo", "version": "3.6.0"}') + foo_dir.join(".piopm").write( + """ +{ + "name": "foo", + "spec": { + "name": "foo", + "owner": null, + "requirements": "^3" + }, + "type": "tool", + "version": "3.6.0" +} +""" + ) + + # test "system" + storage_dir.join("pkg-incompatible-system").mkdir().join("package.json").write( + '{"name": "check-system", "version": "4.0.0", "system": ["unknown"]}' + ) + storage_dir.join("pkg-compatible-system").mkdir().join("package.json").write( + '{"name": "check-system", "version": "3.0.0", "system": "%s"}' + % util.get_systype() + ) + + # invalid package + storage_dir.join("invalid-package").mkdir().join("library.json").write( + '{"name": "SomeLib", "version": "4.0.0"}' + ) + + installed = pm.get_installed() + assert len(installed) == 4 + assert set(["pkg-via-vcs", "foo", "check-system"]) == set( + p.metadata.name for p in installed + ) + assert str(pm.get_package("foo").metadata.version) == "3.6.0" + assert str(pm.get_package("check-system").metadata.version) == "3.0.0" + + +def test_uninstall(isolated_pio_core, tmpdir_factory): + tmp_dir = tmpdir_factory.mktemp("tmp") + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + + # foo @ 1.0.0 + pkg_dir = tmp_dir.join("foo").mkdir() + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.0.0"}') + foo_1_0_0_pkg = lm.install_from_url("file://%s" % pkg_dir, "foo") + # foo @ 1.3.0 + pkg_dir = tmp_dir.join("foo-1.3.0").mkdir() + pkg_dir.join("library.json").write('{"name": "foo", "version": "1.3.0"}') + lm.install_from_url("file://%s" % pkg_dir, "foo") + # bar + pkg_dir = tmp_dir.join("bar").mkdir() + pkg_dir.join("library.json").write('{"name": "bar", "version": "1.0.0"}') + bar_pkg = lm.install("file://%s" % pkg_dir, silent=True) + + assert len(lm.get_installed()) == 3 + assert os.path.isdir(os.path.join(str(storage_dir), "foo")) + assert os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) + + # check detaching + assert lm.uninstall("FOO", silent=True) + assert len(lm.get_installed()) == 2 + assert os.path.isdir(os.path.join(str(storage_dir), "foo")) + assert not os.path.isdir(os.path.join(str(storage_dir), "foo@1.0.0")) + + # uninstall the rest + assert lm.uninstall(foo_1_0_0_pkg.path, silent=True) + assert lm.uninstall(bar_pkg, silent=True) + + assert not lm.get_installed() + + # test uninstall dependencies + assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert len(lm.get_installed()) == 3 + assert lm.uninstall("AsyncMqttClient-esphome", silent=True, skip_dependencies=True) + assert len(lm.get_installed()) == 2 + + lm = LibraryPackageManager(str(storage_dir)) + assert lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + assert lm.uninstall("AsyncMqttClient-esphome", silent=True) + assert not lm.get_installed() + + +def test_registry(isolated_pio_core): + lm = LibraryPackageManager() + + # reveal ID + assert lm.reveal_registry_package_id(PackageSpec(id=13)) == 13 + assert lm.reveal_registry_package_id(PackageSpec(name="OneWire"), silent=True) == 1 + with pytest.raises(UnknownPackageError): + lm.reveal_registry_package_id(PackageSpec(name="/non-existing-package/")) + + # fetch package data + assert lm.fetch_registry_package(PackageSpec(id=1))["name"] == "OneWire" + assert lm.fetch_registry_package(PackageSpec(name="ArduinoJson"))["id"] == 64 + assert ( + lm.fetch_registry_package( + PackageSpec(id=13, owner="adafruit", name="Renamed library") + )["name"] + == "Adafruit GFX Library" + ) + with pytest.raises(UnknownPackageError): + lm.fetch_registry_package( + PackageSpec(owner="unknown<>owner", name="/non-existing-package/") + ) + with pytest.raises(UnknownPackageError): + lm.fetch_registry_package(PackageSpec(name="/non-existing-package/")) + + +def test_update_with_metadata(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + lm = LibraryPackageManager(str(storage_dir)) + + # test non SemVer in registry + pkg = lm.install("RadioHead @ <1.90", silent=True) + outdated = lm.outdated(pkg) + assert str(outdated.current) == "1.89.0" + assert outdated.latest > semantic_version.Version("1.100.0") + + pkg = lm.install("ArduinoJson @ 5.10.1", silent=True) + # tesy latest + outdated = lm.outdated(pkg) + assert str(outdated.current) == "5.10.1" + assert outdated.wanted is None + assert outdated.latest > outdated.current + assert outdated.latest > semantic_version.Version("5.99.99") + + # test wanted + outdated = lm.outdated(pkg, PackageSpec("ArduinoJson@~5")) + assert str(outdated.current) == "5.10.1" + assert str(outdated.wanted) == "5.13.4" + assert outdated.latest > semantic_version.Version("6.16.0") + + # update to the wanted 5.x + new_pkg = lm.update("ArduinoJson@^5", PackageSpec("ArduinoJson@^5"), silent=True) + assert str(new_pkg.metadata.version) == "5.13.4" + # check that old version is removed + assert len(lm.get_installed()) == 2 + + # update to the latest + lm = LibraryPackageManager(str(storage_dir)) + pkg = lm.update("ArduinoJson", silent=True) + assert pkg.metadata.version == outdated.latest + + +def test_update_without_metadata(isolated_pio_core, tmpdir_factory): + storage_dir = tmpdir_factory.mktemp("storage") + storage_dir.join("legacy-package").mkdir().join("library.json").write( + '{"name": "AsyncMqttClient-esphome", "version": "0.8"}' + ) + storage_dir.join("legacy-dep").mkdir().join("library.json").write( + '{"name": "AsyncTCP-esphome", "version": "1.1.1"}' + ) + lm = LibraryPackageManager(str(storage_dir)) + pkg = lm.get_package("AsyncMqttClient-esphome") + outdated = lm.outdated(pkg) + assert len(lm.get_installed()) == 2 + assert str(pkg.metadata.version) == "0.8.0" + assert outdated.latest > semantic_version.Version("0.8.0") + + # update + lm = LibraryPackageManager(str(storage_dir)) + new_pkg = lm.update(pkg, silent=True) + assert len(lm.get_installed()) == 3 + assert new_pkg.metadata.spec.owner == "ottowinter" diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 1ad66a7585..426cbdf17a 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -14,6 +14,7 @@ import os import re +import tarfile import jsondiff import pytest @@ -82,7 +83,7 @@ def test_library_json_parser(): }, "dependencies": [ {"name": "deps1", "version": "1.0.0"}, - {"name": "@owner/deps2", "version": "1.0.0", "frameworks": "arduino, espidf"}, + {"name": "@owner/deps2", "version": "1.0.0", "platforms": "*", "frameworks": "arduino, espidf"}, {"name": "deps3", "version": "1.0.0", "platforms": ["ststm32", "sifive"]} ] } @@ -100,6 +101,7 @@ def test_library_json_parser(): { "name": "@owner/deps2", "version": "1.0.0", + "platforms": ["*"], "frameworks": ["arduino", "espidf"], }, {"name": "deps1", "version": "1.0.0"}, @@ -170,7 +172,7 @@ def test_module_json_parser(): "name": "YottaLibrary", "description": "This is Yotta library", "homepage": "https://yottabuild.org", - "keywords": ["mbed", "Yotta"], + "keywords": ["mbed", "yotta"], "license": "Apache-2.0", "platforms": ["*"], "frameworks": ["mbed"], @@ -196,10 +198,12 @@ def test_library_properties_parser(): contents = """ name=TestPackage version=1.2.3 -author=SomeAuthor +author=SomeAuthor , Maintainer Author (nickname) +maintainer=Maintainer Author (nickname) sentence=This is Arduino library customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third +ignore_empty_field= """ raw_data = parser.LibraryPropertiesManifestParser(contents).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) @@ -215,7 +219,10 @@ def test_library_properties_parser(): "export": { "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, - "authors": [{"email": "info@author.com", "name": "SomeAuthor"}], + "authors": [ + {"name": "SomeAuthor", "email": "info@author.com"}, + {"name": "Maintainer Author", "maintainer": True}, + ], "keywords": ["uncategorized"], "customField": "Custom Value", "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third", @@ -276,7 +283,7 @@ def test_library_properties_parser(): # Author + Maintainer data = parser.LibraryPropertiesManifestParser( """ -author=Rocket Scream Electronics +author=Rocket Scream Electronics maintainer=Rocket Scream Electronics """ ).as_dict() @@ -478,7 +485,7 @@ def test_library_properties_schema(): contents = """ name=Mozzi version=1.0.3 -author=Tim Barrass and contributors as documented in source, and at https://github.com/sensorium/Mozzi/graphs/contributors +author=Lorem Ipsum is simply dummy text of the printing and typesetting industry Lorem Ipsum has been the industry's standard dummy text ever since the 1500s when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries but also the leap into electronic typesetting remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. maintainer=Tim Barrass sentence=Sound synthesis library for Arduino paragraph=With Mozzi, you can construct sounds using familiar synthesis units like oscillators, delays, filters and envelopes. @@ -497,6 +504,7 @@ def test_library_properties_schema(): ), ).as_dict() + errors = None try: ManifestSchema().load_manifest(raw_data) except ManifestValidationError as e: @@ -543,8 +551,8 @@ def test_platform_json_schema(): "name": "atmelavr", "title": "Atmel AVR", "description": "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of performance, power efficiency and design flexibility. Optimized to speed time to market-and easily adapt to new ones-they are based on the industrys most code-efficient architecture for C and assembly programming.", - "url": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", - "homepage": "http://platformio.org/platforms/atmelavr", + "keywords": "arduino, atmel, avr", + "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", "license": "Apache-2.0", "engines": { "platformio": "<5" @@ -602,7 +610,8 @@ def test_platform_json_schema(): "on the industrys most code-efficient architecture for C and " "assembly programming." ), - "homepage": "http://platformio.org/platforms/atmelavr", + "keywords": ["arduino", "atmel", "avr"], + "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", "license": "Apache-2.0", "repository": { "url": "https://github.com/platformio/platform-atmelavr.git", @@ -624,7 +633,9 @@ def test_package_json_schema(): { "name": "tool-scons", "description": "SCons software construction tool", - "url": "http://www.scons.org", + "keywords": "SCons, build", + "homepage": "http://www.scons.org", + "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l"], "version": "3.30101.0" } """ @@ -639,7 +650,9 @@ def test_package_json_schema(): { "name": "tool-scons", "description": "SCons software construction tool", + "keywords": ["scons", "build"], "homepage": "http://www.scons.org", + "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l"], "version": "3.30101.0", }, ) @@ -659,6 +672,20 @@ def test_package_json_schema(): ) assert mp.as_dict()["system"] == ["darwin_x86_64"] + # shortcut repository syntax (npm-style) + contents = """ +{ + "name": "tool-github", + "version": "1.2.0", + "repository": "github:user/repo" +} +""" + raw_data = parser.ManifestParserFactory.new( + contents, parser.ManifestFileType.PACKAGE_JSON + ).as_dict() + data = ManifestSchema().load_manifest(raw_data) + assert data["repository"]["url"] == "https://github.com/user/repo.git" + def test_parser_from_dir(tmpdir_factory): pkg_dir = tmpdir_factory.mktemp("package") @@ -728,7 +755,7 @@ def _to_unix_path(path): return re.sub(r"[\\/]+", "/", path) def _sort_examples(items): - for i, item in enumerate(items): + for i, _ in enumerate(items): items[i]["base"] = _to_unix_path(items[i]["base"]) items[i]["files"] = [_to_unix_path(f) for f in sorted(items[i]["files"])] return sorted(items, key=lambda item: item["name"]) @@ -790,6 +817,21 @@ def _sort_examples(items): ) +def test_parser_from_archive(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + pkg_dir.join("package.json").write('{"name": "package.json"}') + pkg_dir.join("library.json").write('{"name": "library.json"}') + pkg_dir.join("library.properties").write("name=library.properties") + + archive_path = os.path.join(str(pkg_dir), "package.tar.gz") + with tarfile.open(archive_path, mode="w|gz") as tf: + for item in os.listdir(str(pkg_dir)): + tf.add(os.path.join(str(pkg_dir), item), item) + + data = parser.ManifestParserFactory.new_from_archive(archive_path).as_dict() + assert data["name"] == "library.json" + + def test_broken_schemas(): # missing required field with pytest.raises( @@ -825,3 +867,7 @@ def test_broken_schemas(): version="1.2.3", ) ) + + # invalid package name + with pytest.raises(ManifestValidationError, match=("are not allowed")): + ManifestSchema().load_manifest(dict(name="C/C++ :library", version="1.2.3")) diff --git a/tests/package/test_meta.py b/tests/package/test_meta.py new file mode 100644 index 0000000000..0629f274fc --- /dev/null +++ b/tests/package/test_meta.py @@ -0,0 +1,301 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import jsondiff +import semantic_version + +from platformio.package.meta import ( + PackageMetaData, + PackageOutdatedResult, + PackageSpec, + PackageType, +) + + +def test_outdated_result(): + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0") + assert result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.5.4") + assert result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", wanted="1.2.3") + assert not result.is_outdated() + assert result.is_outdated(allow_incompatible=True) + result = PackageOutdatedResult(current="1.2.3", latest="2.0.0", detached=True) + assert not result.is_outdated() + assert not result.is_outdated(allow_incompatible=True) + + +def test_spec_owner(): + assert PackageSpec("alice/foo library") == PackageSpec( + owner="alice", name="foo library" + ) + spec = PackageSpec(" Bob / BarUpper ") + assert spec != PackageSpec(owner="BOB", name="BARUPPER") + assert spec.owner == "Bob" + assert spec.name == "BarUpper" + + +def test_spec_id(): + assert PackageSpec(13) == PackageSpec(id=13) + assert PackageSpec("20") == PackageSpec(id=20) + spec = PackageSpec("id=199") + assert spec == PackageSpec(id=199) + assert isinstance(spec.id, int) + + +def test_spec_name(): + assert PackageSpec("foo") == PackageSpec(name="foo") + assert PackageSpec(" bar-24 ") == PackageSpec(name="bar-24") + + +def test_spec_requirements(): + assert PackageSpec("foo@1.2.3") == PackageSpec(name="foo", requirements="1.2.3") + assert PackageSpec( + name="foo", requirements=semantic_version.Version("1.2.3") + ) == PackageSpec(name="foo", requirements="1.2.3") + assert PackageSpec("bar @ ^1.2.3") == PackageSpec(name="bar", requirements="^1.2.3") + assert PackageSpec("13 @ ~2.0") == PackageSpec(id=13, requirements="~2.0") + assert PackageSpec( + name="hello", requirements=semantic_version.SimpleSpec("~1.2.3") + ) == PackageSpec(name="hello", requirements="~1.2.3") + spec = PackageSpec("id=20 @ !=1.2.3,<2.0") + assert not spec.external + assert isinstance(spec.requirements, semantic_version.SimpleSpec) + assert semantic_version.Version("1.3.0-beta.1") in spec.requirements + assert spec == PackageSpec(id=20, requirements="!=1.2.3,<2.0") + + +def test_spec_local_urls(tmpdir_factory): + assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec( + url="file:///tmp/foo.tar.gz", name="foo" + ) + assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec( + url="file:///tmp/bar.zip", name="customName" + ) + assert PackageSpec("file:///tmp/some-lib/") == PackageSpec( + url="file:///tmp/some-lib/", name="some-lib" + ) + assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec( + url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1" + ) + # detached folder with "@" symbol + pkg_dir = tmpdir_factory.mktemp("storage").join("detached@1.2.3").mkdir() + assert PackageSpec("file://%s" % str(pkg_dir)) == PackageSpec( + name="detached", url="file://%s" % pkg_dir + ) + + +def test_spec_external_urls(): + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/develop.zip", + name="platformio-core", + ) + assert PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" + " @ !=2" + ) == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/" + "develop.zip?param=value", + name="platformio-core", + requirements="!=2", + ) + spec = PackageSpec( + "Custom-Name=" + "https://github.com/platformio/platformio-core/archive/develop.tar.gz@4.4.0" + ) + assert spec.external + assert spec.has_custom_name() + assert spec.name == "Custom-Name" + assert spec == PackageSpec( + url="https://github.com/platformio/platformio-core/archive/develop.tar.gz", + name="Custom-Name", + requirements="4.4.0", + ) + + +def test_spec_vcs_urls(): + assert PackageSpec("https://github.com/platformio/platformio-core") == PackageSpec( + name="platformio-core", url="git+https://github.com/platformio/platformio-core" + ) + assert PackageSpec("https://gitlab.com/username/reponame") == PackageSpec( + name="reponame", url="git+https://gitlab.com/username/reponame" + ) + assert PackageSpec( + "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) == PackageSpec( + name="wolfSSL", url="hg+https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) + assert PackageSpec( + "https://github.com/platformio/platformio-core.git#master" + ) == PackageSpec( + name="platformio-core", + url="git+https://github.com/platformio/platformio-core.git#master", + ) + assert PackageSpec( + "core=git+ssh://github.com/platformio/platformio-core.git#v4.4.0@4.4.0" + ) == PackageSpec( + name="core", + url="git+ssh://github.com/platformio/platformio-core.git#v4.4.0", + requirements="4.4.0", + ) + assert PackageSpec( + "username@github.com:platformio/platformio-core.git" + ) == PackageSpec( + name="platformio-core", + url="git+username@github.com:platformio/platformio-core.git", + ) + assert PackageSpec( + "pkg=git+git@github.com:platformio/platformio-core.git @ ^1.2.3,!=5" + ) == PackageSpec( + name="pkg", + url="git+git@github.com:platformio/platformio-core.git", + requirements="^1.2.3,!=5", + ) + + +def test_spec_as_dict(): + assert not jsondiff.diff( + PackageSpec("bob/foo@1.2.3").as_dict(), + { + "owner": "bob", + "id": None, + "name": "foo", + "requirements": "1.2.3", + "url": None, + }, + ) + assert not jsondiff.diff( + PackageSpec( + "https://github.com/platformio/platformio-core/archive/develop.zip?param=value" + " @ !=2" + ).as_dict(), + { + "owner": None, + "id": None, + "name": "platformio-core", + "requirements": "!=2", + "url": "https://github.com/platformio/platformio-core/archive/develop.zip?param=value", + }, + ) + + +def test_spec_as_dependency(): + assert PackageSpec("owner/pkgname").as_dependency() == "owner/pkgname" + assert PackageSpec(owner="owner", name="pkgname").as_dependency() == "owner/pkgname" + assert PackageSpec("bob/foo @ ^1.2.3").as_dependency() == "bob/foo@^1.2.3" + assert ( + PackageSpec( + "https://github.com/o/r/a/develop.zip?param=value @ !=2" + ).as_dependency() + == "https://github.com/o/r/a/develop.zip?param=value @ !=2" + ) + assert ( + PackageSpec( + "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ).as_dependency() + == "wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/" + ) + + +def test_metadata_as_dict(): + metadata = PackageMetaData(PackageType.LIBRARY, "foo", "1.2.3") + # test setter + metadata.version = "0.1.2+12345" + assert metadata.version == semantic_version.Version("0.1.2+12345") + assert not jsondiff.diff( + metadata.as_dict(), + { + "type": PackageType.LIBRARY, + "name": "foo", + "version": "0.1.2+12345", + "spec": None, + }, + ) + + assert not jsondiff.diff( + PackageMetaData( + PackageType.TOOL, + "toolchain", + "2.0.5", + PackageSpec("platformio/toolchain@~2.0.0"), + ).as_dict(), + { + "type": PackageType.TOOL, + "name": "toolchain", + "version": "2.0.5", + "spec": { + "owner": "platformio", + "id": None, + "name": "toolchain", + "requirements": "~2.0.0", + "url": None, + }, + }, + ) + + +def test_metadata_dump(tmpdir_factory): + pkg_dir = tmpdir_factory.mktemp("package") + metadata = PackageMetaData( + PackageType.TOOL, + "toolchain", + "2.0.5", + PackageSpec("platformio/toolchain@~2.0.0"), + ) + + dst = pkg_dir.join(".piopm") + metadata.dump(str(dst)) + assert os.path.isfile(str(dst)) + contents = dst.read() + assert all(s in contents for s in ("null", '"~2.0.0"')) + + +def test_metadata_load(tmpdir_factory): + contents = """ +{ + "name": "foo", + "spec": { + "name": "foo", + "owner": "username", + "requirements": "!=3.4.5" + }, + "type": "platform", + "version": "0.1.3" +} +""" + pkg_dir = tmpdir_factory.mktemp("package") + dst = pkg_dir.join(".piopm") + dst.write(contents) + metadata = PackageMetaData.load(str(dst)) + assert metadata.version == semantic_version.Version("0.1.3") + assert metadata == PackageMetaData( + PackageType.PLATFORM, + "foo", + "0.1.3", + spec=PackageSpec(owner="username", name="foo", requirements="!=3.4.5"), + ) + + piopm_path = pkg_dir.join(".piopm") + metadata = PackageMetaData( + PackageType.LIBRARY, "mylib", version="1.2.3", spec=PackageSpec("mylib") + ) + metadata.dump(str(piopm_path)) + restored_metadata = PackageMetaData.load(str(piopm_path)) + assert metadata == restored_metadata diff --git a/tests/package/test_pack.py b/tests/package/test_pack.py index 95b435700f..a964f5cdc0 100644 --- a/tests/package/test_pack.py +++ b/tests/package/test_pack.py @@ -61,7 +61,10 @@ def test_filters(tmpdir_factory): ) p = PackagePacker(str(pkg_dir)) with tarfile.open(p.pack(str(pkg_dir)), "r:gz") as tar: - assert set(tar.getnames()) == set(["util/helpers.cpp", "main.cpp"]) + assert set(tar.getnames()) == set( + ["util/helpers.cpp", "main.cpp", "library.json"] + ) + os.unlink(str(src_dir.join("library.json"))) # test include "src" and "include" pkg_dir.join("library.json").write( diff --git a/tests/test_examples.py b/tests/test_examples.py index c9f64c1880..994eb8c014 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -19,9 +19,10 @@ import pytest -from platformio import util +from platformio import fs, proc from platformio.compat import PY2 -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig @@ -34,8 +35,8 @@ def pytest_generate_tests(metafunc): examples_dirs.append(normpath(join(dirname(__file__), "..", "examples"))) # dev/platforms - for manifest in PlatformManager().get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + for pkg in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(pkg) examples_dir = join(p.get_dir(), "examples") assert isdir(examples_dir) examples_dirs.append(examples_dir) @@ -63,14 +64,14 @@ def pytest_generate_tests(metafunc): def test_run(pioproject_dir): - with util.cd(pioproject_dir): + with fs.cd(pioproject_dir): config = ProjectConfig() build_dir = config.get_optional_dir("build") if isdir(build_dir): - util.rmtree_(build_dir) + fs.rmtree(build_dir) env_names = config.envs() - result = util.exec_command( + result = proc.exec_command( ["platformio", "run", "-e", random.choice(env_names)] ) if result["returncode"] != 0: diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index c004f28f87..46bc82c803 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -12,17 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import json +import os import re from time import time from platformio import app, maintenance from platformio.__main__ import cli as cli_pio from platformio.commands import upgrade as cmd_upgrade -from platformio.managers.platform import PlatformManager -def test_check_pio_upgrade(clirunner, isolated_pio_home, validate_cliresult): +def test_check_pio_upgrade(clirunner, isolated_pio_core, validate_cliresult): def _patch_pio_version(version): maintenance.__version__ = version cmd_upgrade.VERSION = version.split(".", 3) @@ -51,7 +53,7 @@ def _patch_pio_version(version): _patch_pio_version(origin_version) -def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): +def test_check_lib_updates(clirunner, isolated_pio_core, validate_cliresult): # install obsolete library result = clirunner.invoke(cli_pio, ["lib", "-g", "install", "ArduinoJson@<6.13"]) validate_cliresult(result) @@ -66,7 +68,7 @@ def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): assert "There are the new updates for libraries (ArduinoJson)" in result.output -def test_check_and_update_libraries(clirunner, isolated_pio_home, validate_cliresult): +def test_check_and_update_libraries(clirunner, isolated_pio_core, validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_libraries", "Yes"] @@ -88,7 +90,10 @@ def test_check_and_update_libraries(clirunner, isolated_pio_home, validate_clire validate_cliresult(result) assert "There are the new updates for libraries (ArduinoJson)" in result.output assert "Please wait while updating libraries" in result.output - assert re.search(r"Updating ArduinoJson\s+@ 6.12.0\s+\[[\d\.]+\]", result.output) + assert re.search( + r"Updating bblanchon/ArduinoJson\s+6\.12\.0\s+\[Outdated [\d\.]+\]", + result.output, + ) # check updated version result = clirunner.invoke(cli_pio, ["lib", "-g", "list", "--json-output"]) @@ -96,16 +101,15 @@ def test_check_and_update_libraries(clirunner, isolated_pio_home, validate_clire assert prev_data[0]["version"] != json.loads(result.output)[0]["version"] -def test_check_platform_updates(clirunner, isolated_pio_home, validate_cliresult): +def test_check_platform_updates(clirunner, isolated_pio_core, validate_cliresult): # install obsolete platform result = clirunner.invoke(cli_pio, ["platform", "install", "native"]) validate_cliresult(result) - manifest_path = isolated_pio_home.join("platforms", "native", "platform.json") + os.remove(str(isolated_pio_core.join("platforms", "native", ".piopm"))) + manifest_path = isolated_pio_core.join("platforms", "native", "platform.json") manifest = json.loads(manifest_path.read()) manifest["version"] = "0.0.0" manifest_path.write(json.dumps(manifest)) - # reset cached manifests - PlatformManager().cache_reset() # reset check time interval = int(app.get_setting("check_platforms_interval")) * 3600 * 24 @@ -117,7 +121,7 @@ def test_check_platform_updates(clirunner, isolated_pio_home, validate_cliresult assert "There are the new updates for platforms (native)" in result.output -def test_check_and_update_platforms(clirunner, isolated_pio_home, validate_cliresult): +def test_check_and_update_platforms(clirunner, isolated_pio_core, validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_platforms", "Yes"] @@ -139,7 +143,7 @@ def test_check_and_update_platforms(clirunner, isolated_pio_home, validate_clire validate_cliresult(result) assert "There are the new updates for platforms (native)" in result.output assert "Please wait while updating platforms" in result.output - assert re.search(r"Updating native\s+@ 0.0.0\s+\[[\d\.]+\]", result.output) + assert re.search(r"Updating native\s+0.0.0\s+\[Outdated [\d\.]+\]", result.output) # check updated version result = clirunner.invoke(cli_pio, ["platform", "list", "--json-output"]) diff --git a/tests/test_managers.py b/tests/test_managers.py deleted file mode 100644 index f4ab2ed852..0000000000 --- a/tests/test_managers.py +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from os.path import join - -from platformio.managers.package import PackageManager -from platformio.project.helpers import get_project_core_dir - - -def test_pkg_input_parser(): - items = [ - ["PkgName", ("PkgName", None, None)], - [("PkgName", "!=1.2.3,<2.0"), ("PkgName", "!=1.2.3,<2.0", None)], - ["PkgName@1.2.3", ("PkgName", "1.2.3", None)], - [("PkgName@1.2.3", "1.2.5"), ("PkgName@1.2.3", "1.2.5", None)], - ["id=13", ("id=13", None, None)], - ["id=13@~1.2.3", ("id=13", "~1.2.3", None)], - [ - get_project_core_dir(), - (".platformio", None, "file://" + get_project_core_dir()), - ], - [ - "LocalName=" + get_project_core_dir(), - ("LocalName", None, "file://" + get_project_core_dir()), - ], - [ - "LocalName=%s@>2.3.0" % get_project_core_dir(), - ("LocalName", ">2.3.0", "file://" + get_project_core_dir()), - ], - [ - "https://github.com/user/package.git", - ("package", None, "git+https://github.com/user/package.git"), - ], - [ - "MyPackage=https://gitlab.com/user/package.git", - ("MyPackage", None, "git+https://gitlab.com/user/package.git"), - ], - [ - "MyPackage=https://gitlab.com/user/package.git@3.2.1,!=2", - ("MyPackage", "3.2.1,!=2", "git+https://gitlab.com/user/package.git"), - ], - [ - "https://somedomain.com/path/LibraryName-1.2.3.zip", - ( - "LibraryName-1.2.3", - None, - "https://somedomain.com/path/LibraryName-1.2.3.zip", - ), - ], - [ - "https://github.com/user/package/archive/branch.zip", - ("branch", None, "https://github.com/user/package/archive/branch.zip"), - ], - [ - "https://github.com/user/package/archive/branch.zip@~1.2.3", - ("branch", "~1.2.3", "https://github.com/user/package/archive/branch.zip"), - ], - [ - "https://github.com/user/package/archive/branch.tar.gz", - ( - "branch.tar", - None, - "https://github.com/user/package/archive/branch.tar.gz", - ), - ], - [ - "https://github.com/user/package/archive/branch.tar.gz@!=5", - ( - "branch.tar", - "!=5", - "https://github.com/user/package/archive/branch.tar.gz", - ), - ], - [ - "https://developer.mbed.org/users/user/code/package/", - ("package", None, "hg+https://developer.mbed.org/users/user/code/package/"), - ], - [ - "https://os.mbed.com/users/user/code/package/", - ("package", None, "hg+https://os.mbed.com/users/user/code/package/"), - ], - [ - "https://github.com/user/package#v1.2.3", - ("package", None, "git+https://github.com/user/package#v1.2.3"), - ], - [ - "https://github.com/user/package.git#branch", - ("package", None, "git+https://github.com/user/package.git#branch"), - ], - [ - "PkgName=https://github.com/user/package.git#a13d344fg56", - ("PkgName", None, "git+https://github.com/user/package.git#a13d344fg56"), - ], - ["user/package", ("package", None, "git+https://github.com/user/package")], - [ - "PkgName=user/package", - ("PkgName", None, "git+https://github.com/user/package"), - ], - [ - "PkgName=user/package#master", - ("PkgName", None, "git+https://github.com/user/package#master"), - ], - [ - "git+https://github.com/user/package", - ("package", None, "git+https://github.com/user/package"), - ], - [ - "hg+https://example.com/user/package", - ("package", None, "hg+https://example.com/user/package"), - ], - [ - "git@github.com:user/package.git", - ("package", None, "git+git@github.com:user/package.git"), - ], - [ - "git@github.com:user/package.git#v1.2.0", - ("package", None, "git+git@github.com:user/package.git#v1.2.0"), - ], - [ - "LocalName=git@github.com:user/package.git#v1.2.0@~1.2.0", - ("LocalName", "~1.2.0", "git+git@github.com:user/package.git#v1.2.0"), - ], - [ - "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", - ( - "package", - None, - "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", - ), - ], - [ - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ( - "package", - None, - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ), - ], - [ - "LocalName=git+ssh://user@gitlab.private-server.com:1234" - "/package#1.2.0@!=13", - ( - "LocalName", - "!=13", - "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0", - ), - ], - ] - for params, result in items: - if isinstance(params, tuple): - assert PackageManager.parse_pkg_uri(*params) == result - else: - assert PackageManager.parse_pkg_uri(params) == result - - -def test_install_packages(isolated_pio_home, tmpdir): - packages = [ - dict(id=1, name="name_1", version="shasum"), - dict(id=1, name="name_1", version="2.0.0"), - dict(id=1, name="name_1", version="2.1.0"), - dict(id=1, name="name_1", version="1.2"), - dict(id=1, name="name_1", version="1.0.0"), - dict(name="name_2", version="1.0.0"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - dict(name="name_2", version="3.0.0", __src_url="git+https://github2.com"), - dict(name="name_2", version="4.0.0", __src_url="git+https://github2.com"), - ] - - pm = PackageManager(join(get_project_core_dir(), "packages")) - for package in packages: - tmp_dir = tmpdir.mkdir("tmp-package") - tmp_dir.join("package.json").write(json.dumps(package)) - pm._install_from_url(package["name"], "file://%s" % str(tmp_dir)) - tmp_dir.remove(rec=1) - - assert len(pm.get_installed()) == len(packages) - 1 - - pkg_dirnames = [ - "name_1_ID1", - "name_1_ID1@1.0.0", - "name_1_ID1@1.2", - "name_1_ID1@2.0.0", - "name_1_ID1@shasum", - "name_2", - "name_2@src-177cbce1f0705580d17790fda1cc2ef5", - "name_2@src-f863b537ab00f4c7b5011fc44b120e1f", - ] - assert set( - [p.basename for p in isolated_pio_home.join("packages").listdir()] - ) == set(pkg_dirnames) - - -def test_get_package(): - tests = [ - [("unknown",), None], - [("1",), None], - [("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")], - [("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("name_1", "<2"), dict(id=1, name="name_1", version="1.2")], - [("name_1", ">2"), None], - [("name_1", "2-0-0"), None], - [("name_2",), dict(name="name_2", version="4.0.0")], - [ - ("url_has_higher_priority", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - ], - [ - ("name_2", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", __src_url="git+https://github.com"), - ], - ] - - pm = PackageManager(join(get_project_core_dir(), "packages")) - for test in tests: - manifest = pm.get_package(*test[0]) - if test[1] is None: - assert manifest is None, test - continue - for key, value in test[1].items(): - assert manifest[key] == value, test diff --git a/tests/test_misc.py b/tests/test_misc.py index aee9f113e3..36574ee426 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -12,31 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-argument + import pytest import requests -from platformio import exception, util +from platformio import __check_internet_hosts__, proc +from platformio.clients import http +from platformio.clients.registry import RegistryClient def test_platformio_cli(): - result = util.exec_command(["pio", "--help"]) + result = proc.exec_command(["pio", "--help"]) assert result["returncode"] == 0 + # pylint: disable=unsupported-membership-test assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result["out"] def test_ping_internet_ips(): - for host in util.PING_REMOTE_HOSTS: + for host in __check_internet_hosts__: requests.get("http://%s" % host, allow_redirects=False, timeout=2) -def test_api_internet_offline(without_internet, isolated_pio_home): - with pytest.raises(exception.InternetIsOffline): - util.get_api_result("/stats") +def test_api_internet_offline(without_internet, isolated_pio_core): + regclient = RegistryClient() + with pytest.raises(http.InternetIsOffline): + regclient.fetch_json_data("get", "/v2/stats") -def test_api_cache(monkeypatch, isolated_pio_home): - api_kwargs = {"url": "/stats", "cache_valid": "10s"} - result = util.get_api_result(**api_kwargs) +def test_api_cache(monkeypatch, isolated_pio_core): + regclient = RegistryClient() + api_kwargs = {"method": "get", "path": "/v2/stats", "cache_valid": "10s"} + result = regclient.fetch_json_data(**api_kwargs) assert result and "boards" in result - monkeypatch.setattr(util, "_internet_on", lambda: False) - assert util.get_api_result(**api_kwargs) == result + monkeypatch.setattr(http, "_internet_on", lambda: False) + assert regclient.fetch_json_data(**api_kwargs) == result diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 4d832c9754..832ceb716d 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=redefined-outer-name + import os import pytest @@ -345,7 +347,7 @@ def test_update_and_save(tmpdir_factory): ["check_types", [("float_option", 13.99), ("bool_option", True)]], ] ) - config.get("platformio", "extra_configs") == "extra.ini" + assert config.get("platformio", "extra_configs") == ["extra.ini"] config.remove_section("platformio") assert config.as_tuple() == [ ("env:myenv", [("board", "myboard"), ("framework", ["espidf", "arduino"])]), diff --git a/tox.ini b/tox.ini index 3db3a8ef64..0faae46b5e 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ passenv = * usedevelop = True deps = py36,py37,py38: black - isort + isort<5 pylint pytest pytest-xdist @@ -31,7 +31,7 @@ commands = [testenv:lint] commands = {envpython} --version - pylint --rcfile=./.pylintrc ./platformio + pylint --rcfile=./.pylintrc ./platformio ./tests [testenv:testcore] commands =