From b3c600c054761143703d67c8ba0afc791f13f267 Mon Sep 17 00:00:00 2001 From: Maciej Lech Date: Mon, 30 Mar 2020 00:56:36 +0200 Subject: [PATCH 01/91] Force adding group for command-line arguments (#306) * Force adding group to an option * Update tests --- nox/_option_set.py | 61 ++++++++++++++++++++----------------- nox/_options.py | 63 ++++++++++++++++++++++----------------- tests/test__option_set.py | 14 +++++++-- 3 files changed, 81 insertions(+), 57 deletions(-) diff --git a/nox/_option_set.py b/nox/_option_set.py index 79ae39bd..0c91e9cc 100644 --- a/nox/_option_set.py +++ b/nox/_option_set.py @@ -20,12 +20,27 @@ import argparse import collections import functools -from argparse import ArgumentError, ArgumentParser, Namespace, _ArgumentGroup -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from argparse import ArgumentError, ArgumentParser, Namespace +from typing import Any, Callable, List, Optional, Tuple, Union import argcomplete +class OptionGroup: + """A single group for command-line options. + + Args: + name (str): The name used to refer to the group. + args: Passed through to``ArgumentParser.add_argument_group``. + kwargs: Passed through to``ArgumentParser.add_argument_group``. + """ + + def __init__(self, name: str, *args: Any, **kwargs: Any) -> None: + self.name = name + self.args = args + self.kwargs = kwargs + + class Option: """A single option that can be specified via command-line or configuration file. @@ -35,8 +50,8 @@ class Option: object. flags (Sequence[str]): The list of flags used by argparse. Effectively the ``*args`` for ``ArgumentParser.add_argument``. + group (OptionGroup): The argument group this option belongs to. help (str): The help string pass to argparse. - group (str): The argument group this option belongs to, if any. noxfile (bool): Whether or not this option can be set in the configuration file. merge_func (Callable[[Namespace, Namespace], Any]): A function that @@ -61,8 +76,8 @@ def __init__( self, name: str, *flags: str, + group: OptionGroup, help: Optional[str] = None, - group: Optional[str] = None, noxfile: bool = False, merge_func: Optional[Callable[[Namespace, Namespace], Any]] = None, finalizer_func: Optional[Callable[[Any, Namespace], Any]] = None, @@ -73,8 +88,8 @@ def __init__( ) -> None: self.name = name self.flags = flags - self.help = help self.group = group + self.help = help self.noxfile = noxfile self.merge_func = merge_func self.finalizer_func = finalizer_func @@ -182,7 +197,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: ) # type: collections.OrderedDict[str, Option] self.groups = ( collections.OrderedDict() - ) # type: collections.OrderedDict[str, Tuple[Tuple[Any, ...], Dict[str, Any]]] + ) # type: collections.OrderedDict[str, OptionGroup] def add_options(self, *args: Option) -> None: """Adds a sequence of Options to the OptionSet. @@ -193,23 +208,14 @@ def add_options(self, *args: Option) -> None: for option in args: self.options[option.name] = option - def add_group(self, name: str, *args: Any, **kwargs: Any) -> None: - """Adds a new argument group. + def add_groups(self, *args: OptionGroup) -> None: + """Adds a sequence of OptionGroups to the OptionSet. - When :func:`parser` is invoked, the OptionSet will turn all distinct - argument groups into separate sections in the ``--help`` output using - ``ArgumentParser.add_argument_group``. + Args: + args (Sequence[OptionGroup]) """ - self.groups[name] = (args, kwargs) - - def _add_to_parser( - self, parser: Union[_ArgumentGroup, ArgumentParser], option: Option - ) -> None: - argument = parser.add_argument( - *option.flags, help=option.help, default=option.default, **option.kwargs - ) - if getattr(option, "completer"): - setattr(argument, "completer", option.completer) + for option_group in args: + self.groups[option_group.name] = option_group def parser(self) -> ArgumentParser: """Returns an ``ArgumentParser`` for this option set. @@ -220,18 +226,19 @@ def parser(self) -> ArgumentParser: parser = argparse.ArgumentParser(*self.parser_args, **self.parser_kwargs) groups = { - name: parser.add_argument_group(*args, **kwargs) - for name, (args, kwargs) in self.groups.items() + name: parser.add_argument_group(*option_group.args, **option_group.kwargs) + for name, option_group in self.groups.items() } for option in self.options.values(): if option.hidden: continue - if option.group is not None: - self._add_to_parser(groups[option.group], option) - else: - self._add_to_parser(parser, option) + argument = groups[option.group.name].add_argument( + *option.flags, help=option.help, default=option.default, **option.kwargs + ) + if getattr(option, "completer"): + setattr(argument, "completer", option.completer) return parser diff --git a/nox/_options.py b/nox/_options.py index 168890a9..782a512f 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -27,15 +27,17 @@ description="Nox is a Python automation toolkit.", add_help=False ) -options.add_group( - "primary", - "Primary arguments", - "These are the most common arguments used when invoking Nox.", -) -options.add_group( - "secondary", - "Additional arguments & flags", - "These arguments are used to control Nox's behavior or control advanced features.", +options.add_groups( + _option_set.OptionGroup( + "primary", + "Primary arguments", + "These are the most common arguments used when invoking Nox.", + ), + _option_set.OptionGroup( + "secondary", + "Additional arguments & flags", + "These arguments are used to control Nox's behavior or control advanced features.", + ), ) @@ -148,14 +150,14 @@ def _session_completer( "help", "-h", "--help", - group="primary", + group=options.groups["primary"], action="store_true", help="Show this help message and exit.", ), _option_set.Option( "version", "--version", - group="primary", + group=options.groups["primary"], action="store_true", help="Show the Nox version and exit.", ), @@ -164,7 +166,7 @@ def _session_completer( "-l", "--list-sessions", "--list", - group="primary", + group=options.groups["primary"], action="store_true", help="List all available sessions and exit.", ), @@ -174,7 +176,7 @@ def _session_completer( "-e", "--sessions", "--session", - group="primary", + group=options.groups["primary"], noxfile=True, merge_func=functools.partial(_session_filters_merge_func, "sessions"), nargs="*", @@ -187,7 +189,7 @@ def _session_completer( "-p", "--pythons", "--python", - group="primary", + group=options.groups["primary"], noxfile=True, merge_func=functools.partial(_session_filters_merge_func, "pythons"), nargs="*", @@ -197,6 +199,7 @@ def _session_completer( "keywords", "-k", "--keywords", + group=options.groups["primary"], noxfile=True, merge_func=functools.partial(_session_filters_merge_func, "keywords"), help="Only run sessions that match the given expression.", @@ -204,7 +207,7 @@ def _session_completer( _option_set.Option( "posargs", "posargs", - group="primary", + group=options.groups["primary"], nargs=argparse.REMAINDER, help="Arguments following ``--`` that are passed through to the session(s).", finalizer_func=_posargs_finalizer, @@ -213,7 +216,7 @@ def _session_completer( "verbose", "-v", "--verbose", - group="secondary", + group=options.groups["secondary"], action="store_true", help="Logs the output of all commands run including commands marked silent.", noxfile=True, @@ -222,14 +225,14 @@ def _session_completer( "reuse_existing_virtualenvs", ("-r", "--reuse-existing-virtualenvs"), ("--no-reuse-existing-virtualenvs",), - group="secondary", + group=options.groups["secondary"], help="Re-use existing virtualenvs instead of recreating them.", ), _option_set.Option( "noxfile", "-f", "--noxfile", - group="secondary", + group=options.groups["secondary"], default="noxfile.py", help="Location of the Python file containing nox sessions.", ), @@ -238,48 +241,48 @@ def _session_completer( "--envdir", noxfile=True, merge_func=_envdir_merge_func, - group="secondary", + group=options.groups["secondary"], help="Directory where nox will store virtualenvs, this is ``.nox`` by default.", ), *_option_set.make_flag_pair( "stop_on_first_error", ("-x", "--stop-on-first-error"), ("--no-stop-on-first-error",), - group="secondary", + group=options.groups["secondary"], help="Stop after the first error.", ), *_option_set.make_flag_pair( "error_on_missing_interpreters", ("--error-on-missing-interpreters",), ("--no-error-on-missing-interpreters",), - group="secondary", + group=options.groups["secondary"], help="Error instead of skipping sessions if an interpreter can not be located.", ), *_option_set.make_flag_pair( "error_on_external_run", ("--error-on-external-run",), ("--no-error-on-external-run",), - group="secondary", + group=options.groups["secondary"], help="Error if run() is used to execute a program that isn't installed in a session's virtualenv.", ), _option_set.Option( "install_only", "--install-only", - group="secondary", + group=options.groups["secondary"], action="store_true", help="Skip session.run invocations in the Noxfile.", ), _option_set.Option( "report", "--report", - group="secondary", + group=options.groups["secondary"], noxfile=True, help="Output a report of all sessions to the given filename.", ), _option_set.Option( "non_interactive", "--non-interactive", - group="secondary", + group=options.groups["secondary"], action="store_true", help="Force session.interactive to always be False, even in interactive sessions.", ), @@ -287,7 +290,7 @@ def _session_completer( "nocolor", "--nocolor", "--no-color", - group="secondary", + group=options.groups["secondary"], default=lambda: "NO_COLOR" in os.environ, action="store_true", help="Disable all color output.", @@ -296,13 +299,17 @@ def _session_completer( "forcecolor", "--forcecolor", "--force-color", - group="secondary", + group=options.groups["secondary"], default=False, action="store_true", help="Force color output, even if stdout is not an interactive terminal.", ), _option_set.Option( - "color", "--color", hidden=True, finalizer_func=_color_finalizer + "color", + "--color", + group=options.groups["secondary"], + hidden=True, + finalizer_func=_color_finalizer, ), ) diff --git a/tests/test__option_set.py b/tests/test__option_set.py index 3bcf4f21..3ad8b593 100644 --- a/tests/test__option_set.py +++ b/tests/test__option_set.py @@ -22,7 +22,12 @@ class TestOptionSet: def test_namespace(self): optionset = _option_set.OptionSet() - optionset.add_options(_option_set.Option("option_a", default="meep")) + optionset.add_groups(_option_set.OptionGroup("group_a")) + optionset.add_options( + _option_set.Option( + "option_a", group=optionset.groups["group_a"], default="meep" + ) + ) namespace = optionset.namespace() @@ -32,7 +37,12 @@ def test_namespace(self): def test_namespace_values(self): optionset = _option_set.OptionSet() - optionset.add_options(_option_set.Option("option_a", default="meep")) + optionset.add_groups(_option_set.OptionGroup("group_a")) + optionset.add_options( + _option_set.Option( + "option_a", group=optionset.groups["group_a"], default="meep" + ) + ) namespace = optionset.namespace(option_a="moop") From 9110f3e5a5cd3e5e0e51d972036f43cbd901672a Mon Sep 17 00:00:00 2001 From: Moshe Zadka Date: Fri, 15 May 2020 23:09:38 -0700 Subject: [PATCH 02/91] Add seasion.create_tmp (#320) * Address #319: Give a create_tmp API * more stuff * fade to black * woops * calculate tmp directly from envdir * cleanup and fix tests --- nox/sessions.py | 18 ++++++++++++++---- noxfile.py | 11 +++++++++-- tests/test_sessions.py | 19 +++++++++++++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/nox/sessions.py b/nox/sessions.py index 59a5cf4b..7fa77eca 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -135,6 +135,13 @@ def bin(self) -> Optional[str]: """The bin directory for the virtualenv.""" return self.virtualenv.bin + def create_tmp(self) -> str: + """Create, and return, a temporary directory.""" + tmpdir = os.path.join(self._runner.envdir, "tmp") + os.makedirs(tmpdir, exist_ok=True) + self.env["TMPDIR"] = tmpdir + return tmpdir + @property def interactive(self) -> bool: """Returns True if Nox is being run in an interactive session or False otherwise.""" @@ -388,33 +395,36 @@ def __str__(self) -> str: def friendly_name(self) -> str: return self.signatures[0] if self.signatures else self.name + @property + def envdir(self) -> str: + return _normalize_path(self.global_config.envdir, self.friendly_name) + def _create_venv(self) -> None: if self.func.python is False: self.venv = ProcessEnv() return - path = _normalize_path(self.global_config.envdir, self.friendly_name) reuse_existing = ( self.func.reuse_venv or self.global_config.reuse_existing_virtualenvs ) if not self.func.venv_backend or self.func.venv_backend == "virtualenv": self.venv = VirtualEnv( - path, + self.envdir, interpreter=self.func.python, # type: ignore reuse_existing=reuse_existing, venv_params=self.func.venv_params, ) elif self.func.venv_backend == "conda": self.venv = CondaEnv( - path, + self.envdir, interpreter=self.func.python, # type: ignore reuse_existing=reuse_existing, venv_params=self.func.venv_params, ) elif self.func.venv_backend == "venv": self.venv = VirtualEnv( - path, + self.envdir, interpreter=self.func.python, # type: ignore reuse_existing=reuse_existing, venv=True, diff --git a/noxfile.py b/noxfile.py index e20e5c1a..5bb786c2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools import os import nox @@ -30,6 +31,7 @@ def is_python_version(session, version): @nox.session(python=["3.5", "3.6", "3.7", "3.8"]) def tests(session): """Run test suite with pytest.""" + session.create_tmp() session.install("-r", "requirements-test.txt") session.install("-e", ".[tox_to_nox]") tests = session.posargs or ["tests/"] @@ -45,6 +47,7 @@ def tests(session): @nox.session(python=["3.5", "3.6", "3.7", "3.8"], venv_backend="conda") def conda_tests(session): """Run test suite with pytest.""" + session.create_tmp() session.conda_install( "--file", "requirements-conda-test.txt", "--channel", "conda-forge" ) @@ -93,11 +96,15 @@ def lint(session): @nox.session(python="3.8") def docs(session): """Build the documentation.""" - session.run("rm", "-rf", "docs/_build", external=True) + output_dir = os.path.join(session.create_tmp(), "output") + doctrees, html = map( + functools.partial(os.path.join, output_dir), ["doctrees", "html"] + ) + session.run("rm", "-rf", output_dir, external=True) session.install("-r", "requirements-test.txt") session.install(".") session.cd("docs") - sphinx_args = ["-b", "html", "-W", "-d", "_build/doctrees", ".", "_build/html"] + sphinx_args = ["-b", "html", "-W", "-d", doctrees, ".", html] if not session.interactive: sphinx_cmd = "sphinx-build" diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 8af3f858..f4c457fb 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -16,6 +16,7 @@ import logging import os import sys +import tempfile from unittest import mock import nox.command @@ -74,6 +75,24 @@ def make_session_and_runner(self): runner.venv.bin = "/no/bin/for/you" return nox.sessions.Session(runner=runner), runner + def test_create_tmp(self): + session, runner = self.make_session_and_runner() + with tempfile.TemporaryDirectory() as root: + runner.global_config.envdir = root + tmpdir = session.create_tmp() + assert session.env["TMPDIR"] == tmpdir + assert tmpdir.startswith(root) + + def test_create_tmp_twice(self): + session, runner = self.make_session_and_runner() + with tempfile.TemporaryDirectory() as root: + runner.global_config.envdir = root + runner.venv.bin = bin + session.create_tmp() + tmpdir = session.create_tmp() + assert session.env["TMPDIR"] == tmpdir + assert tmpdir.startswith(root) + def test_properties(self): session, runner = self.make_session_and_runner() From 3e4eaff1fa7e9c9db1a295aeaa3189f9b257377f Mon Sep 17 00:00:00 2001 From: Kevin Kirsche Date: Wed, 20 May 2020 18:29:27 -0400 Subject: [PATCH 03/91] Typo - formater to formatter (#324) Docstring typo --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 5bb786c2..84845c5f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -71,7 +71,7 @@ def cover(session): @nox.session(python="3.8") def blacken(session): - """Run black code formater.""" + """Run black code formatter.""" session.install("black==19.3b0", "isort==4.3.21") files = ["nox", "tests", "noxfile.py", "setup.py"] session.run("black", *files) From 7f00d107905a20c554de0078ae15dcde5a79b2ac Mon Sep 17 00:00:00 2001 From: Kevin Kirsche Date: Wed, 20 May 2020 18:30:26 -0400 Subject: [PATCH 04/91] Fix overwritten and environment typos in virtualenv.py (#326) Fix overwritten and environment typos in virtualenv.py --- nox/virtualenv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 918525e4..ea5ad6a0 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -66,7 +66,7 @@ def bin(self) -> Optional[str]: return self._bin def create(self) -> bool: - raise NotImplementedError("ProcessEnv.create should be overwitten in subclass") + raise NotImplementedError("ProcessEnv.create should be overwritten in subclass") def locate_via_py(version: str) -> Optional[str]: @@ -142,7 +142,7 @@ def _clean_location(self: "Union[CondaEnv, VirtualEnv]") -> bool: class CondaEnv(ProcessEnv): - """Conda environemnt management class. + """Conda environment management class. Args: location (str): The location on the filesystem where the conda environment From 9d2788d80fba39bbb9ce59e5474abb2a733d6649 Mon Sep 17 00:00:00 2001 From: Kevin Kirsche Date: Wed, 20 May 2020 18:35:37 -0400 Subject: [PATCH 05/91] Typo: Controling -> Controlling (#325) * Typo: Controling -> Controlling Typo in usage.rst * Update usage.rst Co-authored-by: Thea Flowers --- docs/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 6f0e3418..c942de79 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -268,8 +268,8 @@ However, this will never output colorful logs: .. _opt-report: -Controling commands verbosity ------------------------------ +Controlling commands verbosity +------------------------------ By default, Nox will only show output of commands that fail, or, when the commands get passed ``silent=False``. By passing ``--verbose`` to Nox, all output of all commands run is shown, regardless of the silent argument. From 271ff1422041fd7629e453132b377c036e535eb5 Mon Sep 17 00:00:00 2001 From: smarie Date: Sun, 24 May 2020 07:40:35 +0200 Subject: [PATCH 06/91] `venv_backend` new options and choices (#316) * New global option `nox.options.venv_backend` to set the default backend. Fixes #315 * Added doc about the new option * Blackened * fixed tests * Fixed docs * Fixed coverage by adding tests for the venv_backend_completer * fixed test * Added tests for short and long versions of the new option. * Replaced the venv_backend completer with a simple `choices` from argparse :) * Renamed venv_backend to default_venv_backend, and created new force_venv_backend * New "none" choice for venv_backends, equivalent to python=False * Updated doc concerning default_venv_backend and force_venv_backend, as well as the new 'none' backend * Fixed all manifest tests * Fixed test_tasks for venv_backend * Fixed coverage * Blackened code * The warning message was appearing for all sessions, even those deselected. It is now only logged when session is run. * Added `--no-venv` option. Fixes #301 * Blackened * Fixed tests * Improved coverage * Blackened * Fixed an issue with parametrization: warning would not be issued. Added corresponding tests. This should make coverage happy, too. * Blackened * Now `install` and `conda_install` work when there is no venv backend (or python=False). Previously a `ValueError` was raised. Fixes #318 * Fixed test * Minor edit to trigger CI build again as it seems stuck. * Minor doc fix to trigger the CI again (appveyor false fail) Co-authored-by: Sylvain MARIE --- docs/config.rst | 4 +- docs/usage.rst | 41 +++++++++++++- nox/_decorators.py | 6 ++- nox/_options.py | 75 ++++++++++++++++++++++++++ nox/manifest.py | 15 ++++++ nox/sessions.py | 41 ++++++++------ nox/tasks.py | 10 +++- nox/virtualenv.py | 10 ++++ tests/resources/noxfile_pythons.py | 7 +++ tests/test_main.py | 68 +++++++++++++++++++++++- tests/test_manifest.py | 85 +++++++++++++++++++++--------- tests/test_sessions.py | 1 + tests/test_tasks.py | 25 ++++++++- 13 files changed, 339 insertions(+), 49 deletions(-) create mode 100644 tests/resources/noxfile_pythons.py diff --git a/docs/config.rst b/docs/config.rst index e2a83386..39628b08 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -132,7 +132,7 @@ Will produce these sessions: Note that this expansion happens *before* parameterization occurs, so you can still parametrize sessions with multiple interpreters. -If you want to disable virtualenv creation altogether, you can set ``python`` to ``False``: +If you want to disable virtualenv creation altogether, you can set ``python`` to ``False``, or set ``venv_backend`` to ``"none"``, both are equivalent. Note that this can be done temporarily through the :ref:`--no-venv ` commandline flag, too. .. code-block:: python @@ -375,6 +375,8 @@ The following options can be specified in the Noxfile: * ``nox.options.sessions`` is equivalent to specifying :ref:`-s or --sessions `. * ``nox.options.pythons`` is equivalent to specifying :ref:`-p or --pythons `. * ``nox.options.keywords`` is equivalent to specifying :ref:`-k or --keywords `. +* ``nox.options.default_venv_backend`` is equivalent to specifying :ref:`-db or --default-venv-backend `. +* ``nox.options.force_venv_backend`` is equivalent to specifying :ref:`-fb or --force-venv-backend `. * ``nox.options.reuse_existing_virtualenvs`` is equivalent to specifying :ref:`--reuse-existing-virtualenvs `. You can force this off by specifying ``--no-reuse-existing-virtualenvs`` during invocation. * ``nox.options.stop_on_first_error`` is equivalent to specifying :ref:`--stop-on-first-error `. You can force this off by specifying ``--no-stop-on-first-error`` during invocation. * ``nox.options.error_on_missing_interpreters`` is equivalent to specifying :ref:`--error-on-missing-interpreters `. You can force this off by specifying ``--no-error-on-missing-interpreters`` during invocation. diff --git a/docs/usage.rst b/docs/usage.rst index c942de79..ef242260 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -105,12 +105,51 @@ Then running ``nox --session tests`` will actually run all parametrized versions nox --session "tests(django='2.0')" +.. _opt-default-venv-backend: + +Changing the sessions default backend +------------------------------------- + +By default nox uses ``virtualenv`` as the virtual environment backend for the sessions, but it also supports ``conda`` and ``venv`` as well as no backend (passthrough to whatever python environment nox is running on). You can change the default behaviour by using ``-db `` or ``--default-venv-backend ``. Supported names are ``('none', 'virtualenv', 'conda', 'venv')``. + +.. code-block:: console + + nox -db conda + nox --default-venv-backend conda + + +You can also set this option in the Noxfile with ``nox.options.default_venv_backend``. In case both are provided, the commandline argument takes precedence. + +Note that using this option does not change the backend for sessions where ``venv_backend`` is explicitly set. + + +.. _opt-force-venv-backend: + +Forcing the sessions backend +---------------------------- + +You might work in a different environment than a project's default continuous integration setttings, and might wish to get a quick way to execute the same tasks but on a different venv backend. For this purpose, you can temporarily force the backend used by **all** sessions in the current nox execution by using ``-fb `` or ``--force-venv-backend ``. No exceptions are made, the backend will be forced for all sessions run whatever the other options values and nox file configuration. Supported names are ``('none', 'virtualenv', 'conda', 'venv')``. + +.. code-block:: console + + nox -fb conda + nox --force-venv-backend conda + + +You can also set this option in the Noxfile with ``nox.options.force_venv_backend``. In case both are provided, the commandline argument takes precedence. + +Finally note that the ``--no-venv`` flag is a shortcut for ``--force-venv-backend none`` and allows to temporarily run all selected sessions on the current python interpreter (the one running nox). + +.. code-block:: console + + nox --no-venv + .. _opt-reuse-existing-virtualenvs: Re-using virtualenvs -------------------- -By default nox deletes and recreates virtualenvs every time it is run. This is usually fine for most projects and continuous integration environments as `pip's caching `_ makes re-install rather quick. However, there are some situations where it is advantageous to re-use the virtualenvs between runs. Use ``-r`` or ``--reuse-existing-virtualenvs``: +By default, Nox deletes and recreates virtualenvs every time it is run. This is usually fine for most projects and continuous integration environments as `pip's caching `_ makes re-install rather quick. However, there are some situations where it is advantageous to re-use the virtualenvs between runs. Use ``-r`` or ``--reuse-existing-virtualenvs``: .. code-block:: console diff --git a/nox/_decorators.py b/nox/_decorators.py index 32773f83..a4f031f5 100644 --- a/nox/_decorators.py +++ b/nox/_decorators.py @@ -1,7 +1,7 @@ import copy import functools import types -from typing import Any, Callable, Iterable, List, Optional, cast +from typing import Any, Callable, Iterable, List, Dict, Optional, cast from . import _typing @@ -40,12 +40,14 @@ def __init__( name: Optional[str] = None, venv_backend: Any = None, venv_params: Any = None, + should_warn: Dict[str, Any] = None, ): self.func = func self.python = python self.reuse_venv = reuse_venv self.venv_backend = venv_backend self.venv_params = venv_params + self.should_warn = should_warn or dict() def __call__(self, *args: Any, **kwargs: Any) -> Any: return self.func(*args, **kwargs) @@ -58,6 +60,7 @@ def copy(self, name: str = None) -> "Func": name, self.venv_backend, self.venv_params, + self.should_warn, ) @@ -70,6 +73,7 @@ def __init__(self, func: Func, param_spec: "Param") -> None: None, func.venv_backend, func.venv_params, + func.should_warn, ) self.param_spec = param_spec self.session_signature = "({})".format(param_spec) diff --git a/nox/_options.py b/nox/_options.py index 782a512f..a46943a5 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -60,6 +60,49 @@ def _session_filters_merge_func( return getattr(command_args, key) +def _default_venv_backend_merge_func( + command_args: argparse.Namespace, noxfile_args: argparse.Namespace +) -> str: + """Merge default_venv_backend from command args and nox file. Default is "virtualenv". + + Args: + command_args (_option_set.Namespace): The options specified on the + command-line. + noxfile_Args (_option_set.Namespace): The options specified in the + Noxfile. + """ + return ( + command_args.default_venv_backend + or noxfile_args.default_venv_backend + or "virtualenv" + ) + + +def _force_venv_backend_merge_func( + command_args: argparse.Namespace, noxfile_args: argparse.Namespace +) -> str: + """Merge force_venv_backend from command args and nox file. Default is None. + + Args: + command_args (_option_set.Namespace): The options specified on the + command-line. + noxfile_Args (_option_set.Namespace): The options specified in the + Noxfile. + """ + if command_args.no_venv: + if ( + command_args.force_venv_backend is not None + and command_args.force_venv_backend != "none" + ): + raise ValueError( + "You can not use `--no-venv` with a non-none `--force-venv-backend`" + ) + else: + return "none" + else: + return command_args.force_venv_backend or noxfile_args.force_venv_backend + + def _envdir_merge_func( command_args: argparse.Namespace, noxfile_args: argparse.Namespace ) -> str: @@ -221,6 +264,38 @@ def _session_completer( help="Logs the output of all commands run including commands marked silent.", noxfile=True, ), + _option_set.Option( + "default_venv_backend", + "-db", + "--default-venv-backend", + group=options.groups["secondary"], + noxfile=True, + merge_func=_default_venv_backend_merge_func, + help="Virtual environment backend to use by default for nox sessions, this is ``'virtualenv'`` by default but " + "any of ``('virtualenv', 'conda', 'venv')`` are accepted.", + choices=["none", "virtualenv", "conda", "venv"], + ), + _option_set.Option( + "force_venv_backend", + "-fb", + "--force-venv-backend", + group=options.groups["secondary"], + noxfile=True, + merge_func=_force_venv_backend_merge_func, + help="Virtual environment backend to force-use for all nox sessions in this run, overriding any other venv " + "backend declared in the nox file and ignoring the default backend. Any of ``('virtualenv', 'conda', 'venv')`` " + "are accepted.", + choices=["none", "virtualenv", "conda", "venv"], + ), + _option_set.Option( + "no_venv", + "--no-venv", + group=options.groups["secondary"], + default=False, + action="store_true", + help="Runs the selected sessions directly on the current interpreter, without creating a venv. This is an alias " + "for '--force-venv-backend none'.", + ), *_option_set.make_flag_pair( "reuse_existing_virtualenvs", ("-r", "--reuse-existing-virtualenvs"), diff --git a/nox/manifest.py b/nox/manifest.py index d90ba346..f2a008f0 100644 --- a/nox/manifest.py +++ b/nox/manifest.py @@ -21,6 +21,9 @@ from nox.sessions import Session, SessionRunner +WARN_PYTHONS_IGNORED = "python_ignored" + + class Manifest: """Session manifest. @@ -170,6 +173,18 @@ def make_session( """ sessions = [] + # if backend is none we wont parametrize the pythons + backend = ( + self._config.force_venv_backend + or func.venv_backend + or self._config.default_venv_backend + ) + if backend == "none" and isinstance(func.python, (list, tuple, set)): + # we can not log a warning here since the session is maybe deselected. + # instead let's set a flag, to warn later when session is actually run. + func.should_warn[WARN_PYTHONS_IGNORED] = func.python + func.python = False + # If the func has the python attribute set to a list, we'll need # to expand them. if isinstance(func.python, (list, tuple, set)): diff --git a/nox/sessions.py b/nox/sessions.py index 7fa77eca..c193162d 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -24,6 +24,7 @@ Callable, Dict, Iterable, + Tuple, List, Mapping, Optional, @@ -36,7 +37,7 @@ from nox import _typing from nox._decorators import Func from nox.logger import logger -from nox.virtualenv import CondaEnv, ProcessEnv, VirtualEnv +from nox.virtualenv import CondaEnv, ProcessEnv, VirtualEnv, PassthroughEnv if _typing.TYPE_CHECKING: from nox.manifest import Manifest @@ -279,10 +280,15 @@ def conda_install(self, *args: str, **kwargs: Any) -> None: .. _conda install: """ venv = self._runner.venv - if not isinstance(venv, CondaEnv): + + prefix_args = () # type: Tuple[str, ...] + if isinstance(venv, CondaEnv): + prefix_args = ("--prefix", venv.location) + elif not isinstance(venv, PassthroughEnv): # pragma: no cover raise ValueError( "A session without a conda environment can not install dependencies from conda." ) + if not args: raise ValueError("At least one argument required to install().") @@ -290,14 +296,7 @@ def conda_install(self, *args: str, **kwargs: Any) -> None: kwargs["silent"] = True self._run( - "conda", - "install", - "--yes", - "--prefix", - venv.location, - *args, - external="error", - **kwargs + "conda", "install", "--yes", *prefix_args, *args, external="error", **kwargs ) def install(self, *args: str, **kwargs: Any) -> None: @@ -325,7 +324,9 @@ def install(self, *args: str, **kwargs: Any) -> None: .. _pip: https://pip.readthedocs.org """ - if not isinstance(self._runner.venv, (CondaEnv, VirtualEnv)): + if not isinstance( + self._runner.venv, (CondaEnv, VirtualEnv, PassthroughEnv) + ): # pragma: no cover raise ValueError( "A session without a virtualenv can not install dependencies." ) @@ -400,29 +401,35 @@ def envdir(self) -> str: return _normalize_path(self.global_config.envdir, self.friendly_name) def _create_venv(self) -> None: - if self.func.python is False: - self.venv = ProcessEnv() + backend = ( + self.global_config.force_venv_backend + or self.func.venv_backend + or self.global_config.default_venv_backend + ) + + if backend == "none" or self.func.python is False: + self.venv = PassthroughEnv() return reuse_existing = ( self.func.reuse_venv or self.global_config.reuse_existing_virtualenvs ) - if not self.func.venv_backend or self.func.venv_backend == "virtualenv": + if backend is None or backend == "virtualenv": self.venv = VirtualEnv( self.envdir, interpreter=self.func.python, # type: ignore reuse_existing=reuse_existing, venv_params=self.func.venv_params, ) - elif self.func.venv_backend == "conda": + elif backend == "conda": self.venv = CondaEnv( self.envdir, interpreter=self.func.python, # type: ignore reuse_existing=reuse_existing, venv_params=self.func.venv_params, ) - elif self.func.venv_backend == "venv": + elif backend == "venv": self.venv = VirtualEnv( self.envdir, interpreter=self.func.python, # type: ignore @@ -433,7 +440,7 @@ def _create_venv(self) -> None: else: raise ValueError( "Expected venv_backend one of ('virtualenv', 'conda', 'venv'), but got '{}'.".format( - self.func.venv_backend + backend ) ) diff --git a/nox/tasks.py b/nox/tasks.py index 3cbbf3ef..bd5e1c37 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -24,7 +24,7 @@ from colorlog.escape_codes import parse_colors from nox import _options, registry from nox.logger import logger -from nox.manifest import Manifest +from nox.manifest import Manifest, WARN_PYTHONS_IGNORED from nox.sessions import Result @@ -233,6 +233,14 @@ def run_manifest(manifest: Manifest, global_config: Namespace) -> List[Result]: # Note that it is possible for the manifest to be altered in any given # iteration. for session in manifest: + # possibly raise warnings associated with this session + if WARN_PYTHONS_IGNORED in session.func.should_warn: + logger.warning( + "Session {} is set to run with venv_backend='none', IGNORING its python={} parametrization. ".format( + session.name, session.func.should_warn[WARN_PYTHONS_IGNORED] + ) + ) + result = session.execute() result.log( "Session {name} {status}.".format( diff --git a/nox/virtualenv.py b/nox/virtualenv.py index ea5ad6a0..fea02381 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -141,6 +141,16 @@ def _clean_location(self: "Union[CondaEnv, VirtualEnv]") -> bool: return True +class PassthroughEnv(ProcessEnv): + """Represents the environment used to run nox itself + + For now, this class is empty but it might contain tools to grasp some + hints about the actual env. + """ + + pass + + class CondaEnv(ProcessEnv): """Conda environment management class. diff --git a/tests/resources/noxfile_pythons.py b/tests/resources/noxfile_pythons.py new file mode 100644 index 00000000..bfba5a08 --- /dev/null +++ b/tests/resources/noxfile_pythons.py @@ -0,0 +1,7 @@ +import nox + + +@nox.session(python=["3.6"]) +@nox.parametrize("cheese", ["cheddar", "jack", "brie"]) +def snack(unused_session, cheese): + print("Noms, {} so good!".format(cheese)) diff --git a/tests/test_main.py b/tests/test_main.py index 78002581..c9ef5467 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -55,6 +55,7 @@ def test_main_no_args(monkeypatch): config = execute.call_args[1]["global_config"] assert config.noxfile == "noxfile.py" assert config.sessions is None + assert not config.no_venv assert not config.reuse_existing_virtualenvs assert not config.stop_on_first_error assert config.posargs == [] @@ -70,6 +71,11 @@ def test_main_long_form_args(): "--sessions", "1", "2", + "--default-venv-backend", + "venv", + "--force-venv-backend", + "none", + "--no-venv", "--reuse-existing-virtualenvs", "--stop-on-first-error", ] @@ -87,14 +93,72 @@ def test_main_long_form_args(): assert config.noxfile == "noxfile.py" assert config.envdir.endswith(".other") assert config.sessions == ["1", "2"] + assert config.default_venv_backend == "venv" + assert config.force_venv_backend == "none" + assert config.no_venv is True assert config.reuse_existing_virtualenvs is True assert config.stop_on_first_error is True assert config.posargs == [] +def test_main_no_venv(monkeypatch, capsys): + # Check that --no-venv overrides force_venv_backend + monkeypatch.setattr( + sys, + "argv", + [ + "nox", + "--noxfile", + os.path.join(RESOURCES, "noxfile_pythons.py"), + "--no-venv", + "-s", + "snack(cheese='cheddar')", + ], + ) + + with mock.patch("sys.exit") as sys_exit: + nox.__main__.main() + stdout, stderr = capsys.readouterr() + assert stdout == "Noms, cheddar so good!\n" + assert ( + "Session snack is set to run with venv_backend='none', IGNORING its python" + in stderr + ) + assert "Session snack(cheese='cheddar') was successful." in stderr + sys_exit.assert_called_once_with(0) + + +def test_main_no_venv_error(): + # Check that --no-venv can not be set together with a non-none --force-venv-backend + sys.argv = [ + sys.executable, + "--noxfile", + "noxfile.py", + "--force-venv-backend", + "conda", + "--no-venv", + ] + with pytest.raises(ValueError, match="You can not use"): + nox.__main__.main() + + def test_main_short_form_args(monkeypatch): monkeypatch.setattr( - sys, "argv", [sys.executable, "-f", "noxfile.py", "-s", "1", "2", "-r"] + sys, + "argv", + [ + sys.executable, + "-f", + "noxfile.py", + "-s", + "1", + "2", + "-db", + "venv", + "-fb", + "conda", + "-r", + ], ) with mock.patch("nox.workflow.execute") as execute: execute.return_value = 0 @@ -109,6 +173,8 @@ def test_main_short_form_args(monkeypatch): config = execute.call_args[1]["global_config"] assert config.noxfile == "noxfile.py" assert config.sessions == ["1", "2"] + assert config.default_venv_backend == "venv" + assert config.force_venv_backend == "conda" assert config.reuse_existing_virtualenvs is True diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 9864191c..491627a5 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -18,19 +18,31 @@ import nox import pytest from nox._decorators import Func -from nox.manifest import KeywordLocals, Manifest, _null_session_func +from nox.manifest import ( + KeywordLocals, + Manifest, + _null_session_func, + WARN_PYTHONS_IGNORED, +) def create_mock_sessions(): sessions = collections.OrderedDict() - sessions["foo"] = mock.Mock(spec=(), python=None) - sessions["bar"] = mock.Mock(spec=(), python=None) + sessions["foo"] = mock.Mock(spec=(), python=None, venv_backend=None) + sessions["bar"] = mock.Mock(spec=(), python=None, venv_backend=None) return sessions +def create_mock_config(): + cfg = mock.sentinel.CONFIG + cfg.force_venv_backend = None + cfg.default_venv_backend = None + return cfg + + def test_init(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) # Assert that basic properties look correctly. assert len(manifest) == 2 @@ -40,7 +52,7 @@ def test_init(): def test_contains(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) # Establish that contains works pre-iteration. assert "foo" in manifest @@ -60,7 +72,7 @@ def test_contains(): def test_getitem(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) # Establish that each session is present, and a made-up session # is not. @@ -79,7 +91,7 @@ def test_getitem(): def test_iteration(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) # There should be two sessions in the queue. assert len(manifest._queue) == 2 @@ -109,7 +121,7 @@ def test_iteration(): def test_len(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) assert len(manifest) == 2 for session in manifest: assert len(manifest) == 2 @@ -117,7 +129,7 @@ def test_len(): def test_filter_by_name(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) manifest.filter_by_name(("foo",)) assert "foo" in manifest assert "bar" not in manifest @@ -125,21 +137,21 @@ def test_filter_by_name(): def test_filter_by_name_maintains_order(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) manifest.filter_by_name(("bar", "foo")) assert [session.name for session in manifest] == ["bar", "foo"] def test_filter_by_name_not_found(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) with pytest.raises(KeyError): manifest.filter_by_name(("baz",)) def test_filter_by_python_interpreter(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) manifest["foo"].func.python = "3.8" manifest["bar"].func.python = "3.7" manifest.filter_by_python_interpreter(("3.8",)) @@ -149,7 +161,7 @@ def test_filter_by_python_interpreter(): def test_filter_by_keyword(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) assert len(manifest) == 2 manifest.filter_by_keywords("foo or bar") assert len(manifest) == 2 @@ -159,7 +171,7 @@ def test_filter_by_keyword(): def test_list_all_sessions_with_filter(): sessions = create_mock_sessions() - manifest = Manifest(sessions, mock.sentinel.CONFIG) + manifest = Manifest(sessions, create_mock_config()) assert len(manifest) == 2 manifest.filter_by_keywords("foo") assert len(manifest) == 1 @@ -171,15 +183,15 @@ def test_list_all_sessions_with_filter(): def test_add_session_plain(): - manifest = Manifest({}, mock.sentinel.CONFIG) - session_func = mock.Mock(spec=(), python=None) + manifest = Manifest({}, create_mock_config()) + session_func = mock.Mock(spec=(), python=None, venv_backend=None) for session in manifest.make_session("my_session", session_func): manifest.add_session(session) assert len(manifest) == 1 def test_add_session_multiple_pythons(): - manifest = Manifest({}, mock.sentinel.CONFIG) + manifest = Manifest({}, create_mock_config()) def session_func(): pass @@ -192,7 +204,7 @@ def session_func(): def test_add_session_parametrized(): - manifest = Manifest({}, mock.sentinel.CONFIG) + manifest = Manifest({}, create_mock_config()) # Define a session with parameters. @nox.parametrize("param", ("a", "b", "c")) @@ -208,7 +220,7 @@ def my_session(session, param): def test_add_session_parametrized_multiple_pythons(): - manifest = Manifest({}, mock.sentinel.CONFIG) + manifest = Manifest({}, create_mock_config()) # Define a session with parameters. @nox.parametrize("param", ("a", "b")) @@ -224,7 +236,7 @@ def my_session(session, param): def test_add_session_parametrized_noop(): - manifest = Manifest({}, mock.sentinel.CONFIG) + manifest = Manifest({}, create_mock_config()) # Define a session without any parameters. @nox.parametrize("param", ()) @@ -232,6 +244,7 @@ def my_session(session, param): pass my_session.python = None + my_session.venv_backend = None # Add the session to the manifest. for session in manifest.make_session("my_session", my_session): @@ -244,18 +257,20 @@ def my_session(session, param): def test_notify(): - manifest = Manifest({}, mock.sentinel.CONFIG) + manifest = Manifest({}, create_mock_config()) # Define a session. def my_session(session): pass my_session.python = None + my_session.venv_backend = None def notified(session): pass notified.python = None + notified.venv_backend = None # Add the sessions to the manifest. for session in manifest.make_session("my_session", my_session): @@ -274,13 +289,14 @@ def notified(session): def test_notify_noop(): - manifest = Manifest({}, mock.sentinel.CONFIG) + manifest = Manifest({}, create_mock_config()) # Define a session and add it to the manifest. def my_session(session): pass my_session.python = None + my_session.venv_backend = None for session in manifest.make_session("my_session", my_session): manifest.add_session(session) @@ -293,14 +309,14 @@ def my_session(session): def test_notify_error(): - manifest = Manifest({}, mock.sentinel.CONFIG) + manifest = Manifest({}, create_mock_config()) with pytest.raises(ValueError): manifest.notify("does_not_exist") def test_add_session_idempotent(): - manifest = Manifest({}, mock.sentinel.CONFIG) - session_func = mock.Mock(spec=(), python=None) + manifest = Manifest({}, create_mock_config()) + session_func = mock.Mock(spec=(), python=None, venv_backend=None) for session in manifest.make_session("my_session", session_func): manifest.add_session(session) manifest.add_session(session) @@ -322,3 +338,22 @@ def test_keyword_locals_iter(): values = ["foo", "bar"] kw = KeywordLocals(values) assert list(kw) == values + + +def test_no_venv_backend_but_some_pythons(): + manifest = Manifest({}, create_mock_config()) + + # Define a session and add it to the manifest. + def my_session(session): + pass + + # the session sets "no venv backend" but declares some pythons + my_session.python = ["3.7", "3.8"] + my_session.venv_backend = "none" + my_session.should_warn = dict() + + sessions = manifest.make_session("my_session", my_session) + + # check that the pythons were correctly removed (a log warning is also emitted) + assert sessions[0].func.python is False + assert sessions[0].func.should_warn == {WARN_PYTHONS_IGNORED: ["3.7", "3.8"]} diff --git a/tests/test_sessions.py b/tests/test_sessions.py index f4c457fb..7d88de52 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -265,6 +265,7 @@ def test_run_external_with_error_on_external_run_condaenv(self): def test_conda_install_bad_args(self): session, runner = self.make_session_and_runner() runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) + runner.venv.location = "dummy" with pytest.raises(ValueError, match="arg"): session.conda_install() diff --git a/tests/test_tasks.py b/tests/test_tasks.py index e9143136..c81490ad 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -23,7 +23,7 @@ import nox import pytest from nox import _options, sessions, tasks -from nox.manifest import Manifest +from nox.manifest import Manifest, WARN_PYTHONS_IGNORED RESOURCES = os.path.join(os.path.dirname(__file__), "resources") @@ -33,6 +33,8 @@ def session_func(): session_func.python = None +session_func.venv_backend = None +session_func.should_warn = dict() def session_func_with_python(): @@ -40,6 +42,16 @@ def session_func_with_python(): session_func_with_python.python = "3.8" +session_func_with_python.venv_backend = None + + +def session_func_venv_pythons_warning(): + pass + + +session_func_venv_pythons_warning.python = ["3.7"] +session_func_venv_pythons_warning.venv_backend = "none" +session_func_venv_pythons_warning.should_warn = {WARN_PYTHONS_IGNORED: ["3.7"]} def test_load_nox_module(): @@ -185,7 +197,8 @@ def test_verify_manifest_nonempty(): assert return_value == manifest -def test_run_manifest(): +@pytest.mark.parametrize("with_warnings", [False, True], ids="with_warnings={}".format) +def test_run_manifest(with_warnings): # Set up a valid manifest. config = _options.options.namespace(stop_on_first_error=False) sessions_ = [ @@ -200,6 +213,12 @@ def test_run_manifest(): mock_session.execute.return_value = sessions.Result( session=mock_session, status=sessions.Status.SUCCESS ) + # we need the should_warn attribute, add some func + if with_warnings: + mock_session.name = "hello" + mock_session.func = session_func_venv_pythons_warning + else: + mock_session.func = session_func # Run the manifest. results = tasks.run_manifest(manifest, global_config=config) @@ -228,6 +247,8 @@ def test_run_manifest_abort_on_first_failure(): mock_session.execute.return_value = sessions.Result( session=mock_session, status=sessions.Status.FAILED ) + # we need the should_warn attribute, add some func + mock_session.func = session_func # Run the manifest. results = tasks.run_manifest(manifest, global_config=config) From a3cb034accf1ce8ebe10d92a6bf81f3f251478a9 Mon Sep 17 00:00:00 2001 From: Thea Flowers Date: Sun, 24 May 2020 20:54:52 -0700 Subject: [PATCH 07/91] Release 2020.5.24 (#327) --- CHANGELOG.md | 17 +++++++++++++++++ setup.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11046e1c..a7ad61fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 2020.5.24 + +- Add new options for `venv_backend`, including the ability to set the backend globally. (#326) +- Fix various typos in the documentation. (#325, #326, #281) +- Add `session.create_tmp`. (#320) +- Place all of Nox's command-line options into argparse groups. (#306) +- Add the `--pythons` command-line option to allow specifying which versions of Python to run. (#304) +- Add a significant amount of type annotations. (#297, #294, #290, #282, #274) +- Stop building universal wheels since we don't support Python 2. (#293) +- Add the ability to specify additional options for the virtualenv backend using `venv_params`. (#280) +- Prefer `importlib.metadata` for metadata loading, removing our dependency on `pkg_resources`. (#277) +- Add OmegaConf and Hydra to list of projects that use Nox. (#279) +- Use a more accurate error message, along with the cause, if loading of noxfile runs into error. (#272) +- Test against Python 3.8. (#270) +- Fix a syntax highlighting mistake in configuration docs. (#268) +- Use `stdout.isatty` to finalize color instead of `stdin.isatty`. (#267) + ## 2019.11.9 - Fix example installation call for pip. (#259) diff --git a/setup.py b/setup.py index c39ab6f6..27143a0d 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="nox", - version="2019.11.9", + version="2019.5.24", description="Flexible test automation.", long_description=long_description, url="https://nox.thea.codes", From 5a4f54ae71c2ceb6c370866b5443ceb11e181bc0 Mon Sep 17 00:00:00 2001 From: Thea Flowers Date: Mon, 25 May 2020 11:42:57 -0700 Subject: [PATCH 08/91] Fix version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 27143a0d..d368a0c7 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="nox", - version="2019.5.24", + version="2020.5.24", description="Flexible test automation.", long_description=long_description, url="https://nox.thea.codes", From 323ab59a7ac75d8cfb4e92048118245c91767e67 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 8 Jun 2020 10:13:43 -0700 Subject: [PATCH 09/91] Ran `nox -s blacken`. (#332) --- nox/_decorators.py | 2 +- nox/manifest.py | 1 - nox/sessions.py | 4 ++-- nox/tasks.py | 2 +- tests/test_manifest.py | 2 +- tests/test_tasks.py | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/nox/_decorators.py b/nox/_decorators.py index a4f031f5..03878deb 100644 --- a/nox/_decorators.py +++ b/nox/_decorators.py @@ -1,7 +1,7 @@ import copy import functools import types -from typing import Any, Callable, Iterable, List, Dict, Optional, cast +from typing import Any, Callable, Dict, Iterable, List, Optional, cast from . import _typing diff --git a/nox/manifest.py b/nox/manifest.py index f2a008f0..eb6d8b5b 100644 --- a/nox/manifest.py +++ b/nox/manifest.py @@ -20,7 +20,6 @@ from nox._decorators import Call, Func from nox.sessions import Session, SessionRunner - WARN_PYTHONS_IGNORED = "python_ignored" diff --git a/nox/sessions.py b/nox/sessions.py index c193162d..b687898e 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -24,11 +24,11 @@ Callable, Dict, Iterable, - Tuple, List, Mapping, Optional, Sequence, + Tuple, Union, ) @@ -37,7 +37,7 @@ from nox import _typing from nox._decorators import Func from nox.logger import logger -from nox.virtualenv import CondaEnv, ProcessEnv, VirtualEnv, PassthroughEnv +from nox.virtualenv import CondaEnv, PassthroughEnv, ProcessEnv, VirtualEnv if _typing.TYPE_CHECKING: from nox.manifest import Manifest diff --git a/nox/tasks.py b/nox/tasks.py index bd5e1c37..31d4c826 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -24,7 +24,7 @@ from colorlog.escape_codes import parse_colors from nox import _options, registry from nox.logger import logger -from nox.manifest import Manifest, WARN_PYTHONS_IGNORED +from nox.manifest import WARN_PYTHONS_IGNORED, Manifest from nox.sessions import Result diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 491627a5..20f27f66 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -19,10 +19,10 @@ import pytest from nox._decorators import Func from nox.manifest import ( + WARN_PYTHONS_IGNORED, KeywordLocals, Manifest, _null_session_func, - WARN_PYTHONS_IGNORED, ) diff --git a/tests/test_tasks.py b/tests/test_tasks.py index c81490ad..88c5fc4d 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -23,7 +23,7 @@ import nox import pytest from nox import _options, sessions, tasks -from nox.manifest import Manifest, WARN_PYTHONS_IGNORED +from nox.manifest import WARN_PYTHONS_IGNORED, Manifest RESOURCES = os.path.join(os.path.dirname(__file__), "resources") From e3d4696638fc456004360c3259f0ccc7b82a73b0 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 8 Jun 2020 11:10:05 -0700 Subject: [PATCH 10/91] Add `Session.run_always()`. (#331) Fixes #330. --- docs/index.rst | 2 +- docs/tutorial.rst | 24 ++++++++++++++++++++++++ nox/sessions.py | 28 ++++++++++++++++++++++++++++ tests/test_sessions.py | 25 +++++++++++++++++++++++-- 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 5da9abb6..0071cbf6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -50,7 +50,7 @@ Projects that use Nox Nox is lucky to have several wonderful projects that use it and provide feedback and contributions. -- `Bezier `__ +- `Bézier `__ - `gapic-generator-python `__ - `gdbgui `__ - `Google Assistant SDK `__ diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 47a7ae81..8be056dc 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -124,6 +124,30 @@ If your project is a Python package and you want to install it: session.install(".") ... +In some cases such as Python binary extensions, your package may depend on +code compiled outside of the Python ecosystem. To make sure a low-level +dependency (e.g. ``libfoo``) is available during installation + +.. code-block:: python + + @nox.session + def tests(session): + ... + session.run_always( + "cmake", "-DCMAKE_BUILD_TYPE=Debug", + "-S", libfoo_src_dir, + "-B", build_dir, + external=True, + ) + session.run_always( + "cmake", + "--build", build_dir, + "--config", "Debug", + "--target", "install", + external=True, + ) + session.install(".") + ... Running commands ---------------- diff --git a/nox/sessions.py b/nox/sessions.py index b687898e..bac226f1 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -225,6 +225,34 @@ def run( return self._run(*args, env=env, **kwargs) + def run_always( + self, *args: str, env: Mapping[str, str] = None, **kwargs: Any + ) -> Optional[Any]: + """Run a command **always**. + + This is a variant of :meth:`run` that runs in all cases, including in + the presence of ``--install-only``. + + :param env: A dictionary of environment variables to expose to the + command. By default, all environment variables are passed. + :type env: dict or None + :param bool silent: Silence command output, unless the command fails. + ``False`` by default. + :param success_codes: A list of return codes that are considered + successful. By default, only ``0`` is considered success. + :type success_codes: list, tuple, or None + :param external: If False (the default) then programs not in the + virtualenv path will cause a warning. If True, no warning will be + emitted. These warnings can be turned into errors using + ``--error-on-external-run``. This has no effect for sessions that + do not have a virtualenv. + :type external: bool + """ + if not args: + raise ValueError("At least one argument required to run_always().") + + return self._run(*args, env=env, **kwargs) + def _run(self, *args: str, env: Mapping[str, str] = None, **kwargs: Any) -> Any: """Like run(), except that it runs even if --install-only is provided.""" # Legacy support - run a function given. diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 7d88de52..367314b9 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -14,6 +14,7 @@ import argparse import logging +import operator import os import sys import tempfile @@ -151,7 +152,7 @@ def test_run_bad_args(self): def test_run_with_func(self): session, _ = self.make_session_and_runner() - assert session.run(lambda a, b: a + b, 1, 2) == 3 + assert session.run(operator.add, 1, 2) == 3 def test_run_with_func_error(self): session, _ = self.make_session_and_runner() @@ -168,7 +169,7 @@ def test_run_install_only(self, caplog): runner.global_config.install_only = True with mock.patch.object(nox.command, "run") as run: - session.run("spam", "eggs") + assert session.run("spam", "eggs") is None run.assert_not_called() @@ -262,6 +263,26 @@ def test_run_external_with_error_on_external_run_condaenv(self): with pytest.raises(nox.command.CommandFailed, match="External"): session.run(sys.executable, "--version") + def test_run_always_bad_args(self): + session, _ = self.make_session_and_runner() + + with pytest.raises(ValueError) as exc_info: + session.run_always() + + exc_args = exc_info.value.args + assert exc_args == ("At least one argument required to run_always().",) + + def test_run_always_success(self): + session, _ = self.make_session_and_runner() + + assert session.run_always(operator.add, 1300, 37) == 1337 + + def test_run_always_install_only(self, caplog): + session, runner = self.make_session_and_runner() + runner.global_config.install_only = True + + assert session.run_always(operator.add, 23, 19) == 42 + def test_conda_install_bad_args(self): session, runner = self.make_session_and_runner() runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) From 8dcefbf8f59293bc1925dd13376dd6d3aa66e53a Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Thu, 18 Jun 2020 15:22:59 -0700 Subject: [PATCH 11/91] Add --add-timestamp option (#323) * initial pass, some tests failing * lint * hopefully fixed linting on 3.5 --- nox/__main__.py | 4 +- nox/_options.py | 9 +++++ nox/logger.py | 87 +++++++++++++++++++++++++++++++++----------- tests/test_logger.py | 38 +++++++++++++++++++ 4 files changed, 116 insertions(+), 22 deletions(-) diff --git a/nox/__main__.py b/nox/__main__.py index f81412c9..2f551cca 100644 --- a/nox/__main__.py +++ b/nox/__main__.py @@ -41,7 +41,9 @@ def main() -> None: print(metadata.version("nox"), file=sys.stderr) return - setup_logging(color=args.color, verbose=args.verbose) + setup_logging( + color=args.color, verbose=args.verbose, add_timestamp=args.add_timestamp + ) # Execute the appropriate tasks. exit_code = workflow.execute( diff --git a/nox/_options.py b/nox/_options.py index a46943a5..336cff5e 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -264,6 +264,15 @@ def _session_completer( help="Logs the output of all commands run including commands marked silent.", noxfile=True, ), + _option_set.Option( + "add_timestamp", + "-ts", + "--add-timestamp", + group=options.groups["secondary"], + action="store_true", + help="Adds a timestamp to logged output.", + noxfile=True, + ), _option_set.Option( "default_venv_backend", "-db", diff --git a/nox/logger.py b/nox/logger.py index 9d24105b..84d5fcca 100644 --- a/nox/logger.py +++ b/nox/logger.py @@ -21,15 +21,53 @@ OUTPUT = logging.DEBUG - 1 -class NoxFormatter(ColoredFormatter): - def __init__(self, *args: Any, **kwargs: Any) -> None: - super(NoxFormatter, self).__init__(*args, **kwargs) +def _get_format(colorlog: bool, add_timestamp: bool) -> str: + if colorlog: + if add_timestamp: + return "%(cyan)s%(name)s > [%(asctime)s] %(log_color)s%(message)s" + else: + return "%(cyan)s%(name)s > %(log_color)s%(message)s" + else: + if add_timestamp: + return "%(name)s > [%(asctime)s] %(message)s" + else: + return "%(name)s > %(message)s" + + +class NoxFormatter(logging.Formatter): + def __init__(self, add_timestamp: bool = False) -> None: + super().__init__(fmt=_get_format(colorlog=False, add_timestamp=add_timestamp)) self._simple_fmt = logging.Formatter("%(message)s") def format(self, record: Any) -> str: if record.levelname == "OUTPUT": return self._simple_fmt.format(record) - return super(NoxFormatter, self).format(record) + return super().format(record) + + +class NoxColoredFormatter(ColoredFormatter): + def __init__( + self, + datefmt: Any = None, + style: Any = None, + log_colors: Any = None, + reset: bool = True, + secondary_log_colors: Any = None, + add_timestamp: bool = False, + ) -> None: + super().__init__( + fmt=_get_format(colorlog=True, add_timestamp=add_timestamp), + datefmt=datefmt, + style=style, + log_colors=log_colors, + reset=reset, + secondary_log_colors=secondary_log_colors, + ) + + def format(self, record: Any) -> str: + if record.levelname == "OUTPUT": + return self._simple_fmt.format(record) + return super().format(record) class LoggerWithSuccessAndOutput(logging.getLoggerClass()): # type: ignore @@ -55,23 +93,9 @@ def output(self, msg: str, *args: Any, **kwargs: Any) -> None: logger = cast(LoggerWithSuccessAndOutput, logging.getLogger("nox")) -def setup_logging(color: bool, verbose: bool = False) -> None: # pragma: no cover - """Setup logging. - - Args: - color (bool): If true, the output will be colored using - colorlog. Otherwise, it will be plaintext. - """ - root_logger = logging.getLogger() - if verbose: - root_logger.setLevel(OUTPUT) - else: - root_logger.setLevel(logging.DEBUG) - handler = logging.StreamHandler() - +def _get_formatter(color: bool, add_timestamp: bool) -> logging.Formatter: if color is True: - formatter = NoxFormatter( - "%(cyan)s%(name)s > %(log_color)s%(message)s", + return NoxColoredFormatter( reset=True, log_colors={ "DEBUG": "cyan", @@ -81,10 +105,31 @@ def setup_logging(color: bool, verbose: bool = False) -> None: # pragma: no cov "CRITICAL": "red,bg_white", "SUCCESS": "green", }, + style="%", + secondary_log_colors=None, + add_timestamp=add_timestamp, ) + else: + return NoxFormatter(add_timestamp=add_timestamp) + - handler.setFormatter(formatter) +def setup_logging( + color: bool, verbose: bool = False, add_timestamp: bool = False +) -> None: # pragma: no cover + """Setup logging. + + Args: + color (bool): If true, the output will be colored using + colorlog. Otherwise, it will be plaintext. + """ + root_logger = logging.getLogger() + if verbose: + root_logger.setLevel(OUTPUT) + else: + root_logger.setLevel(logging.DEBUG) + handler = logging.StreamHandler() + handler.setFormatter(_get_formatter(color, add_timestamp)) root_logger.addHandler(handler) # Silence noisy loggers diff --git a/tests/test_logger.py b/tests/test_logger.py index 9d8b6fc7..eef9dcf3 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -15,6 +15,8 @@ import logging from unittest import mock +import pytest + from nox import logger @@ -39,6 +41,7 @@ def test_formatter(caplog): logs = [rec for rec in caplog.records if rec.levelname in ("INFO", "OUTPUT")] assert len(logs) == 1 + assert not hasattr(logs[0], "asctime") caplog.clear() with caplog.at_level(logger.OUTPUT): @@ -52,3 +55,38 @@ def test_formatter(caplog): assert len(logs) == 1 # Make sure output level log records are not nox prefixed assert "nox" not in logs[0].message + + +@pytest.mark.parametrize( + "color", + [ + # This currently fails due to some incompatibility between caplog and colorlog + # that causes caplog to not collect the asctime from colorlog. + pytest.param(True, id="color", marks=pytest.mark.xfail), + pytest.param(False, id="no-color"), + ], +) +def test_no_color_timestamp(caplog, color): + logger.setup_logging(color=color, add_timestamp=True) + caplog.clear() + with caplog.at_level(logging.DEBUG): + logger.logger.info("bar") + logger.logger.output("foo") + + logs = [rec for rec in caplog.records if rec.levelname in ("INFO", "OUTPUT")] + assert len(logs) == 1 + assert hasattr(logs[0], "asctime") + + caplog.clear() + with caplog.at_level(logger.OUTPUT): + logger.logger.info("bar") + logger.logger.output("foo") + + logs = [rec for rec in caplog.records if rec.levelname != "OUTPUT"] + assert len(logs) == 1 + assert hasattr(logs[0], "asctime") + + logs = [rec for rec in caplog.records if rec.levelname == "OUTPUT"] + assert len(logs) == 1 + # no timestamp for output + assert not hasattr(logs[0], "asctime") From ae3b37d30249728c435cd772d7cfd30208c16184 Mon Sep 17 00:00:00 2001 From: smarie Date: Mon, 22 Jun 2020 00:58:42 +0200 Subject: [PATCH 12/91] Fixed the default paths for conda on windows where the `python.exe` found was not the correct one (#310) * Virtual environments now have a `bin_paths` attribute instead of a single `bin`. `command.run` and `which` both have their `path` argument renamed `paths` and supporting a list of paths, to handle it. Fixes #256 * Updated tests to handle the `paths` renaming of the `path` argument in `nox.command.run`, and the `bin` to `bin_paths` renaming in `virtualenv` * First conda test on windows * Black-ened code * As per code review: Added a backwards-compatible `session.bin` property * Fixed lint error * Added back `bin` to all envs for compatibility with legacy api Co-authored-by: Sylvain MARIE --- nox/command.py | 16 +++++++++------- nox/sessions.py | 12 +++++++++--- nox/virtualenv.py | 33 +++++++++++++++++++++------------ tests/test_command.py | 10 +++++----- tests/test_sessions.py | 25 ++++++++++++++++++------- tests/test_virtualenv.py | 18 +++++++++++++----- 6 files changed, 75 insertions(+), 39 deletions(-) diff --git a/nox/command.py b/nox/command.py index 766481ea..b2e99cef 100644 --- a/nox/command.py +++ b/nox/command.py @@ -14,7 +14,7 @@ import os import sys -from typing import Any, Iterable, Optional, Sequence, Union +from typing import Any, Iterable, Optional, Sequence, Union, List import py from nox.logger import logger @@ -29,12 +29,12 @@ def __init__(self, reason: str = None) -> None: self.reason = reason -def which(program: str, path: Optional[str]) -> str: +def which(program: str, paths: Optional[List[str]]) -> str: """Finds the full path to an executable.""" full_path = None - if path: - full_path = py.path.local.sysfind(program, paths=[path]) + if paths: + full_path = py.path.local.sysfind(program, paths=paths) if full_path: return full_path.strpath @@ -66,7 +66,7 @@ def run( *, env: Optional[dict] = None, silent: bool = False, - path: Optional[str] = None, + paths: Optional[List[str]] = None, success_codes: Optional[Iterable[int]] = None, log: bool = True, external: bool = False, @@ -80,12 +80,14 @@ def run( cmd, args = args[0], args[1:] full_cmd = "{} {}".format(cmd, " ".join(args)) - cmd_path = which(cmd, path) + cmd_path = which(cmd, paths) if log: logger.info(full_cmd) - is_external_tool = path is not None and not cmd_path.startswith(path) + is_external_tool = paths is not None and not any( + cmd_path.startswith(path) for path in paths + ) if is_external_tool: if external == "error": logger.error( diff --git a/nox/sessions.py b/nox/sessions.py index bac226f1..c35b0c05 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -131,10 +131,16 @@ def python(self) -> Optional[Union[str, Sequence[str], bool]]: """The python version passed into ``@nox.session``.""" return self._runner.func.python + @property + def bin_paths(self) -> Optional[List[str]]: + """The bin directories for the virtualenv.""" + return self.virtualenv.bin_paths + @property def bin(self) -> Optional[str]: - """The bin directory for the virtualenv.""" - return self.virtualenv.bin + """The first bin directory for the virtualenv.""" + paths = self.bin_paths + return paths[0] if paths is not None else None def create_tmp(self) -> str: """Create, and return, a temporary directory.""" @@ -279,7 +285,7 @@ def _run(self, *args: str, env: Mapping[str, str] = None, **kwargs: Any) -> Any: kwargs["external"] = True # Run a shell command. - return nox.command.run(args, env=env, path=self.bin, **kwargs) + return nox.command.run(args, env=env, paths=self.bin_paths, **kwargs) def conda_install(self, *args: str, **kwargs: Any) -> None: """Install invokes `conda install`_ to install packages inside of the diff --git a/nox/virtualenv.py b/nox/virtualenv.py index fea02381..5824bfc6 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -17,7 +17,7 @@ import re import shutil import sys -from typing import Any, Mapping, Optional, Tuple, Union +from typing import Any, Mapping, Optional, Tuple, Union, List import nox.command import py @@ -48,8 +48,8 @@ class ProcessEnv: # Special programs that aren't included in the environment. allowed_globals = () # type: _typing.ClassVar[Tuple[Any, ...]] - def __init__(self, bin: None = None, env: Mapping[str, str] = None) -> None: - self._bin = bin + def __init__(self, bin_paths: None = None, env: Mapping[str, str] = None) -> None: + self._bin_paths = bin_paths self.env = os.environ.copy() if env is not None: @@ -58,12 +58,20 @@ def __init__(self, bin: None = None, env: Mapping[str, str] = None) -> None: for key in _BLACKLISTED_ENV_VARS: self.env.pop(key, None) - if self.bin: - self.env["PATH"] = os.pathsep.join([self.bin, self.env.get("PATH", "")]) + if self.bin_paths: + self.env["PATH"] = os.pathsep.join( + self.bin_paths + [self.env.get("PATH", "")] + ) + + @property + def bin_paths(self) -> Optional[List[str]]: + return self._bin_paths @property def bin(self) -> Optional[str]: - return self._bin + """The first bin directory for the virtualenv.""" + paths = self.bin_paths + return paths[0] if paths is not None else None def create(self) -> bool: raise NotImplementedError("ProcessEnv.create should be overwritten in subclass") @@ -191,12 +199,13 @@ def __init__( _clean_location = _clean_location @property - def bin(self) -> str: + def bin_paths(self) -> List[str]: """Returns the location of the conda env's bin folder.""" + # see https://docs.anaconda.com/anaconda/user-guide/tasks/integration/python-path/#examples if _SYSTEM == "Windows": - return os.path.join(self.location, "Scripts") + return [self.location, os.path.join(self.location, "Scripts")] else: - return os.path.join(self.location, "bin") + return [os.path.join(self.location, "bin")] def create(self) -> bool: """Create the conda env.""" @@ -339,12 +348,12 @@ def _resolved_interpreter(self) -> str: raise self._resolved @property - def bin(self) -> str: + def bin_paths(self) -> List[str]: """Returns the location of the virtualenv's bin folder.""" if _SYSTEM == "Windows": - return os.path.join(self.location, "Scripts") + return [os.path.join(self.location, "Scripts")] else: - return os.path.join(self.location, "bin") + return [os.path.join(self.location, "bin")] def create(self) -> bool: """Create the virtualenv or venv.""" diff --git a/tests/test_command.py b/tests/test_command.py index 3124f904..45919ddb 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -122,7 +122,7 @@ def test_run_path_nonexistent(): result = nox.command.run( [PYTHON, "-c", "import sys; print(sys.executable)"], silent=True, - path="/non/existent", + paths=["/non/existent"], ) assert "/non/existent" not in result @@ -135,14 +135,14 @@ def test_run_path_existent(tmpdir, monkeypatch): with mock.patch("nox.command.popen") as mock_command: mock_command.return_value = (0, "") - nox.command.run(["testexc"], silent=True, path=tmpdir.strpath) + nox.command.run(["testexc"], silent=True, paths=[tmpdir.strpath]) mock_command.assert_called_with([executable.strpath], env=None, silent=True) def test_run_external_warns(tmpdir, caplog): caplog.set_level(logging.WARNING) - nox.command.run([PYTHON, "--version"], silent=True, path=tmpdir.strpath) + nox.command.run([PYTHON, "--version"], silent=True, paths=[tmpdir.strpath]) assert "external=True" in caplog.text @@ -151,7 +151,7 @@ def test_run_external_silences(tmpdir, caplog): caplog.set_level(logging.WARNING) nox.command.run( - [PYTHON, "--version"], silent=True, path=tmpdir.strpath, external=True + [PYTHON, "--version"], silent=True, paths=[tmpdir.strpath], external=True ) assert "external=True" not in caplog.text @@ -162,7 +162,7 @@ def test_run_external_raises(tmpdir, caplog): with pytest.raises(nox.command.CommandFailed): nox.command.run( - [PYTHON, "--version"], silent=True, path=tmpdir.strpath, external="error" + [PYTHON, "--version"], silent=True, paths=[tmpdir.strpath], external="error" ) assert "external=True" in caplog.text diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 367314b9..6afecaf8 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -73,7 +73,7 @@ def make_session_and_runner(self): ) runner.venv = mock.create_autospec(nox.virtualenv.VirtualEnv) runner.venv.env = {} - runner.venv.bin = "/no/bin/for/you" + runner.venv.bin_paths = ["/no/bin/for/you"] return nox.sessions.Session(runner=runner), runner def test_create_tmp(self): @@ -100,9 +100,17 @@ def test_properties(self): assert session.env is runner.venv.env assert session.posargs is runner.global_config.posargs assert session.virtualenv is runner.venv - assert session.bin is runner.venv.bin + assert session.bin_paths is runner.venv.bin_paths + assert session.bin is runner.venv.bin_paths[0] assert session.python is runner.func.python + def test_no_bin_paths(self): + session, runner = self.make_session_and_runner() + + runner.venv.bin_paths = None + assert session.bin is None + assert session.bin_paths is None + def test_virtualenv_as_none(self): session, runner = self.make_session_and_runner() @@ -187,7 +195,7 @@ def test_run_install_only_should_install(self): ("pip", "install", "spam"), env=mock.ANY, external=mock.ANY, - path=mock.ANY, + paths=mock.ANY, silent=mock.ANY, ) @@ -225,7 +233,7 @@ def test_run_external_not_a_virtualenv(self): session.run(sys.executable, "--version") run.assert_called_once_with( - (sys.executable, "--version"), external=True, env=mock.ANY, path=None + (sys.executable, "--version"), external=True, env=mock.ANY, paths=None ) def test_run_external_condaenv(self): @@ -234,14 +242,17 @@ def test_run_external_condaenv(self): runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) runner.venv.allowed_globals = ("conda",) runner.venv.env = {} - runner.venv.bin = "/path/to/env/bin" + runner.venv.bin_paths = ["/path/to/env/bin"] runner.venv.create.return_value = True with mock.patch("nox.command.run", autospec=True) as run: session.run("conda", "--version") run.assert_called_once_with( - ("conda", "--version"), external=True, env=mock.ANY, path="/path/to/env/bin" + ("conda", "--version"), + external=True, + env=mock.ANY, + paths=["/path/to/env/bin"], ) def test_run_external_with_error_on_external_run(self): @@ -256,7 +267,7 @@ def test_run_external_with_error_on_external_run_condaenv(self): session, runner = self.make_session_and_runner() runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) runner.venv.env = {} - runner.venv.bin = "/path/to/env/bin" + runner.venv.bin_paths = ["/path/to/env/bin"] runner.global_config.error_on_external_run = True diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 543d66f2..8a8ed6d5 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -120,7 +120,7 @@ def mock_sysfind(arg): def test_process_env_constructor(): penv = nox.virtualenv.ProcessEnv() - assert not penv.bin + assert not penv.bin_paths penv = nox.virtualenv.ProcessEnv(env={"SIGIL": "123"}) assert penv.env["SIGIL"] == "123" @@ -186,7 +186,7 @@ def test_condaenv_create_interpreter(make_conda): @mock.patch("nox.virtualenv._SYSTEM", new="Windows") def test_condaenv_bin_windows(make_conda): venv, dir_ = make_conda() - assert dir_.join("Scripts").strpath == venv.bin + assert [dir_.strpath, dir_.join("Scripts").strpath] == venv.bin_paths def test_constructor_defaults(make_one): @@ -209,13 +209,16 @@ def test_env(monkeypatch, make_one): monkeypatch.setenv("SIGIL", "123") venv, _ = make_one() assert venv.env["SIGIL"] == "123" - assert venv.bin in venv.env["PATH"] - assert venv.bin not in os.environ["PATH"] + assert len(venv.bin_paths) == 1 + assert venv.bin_paths[0] in venv.env["PATH"] + assert venv.bin_paths[0] not in os.environ["PATH"] def test_blacklisted_env(monkeypatch, make_one): monkeypatch.setenv("__PYVENV_LAUNCHER__", "meep") venv, _ = make_one() + assert len(venv.bin_paths) == 1 + assert venv.bin_paths[0] == venv.bin assert "__PYVENV_LAUNCHER__" not in venv.bin @@ -249,9 +252,12 @@ def test__clean_location(monkeypatch, make_one): assert venv._clean_location() -def test_bin(make_one): +def test_bin_paths(make_one): venv, dir_ = make_one() + assert len(venv.bin_paths) == 1 + assert venv.bin_paths[0] == venv.bin + if IS_WINDOWS: assert dir_.join("Scripts").strpath == venv.bin else: @@ -261,6 +267,8 @@ def test_bin(make_one): @mock.patch("nox.virtualenv._SYSTEM", new="Windows") def test_bin_windows(make_one): venv, dir_ = make_one() + assert len(venv.bin_paths) == 1 + assert venv.bin_paths[0] == venv.bin assert dir_.join("Scripts").strpath == venv.bin From 3098a7aea68c397de6b59ba09bc1f9bb2d4b1ff8 Mon Sep 17 00:00:00 2001 From: smarie Date: Mon, 22 Jun 2020 01:01:27 +0200 Subject: [PATCH 13/91] Auto-offline for `conda_install` (#314) * Offline mode is now auto-detected by default by `conda_install`. This allows users to continue executing nox sessions on already installed environments. This behaviour can be disabled by setting `auto_offline=False`. Fixes #313 * Fixed args order for offline option and added tests * Black-ened code * Fixed conda options order again * Added a log message when doing auto-offline * Fixed mypy errors * Fixed mypy errors (2) * mypy fix (3) * Fixed last failing test * Improved coverage * Last coverage fix ? * Blackened * Last Flake8 fix * removed dependency to `requests` * removed dependency to `urllib3` * Simplified offline checks * fixed import * fixed test * Added pragma no cover Co-authored-by: Sylvain MARIE --- nox/sessions.py | 24 ++++++++++++++++++++++-- nox/virtualenv.py | 23 ++++++++++++++++++++++- tests/test_sessions.py | 12 ++++++++++-- tests/test_virtualenv.py | 5 +++++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/nox/sessions.py b/nox/sessions.py index c35b0c05..643f21c2 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -287,7 +287,9 @@ def _run(self, *args: str, env: Mapping[str, str] = None, **kwargs: Any) -> Any: # Run a shell command. return nox.command.run(args, env=env, paths=self.bin_paths, **kwargs) - def conda_install(self, *args: str, **kwargs: Any) -> None: + def conda_install( + self, *args: str, auto_offline: bool = True, **kwargs: Any + ) -> None: """Install invokes `conda install`_ to install packages inside of the session's environment. @@ -302,6 +304,10 @@ def conda_install(self, *args: str, **kwargs: Any) -> None: session.conda_install('--file', 'requirements.txt') session.conda_install('--file', 'requirements-dev.txt') + By default this method will detect when internet connection is not + available and will add the `--offline` flag automatically in that case. + To disable this behaviour, set `auto_offline=False`. + To install the current package without clobbering conda-installed dependencies:: @@ -329,8 +335,22 @@ def conda_install(self, *args: str, **kwargs: Any) -> None: if "silent" not in kwargs: kwargs["silent"] = True + extraopts = () # type: Tuple[str, ...] + if auto_offline and venv.is_offline(): + logger.warning( + "Automatically setting the `--offline` flag as conda repo seems unreachable." + ) + extraopts = ("--offline",) + self._run( - "conda", "install", "--yes", *prefix_args, *args, external="error", **kwargs + "conda", + "install", + "--yes", + *extraopts, + *prefix_args, + *args, + external="error", + **kwargs ) def install(self, *args: str, **kwargs: Any) -> None: diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 5824bfc6..d3f48b6f 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -16,6 +16,7 @@ import platform import re import shutil +from socket import gethostbyname import sys from typing import Any, Mapping, Optional, Tuple, Union, List @@ -156,7 +157,10 @@ class PassthroughEnv(ProcessEnv): hints about the actual env. """ - pass + @staticmethod + def is_offline() -> bool: + """As of now this is only used in conda_install""" + return CondaEnv.is_offline() # pragma: no cover class CondaEnv(ProcessEnv): @@ -240,6 +244,23 @@ def create(self) -> bool: return True + @staticmethod + def is_offline() -> bool: + """Return `True` if we are sure that the user is not able to connect to https://repo.anaconda.com. + + Since an HTTP proxy might be correctly configured for `conda` using the `.condarc` `proxy_servers` section, + while not being correctly configured in the OS environment variables used by all other tools including python + `urllib` or `requests`, we are basically not able to do much more than testing the DNS resolution. + + See details in this explanation: https://stackoverflow.com/a/62486343/7262247 + """ + try: + # DNS resolution to detect situation (1) or (2). + host = gethostbyname("repo.anaconda.com") + return host is None + except: # pragma: no cover # noqa E722 + return True + class VirtualEnv(ProcessEnv): """Virtualenv management class. diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 6afecaf8..1da6f88f 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -310,7 +310,11 @@ def test_conda_install_not_a_condaenv(self): with pytest.raises(ValueError, match="conda environment"): session.conda_install() - def test_conda_install(self): + @pytest.mark.parametrize( + "auto_offline", [False, True], ids="auto_offline={}".format + ) + @pytest.mark.parametrize("offline", [False, True], ids="offline={}".format) + def test_conda_install(self, auto_offline, offline): runner = nox.sessions.SessionRunner( name="test", signatures=["test"], @@ -321,6 +325,7 @@ def test_conda_install(self): runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) runner.venv.location = "/path/to/conda/env" runner.venv.env = {} + runner.venv.is_offline = lambda: offline class SessionNoSlots(nox.sessions.Session): pass @@ -328,11 +333,13 @@ class SessionNoSlots(nox.sessions.Session): session = SessionNoSlots(runner=runner) with mock.patch.object(session, "_run", autospec=True) as run: - session.conda_install("requests", "urllib3") + args = ("--offline",) if auto_offline and offline else () + session.conda_install("requests", "urllib3", auto_offline=auto_offline) run.assert_called_once_with( "conda", "install", "--yes", + *args, "--prefix", "/path/to/conda/env", "requests", @@ -352,6 +359,7 @@ def test_conda_install_non_default_kwargs(self): runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) runner.venv.location = "/path/to/conda/env" runner.venv.env = {} + runner.venv.is_offline = lambda: False class SessionNoSlots(nox.sessions.Session): pass diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 8a8ed6d5..0f852108 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -189,6 +189,11 @@ def test_condaenv_bin_windows(make_conda): assert [dir_.strpath, dir_.join("Scripts").strpath] == venv.bin_paths +def test_condaenv_(make_conda): + venv, dir_ = make_conda() + assert not venv.is_offline() + + def test_constructor_defaults(make_one): venv, _ = make_one() assert venv.location From 00775b470129787b14c45f2853441f9f1cad006d Mon Sep 17 00:00:00 2001 From: smarie Date: Wed, 24 Jun 2020 22:57:26 +0200 Subject: [PATCH 14/91] `conda_install` and `install` args are now automatically double-quoted when needed. (#312) * `conda_install` and `install` args are now automatically double-quoted when they contain a `<` or a `>`. Fixes #311 * Added a test for the double-quoting fix * Black-ened * Improving coverage: added some tests and improved error checking in _dblquote_pkg_install_arg * Blackened * Fixed mypy error * Fixed test * Now double-quoting arguments only for `conda_install`, not `install`. Apparently, `pip` does not need it. * Fixed mock error in test * blackened * Fixed lint error and code review related mod of comments Co-authored-by: Sylvain MARIE --- nox/sessions.py | 33 +++++++++++++++++++++++++++++++++ tests/test_sessions.py | 40 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/nox/sessions.py b/nox/sessions.py index 643f21c2..e88f3e5d 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -70,6 +70,36 @@ def _normalize_path(envdir: str, path: Union[str, bytes]) -> str: return full_path +def _dblquote_pkg_install_args(args: Tuple[str, ...]) -> Tuple[str, ...]: + """Double-quote package install arguments in case they contain '>' or '<' symbols""" + + # routine used to handle a single arg + def _dblquote_pkg_install_arg(pkg_req_str: str) -> str: + # sanity check: we need an even number of double-quotes + if pkg_req_str.count('"') % 2 != 0: + raise ValueError( + "ill-formated argument with odd number of quotes: %s" % pkg_req_str + ) + + if "<" in pkg_req_str or ">" in pkg_req_str: + if pkg_req_str[0] == '"' and pkg_req_str[-1] == '"': + # already double-quoted string + return pkg_req_str + else: + # need to double-quote string + if '"' in pkg_req_str: + raise ValueError( + "Cannot escape requirement string: %s" % pkg_req_str + ) + return '"%s"' % pkg_req_str + else: + # no dangerous char: no need to double-quote string + return pkg_req_str + + # double-quote all args that need to be and return the result + return tuple(_dblquote_pkg_install_arg(a) for a in args) + + class _SessionQuit(Exception): pass @@ -332,6 +362,9 @@ def conda_install( if not args: raise ValueError("At least one argument required to install().") + # Escape args that should be (conda-specific; pip install does not need this) + args = _dblquote_pkg_install_args(args) + if "silent" not in kwargs: kwargs["silent"] = True diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 1da6f88f..afd691b5 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -302,6 +302,22 @@ def test_conda_install_bad_args(self): with pytest.raises(ValueError, match="arg"): session.conda_install() + def test_conda_install_bad_args_odd_nb_double_quotes(self): + session, runner = self.make_session_and_runner() + runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) + runner.venv.location = "./not/a/location" + + with pytest.raises(ValueError, match="odd number of quotes"): + session.conda_install('a"a') + + def test_conda_install_bad_args_cannot_escape(self): + session, runner = self.make_session_and_runner() + runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) + runner.venv.location = "./not/a/location" + + with pytest.raises(ValueError, match="Cannot escape"): + session.conda_install('a"o" Date: Sat, 22 Aug 2020 19:09:56 -0700 Subject: [PATCH 15/91] Release 2020.8.22 (#341) --- CHANGELOG.md | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7ad61fd..3bfd3579 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 2020.8.22 + +- `conda_install` and `install` args are now automatically double-quoted when needed. (#312) +- Offline mode is now auto-detected by default by `conda_install`. This allows users to continue executing Nox sessions on already installed environments. (#314) +- Fix the default paths for Conda on Windows where the `python.exe` found was not the correct one. (#310) +- Add the `--add-timestamp` option (#323) +- Add `Session.run_always()`. (#331) + ## 2020.5.24 - Add new options for `venv_backend`, including the ability to set the backend globally. (#326) diff --git a/setup.py b/setup.py index d368a0c7..0f21e8b1 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="nox", - version="2020.5.24", + version="2020.8.22", description="Flexible test automation.", long_description=long_description, url="https://nox.thea.codes", From b704678fca92bd6ed11a157740618fe6dc20c55b Mon Sep 17 00:00:00 2001 From: layday <31134424+layday@users.noreply.github.com> Date: Fri, 4 Sep 2020 06:34:22 +0300 Subject: [PATCH 16/91] Export `Session` in `__init__` (#344) Closes #343. --- nox/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nox/__init__.py b/nox/__init__.py index d83dbebb..75392036 100644 --- a/nox/__init__.py +++ b/nox/__init__.py @@ -16,5 +16,6 @@ from nox._parametrize import Param as param from nox._parametrize import parametrize_decorator as parametrize from nox.registry import session_decorator as session +from nox.sessions import Session -__all__ = ["parametrize", "param", "session", "options"] +__all__ = ["parametrize", "param", "session", "options", "Session"] From 4deea887faf00634bd87d6e594c8eca4d8612991 Mon Sep 17 00:00:00 2001 From: layday <31134424+layday@users.noreply.github.com> Date: Fri, 4 Sep 2020 06:36:00 +0300 Subject: [PATCH 17/91] Fully annotate the session decorator (#342) * Fully annotate the session decorator Previously the decorator would obscure the function type. * Try to get setuptools working on Travis See https://github.com/pypa/setuptools/issues/2353. * fixup! Fully annotate the session decorator * Add `@overload` to coverage excludes --- .coveragerc | 1 + .travis.yml | 1 + nox/registry.py | 27 ++++++++++++++++++++++++--- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.coveragerc b/.coveragerc index 7d632a8c..92cae43f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,3 +7,4 @@ omit = exclude_lines = pragma: no cover if _typing.TYPE_CHECKING: + @overload diff --git a/.travis.yml b/.travis.yml index 0a3b27af..053eaa6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ before_install: # Prefer the built-in python binary. - export PATH="$PATH:/home/travis/miniconda3/bin" - conda update --yes conda + - export SETUPTOOLS_USE_DISTUTILS=stdlib install: - pip install --upgrade pip setuptools - pip install . diff --git a/nox/registry.py b/nox/registry.py index 3a3d0a54..3bf74a78 100644 --- a/nox/registry.py +++ b/nox/registry.py @@ -15,23 +15,44 @@ import collections import copy import functools -from typing import Any, Callable, Optional +from typing import Any, Callable, Optional, TypeVar, Union, overload from ._decorators import Func from ._typing import Python +F = TypeVar("F", bound=Callable[..., Any]) +G = Callable[[F], F] + _REGISTRY = collections.OrderedDict() # type: collections.OrderedDict[str, Func] +@overload +def session_decorator(__func: F) -> F: + ... + + +@overload +def session_decorator( + __func: None = ..., + python: Python = ..., + py: Python = ..., + reuse_venv: Optional[bool] = ..., + name: Optional[str] = ..., + venv_backend: Any = ..., + venv_params: Any = ..., +) -> G: + ... + + def session_decorator( - func: Optional[Callable] = None, + func: Optional[F] = None, python: Python = None, py: Python = None, reuse_venv: Optional[bool] = None, name: Optional[str] = None, venv_backend: Any = None, venv_params: Any = None, -) -> Callable: +) -> Union[F, G]: """Designate the decorated function as a session.""" # If `func` is provided, then this is the decorator call with the function # being sent as part of the Python syntax (`@nox.session`). From 9caaba5acbd29e4b28cc02905314c5cd66f5e4f4 Mon Sep 17 00:00:00 2001 From: srenfo <14216995+srenfo@users.noreply.github.com> Date: Tue, 22 Sep 2020 19:54:26 +0200 Subject: [PATCH 18/91] Fix typo in Tutorial (#351) --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 8be056dc..1a15b773 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -110,7 +110,7 @@ To install a ``requirements.txt`` file: @nox.session def tests(session): - # same as pip install -r -requirements.txt + # same as pip install -r requirements.txt session.install("-r", "requirements.txt") ... From e78d8d9a401a0aa0f0c291ab11c04d7b0754e8f2 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Fri, 13 Nov 2020 17:25:59 +0100 Subject: [PATCH 19/91] Do not merge command-line options in place (#357) When merging options specified in the noxfile and on the command-line option, do not use the output parameter `command_args` as the input for the merge; instead, copy `command_args` initially and pass the copy to the merge functions. Merge functions such as `_session_filters_merge_func` inspect `command_args` to see if other options have been specified on the command-line. When the options are merged in place, this check produces false positives. For example, `nox.options.sessions` is copied into `command_args` as a part of the merge; so it will appear to have been specified on the command-line when merging `nox.options.pythons`, causing the latter to be ignored. Co-authored-by: Claudio Jolowicz --- nox/_option_set.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nox/_option_set.py b/nox/_option_set.py index 0c91e9cc..4bcbb357 100644 --- a/nox/_option_set.py +++ b/nox/_option_set.py @@ -304,13 +304,16 @@ def merge_namespaces( self, command_args: Namespace, noxfile_args: Namespace ) -> None: """Merges the command-line options with the noxfile options.""" + command_args_copy = Namespace(**vars(command_args)) for name, option in self.options.items(): if option.merge_func: setattr( - command_args, name, option.merge_func(command_args, noxfile_args) + command_args, + name, + option.merge_func(command_args_copy, noxfile_args), ) elif option.noxfile: - value = getattr(command_args, name, None) or getattr( + value = getattr(command_args_copy, name, None) or getattr( noxfile_args, name, None ) setattr(command_args, name, value) From 6bdf51ca64e33b9de0d1b114d7b892d86f87096d Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sat, 14 Nov 2020 04:29:28 +0100 Subject: [PATCH 20/91] Decouple merging of --python with nox.options from --sessions and --keywords (#359) * Add test fixture to generate noxfile.py The noxfile.py is templated with the default session (`nox.options.sessions`), and the default Python version (`nox.options.pythons`), as well as an alternate Python version. This allows us to avoid the situation where a test case running on one Python version needs to launch Nox using another Python version. As a side-effect, it makes the test cases a bit more explicit. * Add test case for using --pythons with nox.options.sessions * Add test case for using --sessions with nox.options.pythons * Do not ignore nox.options.pythons when --{sessions,keywords} passed * Do not ignore nox.options.{sessions,keywords} when --pythons passed * Rename _{session_filters => sessions_and_keywords}_merge_func Revert function name to the one used before --pythons was introduced. --- nox/_options.py | 17 +++-- tests/resources/noxfile_options_pythons.py | 28 ++++++++ tests/test_main.py | 83 ++++++++++++++++++++++ 3 files changed, 119 insertions(+), 9 deletions(-) create mode 100644 tests/resources/noxfile_options_pythons.py diff --git a/nox/_options.py b/nox/_options.py index 336cff5e..dcdcee2b 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -41,21 +41,21 @@ ) -def _session_filters_merge_func( +def _sessions_and_keywords_merge_func( key: str, command_args: argparse.Namespace, noxfile_args: argparse.Namespace ) -> List[str]: - """Only return the Noxfile value for sessions/pythons/keywords if neither sessions, - pythons or keywords are specified on the command-line. + """Only return the Noxfile value for sessions/keywords if neither sessions + or keywords are specified on the command-line. Args: - key (str): This function is used for the "sessions", "pythons" and "keywords" + key (str): This function is used for both the "sessions" and "keywords" options, this allows using ``funtools.partial`` to pass the same function for both options. command_args (_option_set.Namespace): The options specified on the command-line. - noxfile_args (_option_set.Namespace): The options specified in the + noxfile_Args (_option_set.Namespace): The options specified in the Noxfile.""" - if not any((command_args.sessions, command_args.pythons, command_args.keywords)): + if not command_args.sessions and not command_args.keywords: return getattr(noxfile_args, key) return getattr(command_args, key) @@ -221,7 +221,7 @@ def _session_completer( "--session", group=options.groups["primary"], noxfile=True, - merge_func=functools.partial(_session_filters_merge_func, "sessions"), + merge_func=functools.partial(_sessions_and_keywords_merge_func, "sessions"), nargs="*", default=_sessions_default, help="Which sessions to run. By default, all sessions will run.", @@ -234,7 +234,6 @@ def _session_completer( "--python", group=options.groups["primary"], noxfile=True, - merge_func=functools.partial(_session_filters_merge_func, "pythons"), nargs="*", help="Only run sessions that use the given python interpreter versions.", ), @@ -244,7 +243,7 @@ def _session_completer( "--keywords", group=options.groups["primary"], noxfile=True, - merge_func=functools.partial(_session_filters_merge_func, "keywords"), + merge_func=functools.partial(_sessions_and_keywords_merge_func, "keywords"), help="Only run sessions that match the given expression.", ), _option_set.Option( diff --git a/tests/resources/noxfile_options_pythons.py b/tests/resources/noxfile_options_pythons.py new file mode 100644 index 00000000..000d4683 --- /dev/null +++ b/tests/resources/noxfile_options_pythons.py @@ -0,0 +1,28 @@ +# Copyright 2020 Alethea Katherine Flowers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nox + +nox.options.sessions = ["{default_session}"] +nox.options.pythons = ["{default_python}"] + + +@nox.session(python=["{default_python}", "{alternate_python}"]) +def test(session): + pass + + +@nox.session(python=["{default_python}", "{alternate_python}"]) +def launch_rocket(session): + pass diff --git a/tests/test_main.py b/tests/test_main.py index c9ef5467..0a1322a0 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -14,6 +14,7 @@ import os import sys +from pathlib import Path from unittest import mock import contexter @@ -444,6 +445,88 @@ def test_main_noxfile_options_sessions(monkeypatch): assert config.sessions == ["test"] +@pytest.fixture +def generate_noxfile_options_pythons(tmp_path): + """Generate noxfile.py with test and launch_rocket sessions. + + The sessions are defined for both the default and alternate Python versions. + The ``default_session`` and ``default_python`` parameters determine what + goes into ``nox.options.sessions`` and ``nox.options.pythons``, respectively. + """ + + def generate_noxfile(default_session, default_python, alternate_python): + path = Path(RESOURCES) / "noxfile_options_pythons.py" + text = path.read_text() + text = text.format( + default_session=default_session, + default_python=default_python, + alternate_python=alternate_python, + ) + path = tmp_path / "noxfile.py" + path.write_text(text) + return str(path) + + return generate_noxfile + + +python_current_version = "{}.{}".format(sys.version_info.major, sys.version_info.minor) +python_next_version = "{}.{}".format(sys.version_info.major, sys.version_info.minor + 1) + + +def test_main_noxfile_options_with_pythons_override( + capsys, monkeypatch, generate_noxfile_options_pythons +): + noxfile = generate_noxfile_options_pythons( + default_session="test", + default_python=python_next_version, + alternate_python=python_current_version, + ) + + monkeypatch.setattr( + sys, "argv", ["nox", "--noxfile", noxfile, "--python", python_current_version] + ) + + with mock.patch("sys.exit") as sys_exit: + nox.__main__.main() + _, stderr = capsys.readouterr() + sys_exit.assert_called_once_with(0) + + for python_version in [python_current_version, python_next_version]: + for session in ["test", "launch_rocket"]: + line = "Running session {}-{}".format(session, python_version) + if session == "test" and python_version == python_current_version: + assert line in stderr + else: + assert line not in stderr + + +def test_main_noxfile_options_with_sessions_override( + capsys, monkeypatch, generate_noxfile_options_pythons +): + noxfile = generate_noxfile_options_pythons( + default_session="test", + default_python=python_current_version, + alternate_python=python_next_version, + ) + + monkeypatch.setattr( + sys, "argv", ["nox", "--noxfile", noxfile, "--session", "launch_rocket"] + ) + + with mock.patch("sys.exit") as sys_exit: + nox.__main__.main() + _, stderr = capsys.readouterr() + sys_exit.assert_called_once_with(0) + + for python_version in [python_current_version, python_next_version]: + for session in ["test", "launch_rocket"]: + line = "Running session {}-{}".format(session, python_version) + if session == "launch_rocket" and python_version == python_current_version: + assert line in stderr + else: + assert line not in stderr + + @pytest.mark.parametrize(("isatty_value", "expected"), [(True, True), (False, False)]) def test_main_color_from_isatty(monkeypatch, isatty_value, expected): monkeypatch.setattr(sys, "argv", [sys.executable]) From e5e9869c38d75eee01246b8e380d9867cf706e94 Mon Sep 17 00:00:00 2001 From: Christopher Wilcox Date: Sat, 21 Nov 2020 11:59:28 -0800 Subject: [PATCH 21/91] Update nox to latest supported python versions. (#362) * Update parameters to match latest python releases. * update setup.py, other references to 3.5 * don't use miniconda 3.8 even though https://www.appveyor.com/docs/windows-images-software/ mentions it --- .travis.yml | 4 ++-- CONTRIBUTING.md | 3 ++- appveyor.yml | 17 ++++++++++------- docs/config.rst | 4 ++-- docs/tutorial.rst | 4 ++-- noxfile.py | 4 ++-- setup.py | 4 ++-- 7 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 053eaa6d..f14966fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,14 +2,14 @@ language: python dist: xenial matrix: include: - - python: '3.5' - env: NOXSESSION="tests-3.5" - python: '3.6' env: NOXSESSION="tests-3.6" - python: '3.7' env: NOXSESSION="tests-3.7" - python: '3.8' env: NOXSESSION="tests-3.8" + - python: '3.9' + env: NOXSESSION="tests-3.9" - python: '3.8' env: NOXSESSION="lint" - python: '3.8' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 46b79f0d..f911bd1e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,10 +39,11 @@ To just check for lint errors, run: To run against a particular Python version: - nox --session tests-3.5 nox --session tests-3.6 nox --session tests-3.7 nox --session tests-3.8 + nox --session tests-3.9 + When you send a pull request Travis will handle running everything, but it is recommended to test as much as possible locally before pushing. diff --git a/appveyor.yml b/appveyor.yml index 90cdf763..d853541e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,6 +10,16 @@ environment: # a later point release. # See: http://www.appveyor.com/docs/installed-software#python + - PYTHON: "C:\\Python39" + # There is no miniconda for python3.9 at this time + CONDA: "C:\\Miniconda37" + NOX_SESSION: "tests-3.9" + + - PYTHON: "C:\\Python39-x64" + # There is no miniconda for python3.9 at this time + CONDA: "C:\\Miniconda37-x64" + NOX_SESSION: "tests-3.9" + - PYTHON: "C:\\Python38" # There is no miniconda for python3.8 at this time CONDA: "C:\\Miniconda37" @@ -36,13 +46,6 @@ environment: CONDA: "C:\\Miniconda36-x64" NOX_SESSION: "tests-3.6" - - PYTHON: "C:\\Python35" - CONDA: "C:\\Miniconda35" - NOX_SESSION: "tests-3.5" - - - PYTHON: "C:\\Python35-x64" - CONDA: "C:\\Miniconda35-x64" - NOX_SESSION: "tests-3.5" install: # Add conda command to path. diff --git a/docs/config.rst b/docs/config.rst index 39628b08..f76307bf 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -116,7 +116,7 @@ When collecting your sessions, Nox will create a separate session for each inter .. code-block:: python - @nox.session(python=['2.7', '3.5', '3.6', '3.7', '3.8']) + @nox.session(python=['2.7', '3.6', '3.7', '3.8', '3.9']) def tests(session): pass @@ -125,10 +125,10 @@ Will produce these sessions: .. code-block:: console * tests-2.7 - * tests-3.5 * tests-3.6 * tests-3.7 * tests-3.8 + * tests-3.9 Note that this expansion happens *before* parameterization occurs, so you can still parametrize sessions with multiple interpreters. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 1a15b773..0eead976 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -282,7 +282,7 @@ If you want your session to run against multiple versions of Python: .. code-block:: python - @nox.session(python=["2.7", "3.5", "3.7"]) + @nox.session(python=["2.7", "3.6", "3.7"]) def test(session): ... @@ -294,7 +294,7 @@ been expanded into three distinct sessions: Sessions defined in noxfile.py: * test-2.7 - * test-3.5 + * test-3.6 * test-3.7 You can run all of the ``test`` sessions using ``nox --sessions test`` or run diff --git a/noxfile.py b/noxfile.py index 84845c5f..7d3094e2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -28,7 +28,7 @@ def is_python_version(session, version): return py_version.startswith(version) -@nox.session(python=["3.5", "3.6", "3.7", "3.8"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) def tests(session): """Run test suite with pytest.""" session.create_tmp() @@ -44,7 +44,7 @@ def tests(session): session.notify("cover") -@nox.session(python=["3.5", "3.6", "3.7", "3.8"], venv_backend="conda") +@nox.session(python=["3.6", "3.7", "3.8", "3.9"], venv_backend="conda") def conda_tests(session): """Run test suite with pytest.""" session.create_tmp() diff --git a/setup.py b/setup.py index 0f21e8b1..475d9b7b 100644 --- a/setup.py +++ b/setup.py @@ -35,10 +35,10 @@ "Environment :: Console", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Operating System :: POSIX", "Operating System :: MacOS", "Operating System :: Unix", @@ -68,5 +68,5 @@ "Source Code": "https://github.com/theacodes/nox", "Bug Tracker": "https://github.com/theacodes/nox/issues", }, - python_requires=">=3.5", + python_requires=">=3.6", ) From 495b23ff690049e8f3d25f3d29c06458da2ee274 Mon Sep 17 00:00:00 2001 From: "P. Sai Vinay" <33659563+V1NAY8@users.noreply.github.com> Date: Sun, 22 Nov 2020 01:29:48 +0530 Subject: [PATCH 22/91] Add py.typed to manifest.in (#360) --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 33cd59d9..053167fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include LICENSE recursive-include nox *.jinja2 +include nox/py.typed From 6dc712a01af49e8b1bface8c9ad7979b0511d419 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Mon, 9 Nov 2020 09:44:45 +0100 Subject: [PATCH 23/91] Sort imports using isort Run the blacken session on the codebase. --- nox/command.py | 2 +- nox/virtualenv.py | 4 ++-- tests/test_logger.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nox/command.py b/nox/command.py index b2e99cef..a9a14a82 100644 --- a/nox/command.py +++ b/nox/command.py @@ -14,7 +14,7 @@ import os import sys -from typing import Any, Iterable, Optional, Sequence, Union, List +from typing import Any, Iterable, List, Optional, Sequence, Union import py from nox.logger import logger diff --git a/nox/virtualenv.py b/nox/virtualenv.py index d3f48b6f..54c3b4df 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -16,9 +16,9 @@ import platform import re import shutil -from socket import gethostbyname import sys -from typing import Any, Mapping, Optional, Tuple, Union, List +from socket import gethostbyname +from typing import Any, List, Mapping, Optional, Tuple, Union import nox.command import py diff --git a/tests/test_logger.py b/tests/test_logger.py index eef9dcf3..50563bd5 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -16,7 +16,6 @@ from unittest import mock import pytest - from nox import logger From bbe2cd8405f075d21cd3f0a11effb4562339f090 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Mon, 30 Nov 2020 17:46:55 +0100 Subject: [PATCH 24/91] Support double-digit minor version in `python` keyword (#367) * Add test for resolving python3.10 * Support double-digit minor version in `python` keyword This fixes an issue where "3.10" is resolved as "python3". --- nox/virtualenv.py | 4 ++-- tests/test_virtualenv.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 54c3b4df..4a086e0f 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -330,7 +330,7 @@ def _resolved_interpreter(self) -> str: # If this is just a X, X.Y, or X.Y.Z string, extract just the X / X.Y # part and add Python to the front of it. - match = re.match(r"^(?P\d(\.\d)?)(\.\d+)?$", self.interpreter) + match = re.match(r"^(?P\d(\.\d+)?)(\.\d+)?$", self.interpreter) if match: xy_version = match.group("xy_ver") cleaned_interpreter = "python{}".format(xy_version) @@ -347,7 +347,7 @@ def _resolved_interpreter(self) -> str: raise self._resolved # Allow versions of the form ``X.Y-32`` for Windows. - match = re.match(r"^\d\.\d-32?$", cleaned_interpreter) + match = re.match(r"^\d\.\d+-32?$", cleaned_interpreter) if match: # preserve the "-32" suffix, as the Python launcher expects # it. diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 0f852108..eb67694a 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -330,6 +330,7 @@ def test__resolved_interpreter_none(make_one): ("3", "python3"), ("3.6", "python3.6"), ("3.6.2", "python3.6"), + ("3.10", "python3.10"), ("2.7.15", "python2.7"), ], ) From 4e1842246d8b005e0dadf4b0f3e9d1123cefa651 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Tue, 1 Dec 2020 14:45:17 +0100 Subject: [PATCH 25/91] Add isort check to lint session (#366) --- noxfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 7d3094e2..063e36cb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -80,7 +80,7 @@ def blacken(session): @nox.session(python="3.8") def lint(session): - session.install("flake8==3.7.8", "black==19.3b0", "mypy==0.720") + session.install("flake8==3.7.8", "black==19.3b0", "isort==4.3.21", "mypy==0.720") session.run( "mypy", "--disallow-untyped-defs", @@ -90,6 +90,7 @@ def lint(session): ) files = ["nox", "tests", "noxfile.py", "setup.py"] session.run("black", "--check", *files) + session.run("isort", "--check", "--recursive", *files) session.run("flake8", "nox", *files) From a188be7834ef153105eab72b5cb18a2b465df57e Mon Sep 17 00:00:00 2001 From: layday <31134424+layday@users.noreply.github.com> Date: Sat, 19 Dec 2020 03:58:40 +0200 Subject: [PATCH 26/91] Fix nested decorator annotation (#370) Type checkers cannot solve an aliased, generic callable. --- nox/registry.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nox/registry.py b/nox/registry.py index 3bf74a78..f08b188a 100644 --- a/nox/registry.py +++ b/nox/registry.py @@ -21,7 +21,6 @@ from ._typing import Python F = TypeVar("F", bound=Callable[..., Any]) -G = Callable[[F], F] _REGISTRY = collections.OrderedDict() # type: collections.OrderedDict[str, Func] @@ -40,7 +39,7 @@ def session_decorator( name: Optional[str] = ..., venv_backend: Any = ..., venv_params: Any = ..., -) -> G: +) -> Callable[[F], F]: ... @@ -52,7 +51,7 @@ def session_decorator( name: Optional[str] = None, venv_backend: Any = None, venv_params: Any = None, -) -> Union[F, G]: +) -> Union[F, Callable[[F], F]]: """Designate the decorated function as a session.""" # If `func` is provided, then this is the decorator call with the function # being sent as part of the Python syntax (`@nox.session`). From cdb391fa24a766138e596ee6b819ae49add5ec4f Mon Sep 17 00:00:00 2001 From: "P. Sai Vinay" <33659563+V1NAY8@users.noreply.github.com> Date: Tue, 29 Dec 2020 05:23:44 +0530 Subject: [PATCH 27/91] Refactor pip as python -m pip (#363) --- .travis.yml | 4 ++-- docs/config.rst | 4 ++-- docs/usage.rst | 4 ++-- nox/sessions.py | 2 +- tests/test_sessions.py | 20 +++++++++++++++++--- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index f14966fd..34f9411d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,8 @@ before_install: - conda update --yes conda - export SETUPTOOLS_USE_DISTUTILS=stdlib install: - - pip install --upgrade pip setuptools - - pip install . + - python -m pip install --upgrade pip setuptools + - python -m pip install . script: nox --non-interactive --session "$NOXSESSION" deploy: provider: pypi diff --git a/docs/config.rst b/docs/config.rst index f76307bf..14b17da1 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -237,10 +237,10 @@ When you run ``nox``, it will create a two distinct sessions: $ nox nox > Running session tests(django='1.9') - nox > pip install django==1.9 + nox > python -m pip install django==1.9 ... nox > Running session tests(django='2.0') - nox > pip install django==2.0 + nox > python -m pip install django==2.0 :func:`nox.parametrize` has an interface and usage intentionally similar to `pytest's parametrize `_. diff --git a/docs/usage.rst b/docs/usage.rst index ef242260..f3bf4fd3 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -249,8 +249,8 @@ Would run both ``install`` commands, but skip the ``run`` command: nox > Running session tests nox > Creating virtualenv using python3.7 in ./.nox/tests - nox > pip install pytest - nox > pip install . + nox > python -m pip install pytest + nox > python -m pip install . nox > Skipping pytest run, as --install-only is set. nox > Session tests was successful. diff --git a/nox/sessions.py b/nox/sessions.py index e88f3e5d..396a09fc 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -423,7 +423,7 @@ def install(self, *args: str, **kwargs: Any) -> None: if "silent" not in kwargs: kwargs["silent"] = True - self._run("pip", "install", *args, external="error", **kwargs) + self._run("python", "-m", "pip", "install", *args, external="error", **kwargs) def notify(self, target: "Union[str, SessionRunner]") -> None: """Place the given session at the end of the queue. diff --git a/tests/test_sessions.py b/tests/test_sessions.py index afd691b5..e1e19eae 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -192,7 +192,7 @@ def test_run_install_only_should_install(self): session.run("spam", "eggs") run.assert_called_once_with( - ("pip", "install", "spam"), + ("python", "-m", "pip", "install", "spam"), env=mock.ANY, external=mock.ANY, paths=mock.ANY, @@ -445,7 +445,14 @@ class SessionNoSlots(nox.sessions.Session): with mock.patch.object(session, "_run", autospec=True) as run: session.install("requests", "urllib3") run.assert_called_once_with( - "pip", "install", "requests", "urllib3", silent=True, external="error" + "python", + "-m", + "pip", + "install", + "requests", + "urllib3", + silent=True, + external="error", ) def test_install_non_default_kwargs(self): @@ -467,7 +474,14 @@ class SessionNoSlots(nox.sessions.Session): with mock.patch.object(session, "_run", autospec=True) as run: session.install("requests", "urllib3", silent=False) run.assert_called_once_with( - "pip", "install", "requests", "urllib3", silent=False, external="error" + "python", + "-m", + "pip", + "install", + "requests", + "urllib3", + silent=False, + external="error", ) def test_notify(self): From e2db5ed44ce7c53970c9b097a7c3143219c74fd9 Mon Sep 17 00:00:00 2001 From: Christopher Wilcox Date: Tue, 29 Dec 2020 15:09:18 -0800 Subject: [PATCH 28/91] feat: support users specifying an undeclared parametrization of python (#361) * feat: support users specifying an undeclared parametrization of python Co-authored-by: Danny Hermes Co-authored-by: Claudio Jolowicz --- docs/usage.rst | 22 ++++++++++++++++++++++ nox/_options.py | 8 ++++++++ nox/manifest.py | 21 +++++++++++++++++++++ tests/test_manifest.py | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index f3bf4fd3..098609f2 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -159,6 +159,28 @@ By default, Nox deletes and recreates virtualenvs every time it is run. This is If the Noxfile sets ``nox.options.reuse_existing_virtualenvs``, you can override the Noxfile setting from the command line by using ``--no-reuse-existing-virtualenvs``. +.. _opt-running-extra-pythons: + +Running additional Python versions +---------------------------------- +In addition to Nox supporting executing single sessions, it also supports runnings python versions that aren't specified using ``--extra-pythons``. + +.. code-block:: console + + nox --extra-pythons 3.8 3.9 + +This will, in addition to specified python versions in the Noxfile, also create sessions for the specified versions. + +This option can be combined with ``--python`` to replace, instead of appending, the Python interpreter for a given session:: + + nox --python 3.10 --extra-python 3.10 -s lint + +Also, you can specify ``python`` in place of a specific version. This will run the session +using the ``python`` specified for the current ``PATH``:: + + nox --python python --extra-python python -s lint + + .. _opt-stop-on-first-error: Stopping if any session fails diff --git a/nox/_options.py b/nox/_options.py index dcdcee2b..8f66b1aa 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -327,6 +327,14 @@ def _session_completer( group=options.groups["secondary"], help="Directory where nox will store virtualenvs, this is ``.nox`` by default.", ), + _option_set.Option( + "extra_pythons", + "--extra-pythons", + "--extra-python", + group=options.groups["secondary"], + nargs="*", + help="Additionally, run sessions using the given python interpreter versions.", + ), *_option_set.make_flag_pair( "stop_on_first_error", ("-x", "--stop-on-first-error"), diff --git a/nox/manifest.py b/nox/manifest.py index eb6d8b5b..616de130 100644 --- a/nox/manifest.py +++ b/nox/manifest.py @@ -15,6 +15,7 @@ import argparse import collections.abc import itertools +from collections import OrderedDict from typing import Any, Iterable, Iterator, List, Mapping, Sequence, Set, Tuple, Union from nox._decorators import Call, Func @@ -23,6 +24,11 @@ WARN_PYTHONS_IGNORED = "python_ignored" +def _unique_list(*args: str) -> List[str]: + """Return a list without duplicates, while preserving order.""" + return list(OrderedDict.fromkeys(args)) + + class Manifest: """Session manifest. @@ -184,6 +190,21 @@ def make_session( func.should_warn[WARN_PYTHONS_IGNORED] = func.python func.python = False + if self._config.extra_pythons: + # If extra python is provided, expand the func.python list to + # include additional python interpreters + extra_pythons = self._config.extra_pythons # type: List[str] + if isinstance(func.python, (list, tuple, set)): + func.python = _unique_list(*func.python, *extra_pythons) + elif not multi and func.python: + # If this is multi, but there is only a single interpreter, it + # is the reentrant case. The extra_python interpreter shouldn't + # be added in that case. If func.python is False, the session + # has no backend; if None, it uses the same interpreter as Nox. + # Otherwise, add the extra specified python. + assert isinstance(func.python, str) + func.python = _unique_list(func.python, *extra_pythons) + # If the func has the python attribute set to a list, we'll need # to expand them. if isinstance(func.python, (list, tuple, set)): diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 20f27f66..a3631511 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -37,6 +37,7 @@ def create_mock_config(): cfg = mock.sentinel.CONFIG cfg.force_venv_backend = None cfg.default_venv_backend = None + cfg.extra_pythons = None return cfg @@ -203,6 +204,42 @@ def session_func(): assert len(manifest) == 2 +@pytest.mark.parametrize( + "python,extra_pythons,expected", + [ + (None, [], [None]), + (None, ["3.8"], [None]), + (None, ["3.8", "3.9"], [None]), + (False, [], [False]), + (False, ["3.8"], [False]), + (False, ["3.8", "3.9"], [False]), + ("3.5", [], ["3.5"]), + ("3.5", ["3.8"], ["3.5", "3.8"]), + ("3.5", ["3.8", "3.9"], ["3.5", "3.8", "3.9"]), + (["3.5", "3.9"], [], ["3.5", "3.9"]), + (["3.5", "3.9"], ["3.8"], ["3.5", "3.9", "3.8"]), + (["3.5", "3.9"], ["3.8", "3.4"], ["3.5", "3.9", "3.8", "3.4"]), + (["3.5", "3.9"], ["3.5", "3.9"], ["3.5", "3.9"]), + ], +) +def test_extra_pythons(python, extra_pythons, expected): + cfg = mock.sentinel.CONFIG + cfg.force_venv_backend = None + cfg.default_venv_backend = None + cfg.extra_pythons = extra_pythons + + manifest = Manifest({}, cfg) + + def session_func(): + pass + + func = Func(session_func, python=python) + for session in manifest.make_session("my_session", func): + manifest.add_session(session) + + assert expected == [session.func.python for session in manifest._all_sessions] + + def test_add_session_parametrized(): manifest = Manifest({}, create_mock_config()) From 4d42022764386eaf96639bd08030483604de5696 Mon Sep 17 00:00:00 2001 From: Tolker-KU <55140581+Tolker-KU@users.noreply.github.com> Date: Thu, 31 Dec 2020 09:40:08 +0100 Subject: [PATCH 29/91] Use conda remove to clean up existing conda environment. Closes https://github.com/theacodes/nox/issues/372. (#373) Co-authored-by: Santos Gallegos Co-authored-by: Christopher Wilcox --- nox/virtualenv.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 4a086e0f..c32efaf2 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -139,17 +139,6 @@ def locate_using_path_and_version(version: str) -> Optional[str]: return None -def _clean_location(self: "Union[CondaEnv, VirtualEnv]") -> bool: - """Deletes any existing path-based environment""" - if os.path.exists(self.location): - if self.reuse_existing: - return False - else: - shutil.rmtree(self.location) - - return True - - class PassthroughEnv(ProcessEnv): """Represents the environment used to run nox itself @@ -200,7 +189,21 @@ def __init__( self.venv_params = venv_params if venv_params else [] super(CondaEnv, self).__init__() - _clean_location = _clean_location + def _clean_location(self) -> bool: + """Deletes existing conda environment""" + if os.path.exists(self.location): + if self.reuse_existing: + return False + else: + cmd = ["conda", "remove", "--yes", "--prefix", self.location, "--all"] + nox.command.run(cmd, silent=True, log=False) + # Make sure that location is clean + try: + shutil.rmtree(self.location) + except FileNotFoundError: + pass + + return True @property def bin_paths(self) -> List[str]: @@ -302,7 +305,15 @@ def __init__( self.venv_params = venv_params if venv_params else [] super(VirtualEnv, self).__init__(env={"VIRTUAL_ENV": self.location}) - _clean_location = _clean_location + def _clean_location(self) -> bool: + """Deletes any existing virtual environment""" + if os.path.exists(self.location): + if self.reuse_existing: + return False + else: + shutil.rmtree(self.location) + + return True @property def _resolved_interpreter(self) -> str: From 6f239140895e7e9fcea96f68f853c5a304d92d60 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 31 Dec 2020 11:57:44 -0500 Subject: [PATCH 30/91] Fix NoxColoredFormatter.format (#374) That method is making use of `_simple_fmt`, but isn't defined in the class. I used the same value as `NoxFormatter` --- nox/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nox/logger.py b/nox/logger.py index 84d5fcca..599cc238 100644 --- a/nox/logger.py +++ b/nox/logger.py @@ -63,6 +63,7 @@ def __init__( reset=reset, secondary_log_colors=secondary_log_colors, ) + self._simple_fmt = logging.Formatter("%(message)s") def format(self, record: Any) -> str: if record.levelname == "OUTPUT": @@ -72,7 +73,7 @@ def format(self, record: Any) -> str: class LoggerWithSuccessAndOutput(logging.getLoggerClass()): # type: ignore def __init__(self, name: str, level: int = logging.NOTSET): - super(LoggerWithSuccessAndOutput, self).__init__(name, level) + super().__init__(name, level) logging.addLevelName(SUCCESS, "SUCCESS") logging.addLevelName(OUTPUT, "OUTPUT") From 319c796f6de7d5706d75da107ec20568ad5baf51 Mon Sep 17 00:00:00 2001 From: Christopher Wilcox Date: Thu, 31 Dec 2020 17:11:36 -0800 Subject: [PATCH 31/91] Release 2020.12.31 (#375) --- CHANGELOG.md | 10 ++++++++++ setup.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bfd3579..3a6675d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 2020.12.31 +- Fix `NoxColoredFormatter.format`(#374) +- Use conda remove to clean up existing conda environments (#373) +- Support users specifying an undeclared parametrization of python via `--extra-python` (#361) +- Support double-digit minor version in `python` keyword (#367) +- Add `py.typed` to `manifest.in` (#360) +- Update nox to latest supported python versions. (#362) +- Decouple merging of `--python` with `nox.options` from `--sessions` and `--keywords` (#359) +- Do not merge command-line options in place (#357) + ## 2020.8.22 - `conda_install` and `install` args are now automatically double-quoted when needed. (#312) diff --git a/setup.py b/setup.py index 475d9b7b..cd8d73a7 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="nox", - version="2020.8.22", + version="2020.12.31", description="Flexible test automation.", long_description=long_description, url="https://nox.thea.codes", From 156765c343942233bf6cbfa59391ec63d6fedc29 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Fri, 22 Jan 2021 09:18:45 -0800 Subject: [PATCH 32/91] Allow passing PathLike types to Session.chdir() (#376) os.chdir() has supported PathLib since Python 3.6. Discovered while adding types pip's noxfile.py: https://github.com/pypa/pip/pull/9411 --- nox/sessions.py | 2 +- tests/test_sessions.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/nox/sessions.py b/nox/sessions.py index 396a09fc..d44f10b9 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -184,7 +184,7 @@ def interactive(self) -> bool: """Returns True if Nox is being run in an interactive session or False otherwise.""" return not self._runner.global_config.non_interactive and sys.stdin.isatty() - def chdir(self, dir: str) -> None: + def chdir(self, dir: Union[str, os.PathLike]) -> None: """Change the current working directory.""" self.log("cd {}".format(dir)) os.chdir(dir) diff --git a/tests/test_sessions.py b/tests/test_sessions.py index e1e19eae..326133e5 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -18,6 +18,7 @@ import os import sys import tempfile +from pathlib import Path from unittest import mock import nox.command @@ -151,6 +152,17 @@ def test_chdir(self, tmpdir): assert os.getcwd() == cdto os.chdir(current_cwd) + def test_chdir_pathlib(self, tmpdir): + cdto = str(tmpdir.join("cdbby").ensure(dir=True)) + current_cwd = os.getcwd() + + session, _ = self.make_session_and_runner() + + session.chdir(Path(cdto)) + + assert os.getcwd() == cdto + os.chdir(current_cwd) + def test_run_bad_args(self): session, _ = self.make_session_and_runner() From 10256820bd88847b275e435bdf9ee6fd3eb637b1 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 7 Feb 2021 17:05:38 -0800 Subject: [PATCH 33/91] Make ProcessEnv.bin always return a str (#378) The property never returns None. This simplifies user code containing types that already knows a bin directory must exist. It avoids the need to pepper calling code with: assert session.bin is not None Or # type: ignore For example, in pip: https://github.com/pypa/pip/blob/062f0e54d99f58e53be36be5a45adad89e2429fb/tools/automation/release/__init__.py#L29 A noxfile that tries to access a bin directory that doesn't exist will now raise an exception. --- nox/sessions.py | 6 ++++-- nox/virtualenv.py | 6 ++++-- tests/test_sessions.py | 5 ++++- tests/test_virtualenv.py | 7 +++++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/nox/sessions.py b/nox/sessions.py index d44f10b9..96d80eb3 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -167,10 +167,12 @@ def bin_paths(self) -> Optional[List[str]]: return self.virtualenv.bin_paths @property - def bin(self) -> Optional[str]: + def bin(self) -> str: """The first bin directory for the virtualenv.""" paths = self.bin_paths - return paths[0] if paths is not None else None + if paths is None: + raise ValueError("The environment does not have a bin directory.") + return paths[0] def create_tmp(self) -> str: """Create, and return, a temporary directory.""" diff --git a/nox/virtualenv.py b/nox/virtualenv.py index c32efaf2..eba573e7 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -69,10 +69,12 @@ def bin_paths(self) -> Optional[List[str]]: return self._bin_paths @property - def bin(self) -> Optional[str]: + def bin(self) -> str: """The first bin directory for the virtualenv.""" paths = self.bin_paths - return paths[0] if paths is not None else None + if paths is None: + raise ValueError("The environment does not have a bin directory.") + return paths[0] def create(self) -> bool: raise NotImplementedError("ProcessEnv.create should be overwritten in subclass") diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 326133e5..2d4cf119 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -109,7 +109,10 @@ def test_no_bin_paths(self): session, runner = self.make_session_and_runner() runner.venv.bin_paths = None - assert session.bin is None + with pytest.raises( + ValueError, match=r"^The environment does not have a bin directory\.$" + ): + session.bin assert session.bin_paths is None def test_virtualenv_as_none(self): diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index eb67694a..7200d2d1 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -121,10 +121,17 @@ def mock_sysfind(arg): def test_process_env_constructor(): penv = nox.virtualenv.ProcessEnv() assert not penv.bin_paths + with pytest.raises( + ValueError, match=r"^The environment does not have a bin directory\.$" + ): + penv.bin penv = nox.virtualenv.ProcessEnv(env={"SIGIL": "123"}) assert penv.env["SIGIL"] == "123" + penv = nox.virtualenv.ProcessEnv(bin_paths=["/bin"]) + assert penv.bin == "/bin" + def test_process_env_create(): penv = nox.virtualenv.ProcessEnv() From d6a0ba5f8a8d34198ecd381faf9093ab77c3cb5e Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 7 Feb 2021 17:06:30 -0800 Subject: [PATCH 34/91] Define the location attribute on base class ProcessEnv (#377) Fixes errors when running mypy on a project's noxfile.py: noxfile.py:42: error: "ProcessEnv" has no attribute "location" Discovered while adding types pip's noxfile.py: pypa/pip#9411 --- nox/virtualenv.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index eba573e7..0e3bbca3 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -43,6 +43,8 @@ def __init__(self, interpreter: str) -> None: class ProcessEnv: """A environment with a 'bin' directory and a set of 'env' vars.""" + location: str + # Does this environment provide any process isolation? is_sandboxed = False From a70df3c8775b9f55fd9cadc31b36a9f3c5ab509a Mon Sep 17 00:00:00 2001 From: Brecht Machiels Date: Wed, 10 Feb 2021 18:56:17 +0100 Subject: [PATCH 35/91] Docs: remove outdated notes on Windows compatibility (#382) * Docs: remove outdated notes on Windows compatibility Closes #379 * Docs: add Python Launcher notes to config section --- docs/config.rst | 11 ++++++++++- docs/usage.rst | 17 ----------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 14b17da1..99580d17 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -88,7 +88,7 @@ Configuring a session's virtualenv By default, Nox will create a new virtualenv for each session using the same interpreter that Nox uses. If you installed Nox using Python 3.6, Nox will use Python 3.6 by default for all of your sessions. -You can tell Nox to use a different Python interpreter/version by specifying the ``python`` argument (or its alias ``py``) to ``@nox.session``: +You can tell Nox to use a different Python interpreter/version by specifying the ``python`` argument (or its alias ``py``) to ``@nox.session``: .. code-block:: python @@ -96,6 +96,15 @@ You can tell Nox to use a different Python interpreter/version by specifying the def tests(session): pass +.. note:: + + The Python binaries on Windows are found via the Python `Launcher`_ for + Windows (``py``). For example, Python 3.9 can be found by determining which + executable is invoked by ``py -3.9``. If a given test needs to use the 32-bit + version of a given Python, then ``X.Y-32`` should be used as the version. + + .. _Launcher: https://docs.python.org/3/using/windows.html#python-launcher-for-windows + You can also tell Nox to run your session against multiple Python interpreters. Nox will create a separate virtualenv and run the session for each interpreter you specify. For example, this session will run twice - once for Python 2.7 and once for Python 3.6: .. code-block:: python diff --git a/docs/usage.rst b/docs/usage.rst index 098609f2..bca73f39 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -346,23 +346,6 @@ You can output a report in ``json`` format by specifying ``--report``: nox --report status.json -Windows -------- - -Nox has provisional support for running on Windows. However, depending on your Windows, Python, and virtualenv versions there may be issues. See the following threads for more info: - -* `tox issue 260 `_ -* `Python issue 24493 `_ -* `Virtualenv issue 774 `_ - -The Python binaries on Windows are found via the Python `Launcher`_ for -Windows (``py``). For example, Python 3.5 can be found by determining which -executable is invoked by ``py -3.5``. If a given test needs to use the 32-bit -version of a given Python, then ``X.Y-32`` should be used as the version. - -.. _Launcher: https://docs.python.org/3/using/windows.html#python-launcher-for-windows - - Converting from tox ------------------- From 211702da6112cf390b50b807ff1ebdaf844f73e2 Mon Sep 17 00:00:00 2001 From: Christian Riedel Date: Thu, 11 Feb 2021 22:09:01 +0100 Subject: [PATCH 36/91] Decode popen output using the system locale if UTF-8 decoding fails. (#380) * add decode_output func to decode popen output fixes #309 by trying to decode popen output with utf8 first and on error tries other encodings provided by the systems preferences. * simplified decode_output function * fix linting issues * fix double hard coded utf8 * change utf8 to utf-8 * add tests for decode_output * fix linting issues * simplify nested try...except block and fix nested exception msg Co-authored-by: Claudio Jolowicz * fix tests for simplified deode_output Co-authored-by: Claudio Jolowicz --- nox/popen.py | 20 +++++++++++++++++++- tests/test_command.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/nox/popen.py b/nox/popen.py index 48fc9726..c152f7d0 100644 --- a/nox/popen.py +++ b/nox/popen.py @@ -12,11 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. +import locale import subprocess import sys from typing import IO, Mapping, Sequence, Tuple, Union +def decode_output(output: bytes) -> str: + """Try to decode the given bytes with encodings from the system. + + :param output: output to decode + :raises UnicodeDecodeError: if all encodings fail + :return: decoded string + """ + try: + return output.decode("utf-8") + except UnicodeDecodeError: + second_encoding = locale.getpreferredencoding() + if second_encoding.casefold() in ("utf8", "utf-8"): + raise + + return output.decode(second_encoding) + + def popen( args: Sequence[str], env: Mapping[str, str] = None, @@ -45,4 +63,4 @@ def popen( return_code = proc.wait() - return return_code, out.decode("utf-8") if out else "" + return return_code, decode_output(out) if out else "" diff --git a/tests/test_command.py b/tests/test_command.py index 45919ddb..01fb06e4 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -18,6 +18,7 @@ from unittest import mock import nox.command +import nox.popen import pytest PYTHON = sys.executable @@ -294,3 +295,43 @@ def test_custom_stderr_failed_command(capsys, tmpdir): tempfile_contents = stderr.read().decode("utf-8") assert "out" not in tempfile_contents assert "err" in tempfile_contents + + +def test_output_decoding() -> None: + result = nox.popen.decode_output(b"abc") + + assert result == "abc" + + +def test_output_decoding_non_ascii() -> None: + result = nox.popen.decode_output("ü".encode("utf-8")) + + assert result == "ü" + + +def test_output_decoding_utf8_only_fail(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(nox.popen.locale, "getpreferredencoding", lambda: "utf8") + + with pytest.raises(UnicodeDecodeError) as exc: + nox.popen.decode_output(b"\x95") + + assert exc.value.encoding == "utf-8" + + +def test_output_decoding_utf8_fail_cp1252_success( + monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(nox.popen.locale, "getpreferredencoding", lambda: "cp1252") + + result = nox.popen.decode_output(b"\x95") + + assert result == "•" # U+2022 + + +def test_output_decoding_both_fail(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(nox.popen.locale, "getpreferredencoding", lambda: "ascii") + + with pytest.raises(UnicodeDecodeError) as exc: + nox.popen.decode_output(b"\x95") + + assert exc.value.encoding == "ascii" From db6378377a58dd678518b1fc1c50fb3c49dbb98a Mon Sep 17 00:00:00 2001 From: David Hagen Date: Sat, 13 Feb 2021 21:27:10 -0500 Subject: [PATCH 37/91] Add `Session.name` (#386) --- nox/sessions.py | 5 +++++ tests/test_sessions.py | 1 + 2 files changed, 6 insertions(+) diff --git a/nox/sessions.py b/nox/sessions.py index 96d80eb3..78cdcbba 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -137,6 +137,11 @@ def __dict__(self) -> "Dict[str, SessionRunner]": # type: ignore """ return {"_runner": self._runner} + @property + def name(self) -> str: + """The name of this session.""" + return self._runner.friendly_name + @property def env(self) -> dict: """A dictionary of environment variables to pass into all commands.""" diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 2d4cf119..8bb9b787 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -98,6 +98,7 @@ def test_create_tmp_twice(self): def test_properties(self): session, runner = self.make_session_and_runner() + assert session.name is runner.friendly_name assert session.env is runner.venv.env assert session.posargs is runner.global_config.posargs assert session.virtualenv is runner.venv From d4cf5475c5a757b7de6919c8ec480c3238557ba0 Mon Sep 17 00:00:00 2001 From: Stargirl Flowers Date: Sat, 20 Feb 2021 10:07:44 -0800 Subject: [PATCH 38/91] Setup GitHub actions, remove Travis (#389) --- .github/workflows/ci.yml | 69 ++++++++++++++++++++++++++++++++++++++++ .travis.yml | 38 ---------------------- 2 files changed, 69 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e94ddb92 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,69 @@ +name: CI +on: [push, pull_request] +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04] + python-version: [3.6, 3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Set up Miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + - name: Install Nox-under-test + run: | + python -m pip install --disable-pip-version-check . + - name: Run tests on ${{ matrix.os }} + run: nox --non-interactive --session "tests-${{ matrix.python-version }}" + lint: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Nox-under-test + run: | + python -m pip install --disable-pip-version-check . + - name: Lint + run: nox --non-interactive --session "lint" + docs: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Nox-under-test + run: | + python -m pip install --disable-pip-version-check . + - name: Docs + run: nox --non-interactive --session "docs" + deploy: + needs: build + runs-on: ubuntu-20.04 + if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install setuptools and wheel + run: python -m pip install --upgrade --user setuptools wheel + - name: Build sdist and wheel + run: python setup.py sdist bdist_wheel + - name: Publish distribution PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 34f9411d..00000000 --- a/.travis.yml +++ /dev/null @@ -1,38 +0,0 @@ -language: python -dist: xenial -matrix: - include: - - python: '3.6' - env: NOXSESSION="tests-3.6" - - python: '3.7' - env: NOXSESSION="tests-3.7" - - python: '3.8' - env: NOXSESSION="tests-3.8" - - python: '3.9' - env: NOXSESSION="tests-3.9" - - python: '3.8' - env: NOXSESSION="lint" - - python: '3.8' - env: NOXSESSION="docs" -before_install: - - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - - chmod +x miniconda.sh - - ./miniconda.sh -b - # Prefer the built-in python binary. - - export PATH="$PATH:/home/travis/miniconda3/bin" - - conda update --yes conda - - export SETUPTOOLS_USE_DISTUTILS=stdlib -install: - - python -m pip install --upgrade pip setuptools - - python -m pip install . -script: nox --non-interactive --session "$NOXSESSION" -deploy: - provider: pypi - user: theacodes - password: - secure: ETRTnYg+8cilT0/HidhyPljERgE/u0boKdH9TW+JrY0De40Km5C+TUmPagKJuwPx1Gw8HNN1vN7M1pqaQ/flQaY9iNbuJZr5ZaApiZW1pw5/nO2wWoANx0hiChdjvwbJZdqUFEoba6MS9aBY7TroFlLjW6dUg8MZFSiUFRQDF9rTCyzB/juC7wiLTgrjlFpOvaOmf1qpVOajY5kfn8MLELms8itRUa04X4kqqgtOfifoA1CevObrScGSXlpPtqmoxUrCmwbnHu9qnqgAWLHe3y7fI4ZqscYQv/JCW8NdJgqMTn0jctLXibHt5vC/DtUYo47mFSRBfn55ZwAFiV6IiwVbtDKby0ZdNO2uIFn4B/7l0qrLTwnZbRy4vkPqEeJoS75vl4JQrauGmI+hgdtesHjZxLzs94H4vINVt0fGpkYqbgtMQO8HUQnnj0FJXcGKo4A5OuLjnk5+rgTSvLT/5qNg/cyve5BXkn1ib6ecah21MHSQyhl5CxIFBH6S6BRrGoxXluLqXPVv/w+QA0lxXCpAPfbHuMt4r9522YN/XfGQNHfNqK/836UbLEX5ZXZiZLl01IvVPl+3eC/Qmpc+tNXb51d53Qsm89VtaNGGvuV2eLPBR+gfXcQ8wFB1HW3Q3oshHCGW4KKApyzyYKWq27JPlGV13Yh+NMHWs9PGHyI= - on: - tags: true - distributions: sdist bdist_wheel - repo: theacodes/nox - condition: "$NOXSESSION = \"tests-3.6\"" From 142092f835ab3dd2dbc55b3dba576e616c4b28f8 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sat, 20 Feb 2021 19:15:17 +0100 Subject: [PATCH 39/91] Add nox.needs_version to specify Nox version requirements (#388) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add nox.needs_version * build: Add dependency on packaging >= 20.9 The packaging library is needed to parse the version specifier in the `nox.needs_version` attribute set by user's `noxfile.py` files. We use the current version as the lower bound. The upper bound is omitted; packaging uses calendar versioning with a YY.N scheme. * Use admonition for version specification warning * Use a fixture to take care of common noxfile creation code Co-authored-by: Paulius Maruška Co-authored-by: Thea Flowers --- docs/config.rst | 27 ++++++++- nox/__init__.py | 6 +- nox/__main__.py | 8 +-- nox/_version.py | 113 ++++++++++++++++++++++++++++++++++ nox/tasks.py | 7 +++ setup.py | 1 + tests/test__version.py | 135 +++++++++++++++++++++++++++++++++++++++++ tests/test_tasks.py | 39 ++++++++++++ 8 files changed, 328 insertions(+), 8 deletions(-) create mode 100644 nox/_version.py create mode 100644 tests/test__version.py diff --git a/docs/config.rst b/docs/config.rst index 99580d17..a8da73db 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -339,7 +339,6 @@ Produces these sessions when running ``nox --list``: * tests(mysql, new) - The session object ------------------ @@ -394,3 +393,29 @@ The following options can be specified in the Noxfile: When invoking ``nox``, any options specified on the command line take precedence over the options specified in the Noxfile. If either ``--sessions`` or ``--keywords`` is specified on the command line, *both* options specified in the Noxfile will be ignored. + + +Nox version requirements +------------------------ + +Nox version requirements can be specified in your Noxfile by setting +``nox.needs_version``. If the Nox version does not satisfy the requirements, Nox +exits with a friendly error message. For example: + +.. code-block:: python + + import nox + + nox.needs_version = ">=2019.5.30" + + @nox.session(name="test") # name argument was added in 2019.5.30 + def pytest(session): + session.run("pytest") + +Any of the version specifiers defined in `PEP 440`_ can be used. + +.. warning:: Version requirements *must* be specified as a string literal, + using a simple assignment to ``nox.needs_version`` at the module level. This + allows Nox to check the version without importing the Noxfile. + +.. _PEP 440: https://www.python.org/dev/peps/pep-0440/ diff --git a/nox/__init__.py b/nox/__init__.py index 75392036..78c5283d 100644 --- a/nox/__init__.py +++ b/nox/__init__.py @@ -12,10 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Optional + from nox._options import noxfile_options as options from nox._parametrize import Param as param from nox._parametrize import parametrize_decorator as parametrize from nox.registry import session_decorator as session from nox.sessions import Session -__all__ = ["parametrize", "param", "session", "options", "Session"] +needs_version: Optional[str] = None + +__all__ = ["needs_version", "parametrize", "param", "session", "options", "Session"] diff --git a/nox/__main__.py b/nox/__main__.py index 2f551cca..d6043ac0 100644 --- a/nox/__main__.py +++ b/nox/__main__.py @@ -22,13 +22,9 @@ import sys from nox import _options, tasks, workflow +from nox._version import get_nox_version from nox.logger import setup_logging -try: - import importlib.metadata as metadata -except ImportError: # pragma: no cover - import importlib_metadata as metadata - def main() -> None: args = _options.options.parse_args() @@ -38,7 +34,7 @@ def main() -> None: return if args.version: - print(metadata.version("nox"), file=sys.stderr) + print(get_nox_version(), file=sys.stderr) return setup_logging( diff --git a/nox/_version.py b/nox/_version.py new file mode 100644 index 00000000..296f2e67 --- /dev/null +++ b/nox/_version.py @@ -0,0 +1,113 @@ +# Copyright 2021 Alethea Katherine Flowers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import contextlib +import sys +from typing import Optional + +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import InvalidVersion, Version + +try: + import importlib.metadata as metadata +except ImportError: # pragma: no cover + import importlib_metadata as metadata + + +class VersionCheckFailed(Exception): + """The Nox version does not satisfy what ``nox.needs_version`` specifies.""" + + +class InvalidVersionSpecifier(Exception): + """The ``nox.needs_version`` specifier cannot be parsed.""" + + +def get_nox_version() -> str: + """Return the version of the installed Nox package.""" + return metadata.version("nox") + + +def _parse_string_constant(node: ast.AST) -> Optional[str]: # pragma: no cover + """Return the value of a string constant.""" + if sys.version_info < (3, 8): + if isinstance(node, ast.Str) and isinstance(node.s, str): + return node.s + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return None + + +def _parse_needs_version(source: str, filename: str = "") -> Optional[str]: + """Parse ``nox.needs_version`` from the user's noxfile.""" + value: Optional[str] = None + module: ast.Module = ast.parse(source, filename=filename) + for statement in module.body: + if isinstance(statement, ast.Assign): + for target in statement.targets: + if ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "nox" + and target.attr == "needs_version" + ): + value = _parse_string_constant(statement.value) + return value + + +def _read_needs_version(filename: str) -> Optional[str]: + """Read ``nox.needs_version`` from the user's noxfile.""" + with open(filename) as io: + source = io.read() + + return _parse_needs_version(source, filename=filename) + + +def _check_nox_version_satisfies(needs_version: str) -> None: + """Check if the Nox version satisfies the given specifiers.""" + version = Version(get_nox_version()) + + try: + specifiers = SpecifierSet(needs_version) + except InvalidSpecifier as error: + message = f"Cannot parse `nox.needs_version`: {error}" + with contextlib.suppress(InvalidVersion): + Version(needs_version) + message += f", did you mean '>= {needs_version}'?" + raise InvalidVersionSpecifier(message) + + if not specifiers.contains(version, prereleases=True): + raise VersionCheckFailed( + f"The Noxfile requires Nox {specifiers}, you have {version}" + ) + + +def check_nox_version(filename: str) -> None: + """Check if ``nox.needs_version`` in the user's noxfile is satisfied. + + Args: + + filename: The location of the user's noxfile. ``nox.needs_version`` is + read from the noxfile by parsing the AST. + + Raises: + VersionCheckFailed: The Nox version does not satisfy what + ``nox.needs_version`` specifies. + InvalidVersionSpecifier: The ``nox.needs_version`` specifier cannot be + parsed. + """ + needs_version = _read_needs_version(filename) + + if needs_version is not None: + _check_nox_version_satisfies(needs_version) diff --git a/nox/tasks.py b/nox/tasks.py index 31d4c826..c8d99b0b 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -23,6 +23,7 @@ import nox from colorlog.escape_codes import parse_colors from nox import _options, registry +from nox._version import InvalidVersionSpecifier, VersionCheckFailed, check_nox_version from nox.logger import logger from nox.manifest import WARN_PYTHONS_IGNORED, Manifest from nox.sessions import Result @@ -51,6 +52,9 @@ def load_nox_module(global_config: Namespace) -> Union[types.ModuleType, int]: os.path.expandvars(global_config.noxfile) ) + # Check ``nox.needs_version`` by parsing the AST. + check_nox_version(global_config.noxfile) + # Move to the path where the Noxfile is. # This will ensure that the Noxfile's path is on sys.path, and that # import-time path resolutions work the way the Noxfile author would @@ -60,6 +64,9 @@ def load_nox_module(global_config: Namespace) -> Union[types.ModuleType, int]: "user_nox_module", global_config.noxfile ).load_module() # type: ignore + except (VersionCheckFailed, InvalidVersionSpecifier) as error: + logger.error(str(error)) + return 2 except (IOError, OSError): logger.exception("Failed to load Noxfile {}".format(global_config.noxfile)) return 2 diff --git a/setup.py b/setup.py index cd8d73a7..c8de26b1 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ install_requires=[ "argcomplete>=1.9.4,<2.0", "colorlog>=2.6.1,<5.0.0", + "packaging>=20.9", "py>=1.4.0,<2.0.0", "virtualenv>=14.0.0", "importlib_metadata; python_version < '3.8'", diff --git a/tests/test__version.py b/tests/test__version.py new file mode 100644 index 00000000..2606952c --- /dev/null +++ b/tests/test__version.py @@ -0,0 +1,135 @@ +# Copyright 2021 Alethea Katherine Flowers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from textwrap import dedent +from typing import Optional + +import pytest +from nox import needs_version +from nox._version import ( + InvalidVersionSpecifier, + VersionCheckFailed, + _parse_needs_version, + check_nox_version, + get_nox_version, +) + + +@pytest.fixture +def temp_noxfile(tmp_path: Path): + def make_temp_noxfile(content: str) -> str: + path = tmp_path / "noxfile.py" + path.write_text(content) + return str(path) + + return make_temp_noxfile + + +def test_needs_version_default() -> None: + """It is None by default.""" + assert needs_version is None + + +def test_get_nox_version() -> None: + """It returns something that looks like a Nox version.""" + result = get_nox_version() + year, month, day = [int(part) for part in result.split(".")[:3]] + assert year >= 2020 + + +@pytest.mark.parametrize( + "text,expected", + [ + ("", None), + ( + dedent( + """ + import nox + nox.needs_version = '>=2020.12.31' + """ + ), + ">=2020.12.31", + ), + ( + dedent( + """ + import nox + nox.needs_version = 'bogus' + nox.needs_version = '>=2020.12.31' + """ + ), + ">=2020.12.31", + ), + ( + dedent( + """ + import nox.sessions + nox.needs_version = '>=2020.12.31' + """ + ), + ">=2020.12.31", + ), + ( + dedent( + """ + import nox as _nox + _nox.needs_version = '>=2020.12.31' + """ + ), + None, + ), + ], +) +def test_parse_needs_version(text: str, expected: Optional[str]) -> None: + """It is parsed successfully.""" + assert expected == _parse_needs_version(text) + + +@pytest.mark.parametrize("specifiers", ["", ">=2020.12.31", ">=2020.12.31,<9999.99.99"]) +def test_check_nox_version_succeeds(temp_noxfile, specifiers: str) -> None: + """It does not raise if the version specifiers are satisfied.""" + text = dedent( + f""" + import nox + nox.needs_version = "{specifiers}" + """ + ) + check_nox_version(temp_noxfile(text)) + + +@pytest.mark.parametrize("specifiers", [">=9999.99.99"]) +def test_check_nox_version_fails(temp_noxfile, specifiers: str) -> None: + """It raises an exception if the version specifiers are not satisfied.""" + text = dedent( + f""" + import nox + nox.needs_version = "{specifiers}" + """ + ) + with pytest.raises(VersionCheckFailed): + check_nox_version(temp_noxfile(text)) + + +@pytest.mark.parametrize("specifiers", ["invalid", "2020.12.31"]) +def test_check_nox_version_invalid(temp_noxfile, specifiers: str) -> None: + """It raises an exception if the version specifiers cannot be parsed.""" + text = dedent( + f""" + import nox + nox.needs_version = "{specifiers}" + """ + ) + with pytest.raises(InvalidVersionSpecifier): + check_nox_version(temp_noxfile(text)) diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 88c5fc4d..65f3112d 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -18,6 +18,7 @@ import json import os import platform +from textwrap import dedent from unittest import mock import nox @@ -79,6 +80,44 @@ def test_load_nox_module_not_found(): assert tasks.load_nox_module(config) == 2 +@pytest.fixture +def reset_needs_version(): + """Do not leak ``nox.needs_version`` between tests.""" + try: + yield + finally: + nox.needs_version = None + + +def test_load_nox_module_needs_version_static(reset_needs_version, tmp_path): + text = dedent( + """ + import nox + nox.needs_version = ">=9999.99.99" + """ + ) + noxfile = tmp_path / "noxfile.py" + noxfile.write_text(text) + config = _options.options.namespace(noxfile=str(noxfile)) + assert tasks.load_nox_module(config) == 2 + + +def test_load_nox_module_needs_version_dynamic(reset_needs_version, tmp_path): + text = dedent( + """ + import nox + NOX_NEEDS_VERSION = ">=9999.99.99" + nox.needs_version = NOX_NEEDS_VERSION + """ + ) + noxfile = tmp_path / "noxfile.py" + noxfile.write_text(text) + config = _options.options.namespace(noxfile=str(noxfile)) + tasks.load_nox_module(config) + # Dynamic version requirements are not checked. + assert nox.needs_version == ">=9999.99.99" + + def test_discover_session_functions_decorator(): # Define sessions using the decorator. @nox.session From a078d3cc82c09a3af068d7c9d230364ba5adefbb Mon Sep 17 00:00:00 2001 From: Stargirl Flowers Date: Sat, 20 Feb 2021 10:38:10 -0800 Subject: [PATCH 40/91] Add Windows to GitHub Actions, remove AppVeyor (#390) --- .github/workflows/ci.yml | 4 +- appveyor.yml | 90 ---------------------------------------- noxfile.py | 12 +++--- 3 files changed, 8 insertions(+), 98 deletions(-) delete mode 100644 appveyor.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e94ddb92..9d9d7b42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-20.04, windows-2019] python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 @@ -22,7 +22,7 @@ jobs: run: | python -m pip install --disable-pip-version-check . - name: Run tests on ${{ matrix.os }} - run: nox --non-interactive --session "tests-${{ matrix.python-version }}" + run: nox --non-interactive --session "tests-${{ matrix.python-version }}" -- --full-trace lint: runs-on: ubuntu-20.04 steps: diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index d853541e..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,90 +0,0 @@ -version: 1.0.{build}.{branch} - -matrix: - fast_finish: true - -environment: - matrix: - - # Pre-installed Python versions, which AppVeyor may upgrade to - # a later point release. - # See: http://www.appveyor.com/docs/installed-software#python - - - PYTHON: "C:\\Python39" - # There is no miniconda for python3.9 at this time - CONDA: "C:\\Miniconda37" - NOX_SESSION: "tests-3.9" - - - PYTHON: "C:\\Python39-x64" - # There is no miniconda for python3.9 at this time - CONDA: "C:\\Miniconda37-x64" - NOX_SESSION: "tests-3.9" - - - PYTHON: "C:\\Python38" - # There is no miniconda for python3.8 at this time - CONDA: "C:\\Miniconda37" - NOX_SESSION: "tests-3.8" - - - PYTHON: "C:\\Python38-x64" - # There is no miniconda for python3.8 at this time - CONDA: "C:\\Miniconda37-x64" - NOX_SESSION: "tests-3.8" - - - PYTHON: "C:\\Python37" - CONDA: "C:\\Miniconda37" - NOX_SESSION: "tests-3.7" - - - PYTHON: "C:\\Python37-x64" - CONDA: "C:\\Miniconda37-x64" - NOX_SESSION: "tests-3.7" - - - PYTHON: "C:\\Python36" - CONDA: "C:\\Miniconda36" - NOX_SESSION: "tests-3.6" - - - PYTHON: "C:\\Python36-x64" - CONDA: "C:\\Miniconda36-x64" - NOX_SESSION: "tests-3.6" - - -install: - # Add conda command to path. - # https://www.tjelvarolsson.com/blog/how-to-continuously-test-your-python-code-on-windows-using-appveyor/ - - "SET PATH=%CONDA%;%CONDA%\\Scripts;%PATH%" - - conda config --set changeps1 no - - conda update -q --yes conda - # Get Python from conda-forge. - - conda config --add channels conda-forge - - conda info -a - - # Prepend newly installed Python to the PATH of this build (this cannot be - # done from inside the powershell script as it would require to restart - # the parent CMD process). - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - # Check that we have the expected version and architecture for Python - - "python --version" - - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" - - # Upgrade to the latest version of pip to avoid it displaying warnings - # about it being out of date. - - "python -m pip install --disable-pip-version-check --user --upgrade pip" - - # Install the build dependencies of the project. If some dependencies contain - # compiled extensions and are not provided as pre-built wheel packages, - # pip will build them from source using the MSVC compiler matching the - # target Python version and architecture - - "python -m pip install wheel" - -# init: -# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - -# on_finish: -# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - -build_script: - - "python -m pip install ." - -test_script: - # Run the project tests - - "nox.exe --session \"%NOX_SESSION%\"" diff --git a/noxfile.py b/noxfile.py index 063e36cb..5eb89ae5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -14,10 +14,11 @@ import functools import os +import platform import nox -ON_APPVEYOR = os.environ.get("APPVEYOR") == "True" +ON_WINDOWS_CI = "CI" in os.environ and platform.system() == "Windows" def is_python_version(session, version): @@ -60,12 +61,11 @@ def conda_tests(session): @nox.session def cover(session): """Coverage analysis.""" + if ON_WINDOWS_CI: + return + session.install("coverage") - if ON_APPVEYOR: - fail_under = "--fail-under=99" - else: - fail_under = "--fail-under=100" - session.run("coverage", "report", fail_under, "--show-missing") + session.run("coverage", "report", "--fail-under=100", "--show-missing") session.run("coverage", "erase") From 7ad30a0eedc6b6e477be34010153267bd5ff13f4 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Thu, 4 Mar 2021 20:04:25 +0100 Subject: [PATCH 41/91] Gracefully shutdown child processes (#393) * Allow child processes to handle keyboard interrupts * Add tests for keyboard interrupts during commands * Remove obsolete u"" literal * Fix spurious interrupts in Windows tests * Do not send keyboard interrupts in Windows CI * Run keyboard interrupt tests on Windows detached from console * Avoid misleading naming in interrupt tests * Do not assume that the correct pytest is on PATH --- nox/popen.py | 25 ++++++- tests/test_command.py | 169 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 183 insertions(+), 11 deletions(-) diff --git a/nox/popen.py b/nox/popen.py index c152f7d0..8e981e1b 100644 --- a/nox/popen.py +++ b/nox/popen.py @@ -12,10 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import locale import subprocess import sys -from typing import IO, Mapping, Sequence, Tuple, Union +from typing import IO, Mapping, Optional, Sequence, Tuple, Union + + +def shutdown_process(proc: subprocess.Popen) -> Tuple[Optional[bytes], Optional[bytes]]: + """Gracefully shutdown a child process.""" + + with contextlib.suppress(subprocess.TimeoutExpired): + return proc.communicate(timeout=0.3) + + proc.terminate() + + with contextlib.suppress(subprocess.TimeoutExpired): + return proc.communicate(timeout=0.2) + + proc.kill() + + return proc.communicate() def decode_output(output: bytes) -> str: @@ -57,9 +74,9 @@ def popen( sys.stdout.flush() except KeyboardInterrupt: - proc.terminate() - proc.wait() - raise + out, err = shutdown_process(proc) + if proc.returncode != 0: + raise return_code = proc.wait() diff --git a/tests/test_command.py b/tests/test_command.py index 01fb06e4..fec272b8 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -12,9 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ctypes import logging import os +import platform +import signal +import subprocess import sys +import time +from textwrap import dedent from unittest import mock import nox.command @@ -23,6 +29,15 @@ PYTHON = sys.executable +skip_on_windows_primary_console_session = pytest.mark.skipif( + platform.system() == "Windows" and "SECONDARY_CONSOLE_SESSION" not in os.environ, + reason="On Windows, this test must run in a separate console session.", +) + +only_on_windows = pytest.mark.skipif( + platform.system() != "Windows", reason="Only run this test on Windows." +) + def test_run_defaults(capsys): result = nox.command.run([PYTHON, "-c", "print(123)"]) @@ -98,7 +113,7 @@ def test_run_env_unicode(): result = nox.command.run( [PYTHON, "-c", 'import os; print(os.environ["SIGIL"])'], silent=True, - env={u"SIGIL": u"123"}, + env={"SIGIL": "123"}, ) assert "123" in result @@ -196,13 +211,153 @@ def test_fail_with_silent(capsys): assert "err" in err -def test_interrupt(): - mock_proc = mock.Mock() - mock_proc.communicate.side_effect = KeyboardInterrupt() +@pytest.fixture +def marker(tmp_path): + """A marker file for process communication.""" + return tmp_path / "marker" + + +def enable_ctrl_c(enabled): + """Enable keyboard interrupts (CTRL-C) on Windows.""" + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + + if not kernel32.SetConsoleCtrlHandler(None, not enabled): + raise ctypes.WinError(ctypes.get_last_error()) + + +def interrupt_process(proc): + """Send SIGINT or CTRL_C_EVENT to the process.""" + if platform.system() == "Windows": + # Disable Ctrl-C so we don't terminate ourselves. + enable_ctrl_c(False) + + # Send the keyboard interrupt to all processes attached to the current + # console session. + os.kill(0, signal.CTRL_C_EVENT) + else: + proc.send_signal(signal.SIGINT) + + +@pytest.fixture +def command_with_keyboard_interrupt(monkeypatch, marker): + """Monkeypatch Popen.communicate to raise KeyboardInterrupt.""" + if platform.system() == "Windows": + # Enable Ctrl-C because the child inherits the setting from us. + enable_ctrl_c(True) + + communicate = subprocess.Popen.communicate + + def wrapper(proc, *args, **kwargs): + # Raise the interrupt only on the first call, so Nox has a chance to + # shut down the child process subsequently. + + if wrapper.firstcall: + wrapper.firstcall = False + + # Give the child time to install its signal handlers. + while not marker.exists(): + time.sleep(0.05) + + # Send a real keyboard interrupt to the child. + interrupt_process(proc) + + # Fake a keyboard interrupt in the parent. + raise KeyboardInterrupt + + return communicate(proc, *args, **kwargs) + + wrapper.firstcall = True + + monkeypatch.setattr("subprocess.Popen.communicate", wrapper) + + +def format_program(program, marker): + """Preprocess the Python program run by the child process.""" + main = f""" + import time + from pathlib import Path + + Path({str(marker)!r}).touch() + time.sleep(3) + """ + return dedent(program).format(MAIN=dedent(main)) + + +def run_pytest_in_new_console_session(test): + """Run the given test in a separate console session.""" + env = dict(os.environ, SECONDARY_CONSOLE_SESSION="") + creationflags = ( + subprocess.CREATE_NO_WINDOW + if sys.version_info[:2] >= (3, 7) + else subprocess.CREATE_NEW_CONSOLE + ) + + subprocess.run( + [sys.executable, "-m", "pytest", f"{__file__}::{test}"], + env=env, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + creationflags=creationflags, + ) + + +@skip_on_windows_primary_console_session +@pytest.mark.parametrize( + "program", + [ + """ + {MAIN} + """, + """ + import signal + + signal.signal(signal.SIGINT, signal.SIG_IGN) + + {MAIN} + """, + """ + import signal + + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + + {MAIN} + """, + ], +) +def test_interrupt_raises(command_with_keyboard_interrupt, program, marker): + """It kills the process and reraises the keyboard interrupt.""" + with pytest.raises(KeyboardInterrupt): + nox.command.run([PYTHON, "-c", format_program(program, marker)]) + + +@only_on_windows +def test_interrupt_raises_on_windows(): + """It kills the process and reraises the keyboard interrupt.""" + run_pytest_in_new_console_session("test_interrupt_raises") + + +@skip_on_windows_primary_console_session +def test_interrupt_handled(command_with_keyboard_interrupt, marker): + """It does not raise if the child handles the keyboard interrupt.""" + program = """ + import signal + + def exithandler(sig, frame): + raise SystemExit() + + signal.signal(signal.SIGINT, exithandler) + + {MAIN} + """ + nox.command.run([PYTHON, "-c", format_program(program, marker)]) + - with mock.patch("subprocess.Popen", return_value=mock_proc): - with pytest.raises(KeyboardInterrupt): - nox.command.run([PYTHON, "-c" "123"]) +@only_on_windows +def test_interrupt_handled_on_windows(): + """It does not raise if the child handles the keyboard interrupt.""" + run_pytest_in_new_console_session("test_interrupt_handled") def test_custom_stdout(capsys, tmpdir): From 082c26636393fc293b1551ebeb4f0209d4973f17 Mon Sep 17 00:00:00 2001 From: Peilonrayz Date: Mon, 8 Mar 2021 17:34:04 +0000 Subject: [PATCH 42/91] Improve automated tests support on Windows (#300) * Allow sphinx-autobuild to run on Windows * Improve coverage when selecting bin folder * Test CondaEnv interpreter on Windows * Test VirtualEnv interpreter on Windows * Merge coverage results. * Undo changes to `test_create_interpreter`. * Remove an unused shutil import --- noxfile.py | 11 +++++++++-- tests/test_virtualenv.py | 11 ++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/noxfile.py b/noxfile.py index 5eb89ae5..24e41a7b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -40,7 +40,13 @@ def tests(session): session.run("pytest", *tests) return session.run( - "pytest", "--cov=nox", "--cov-config", ".coveragerc", "--cov-report=", *tests + "pytest", + "--cov=nox", + "--cov-config", + ".coveragerc", + "--cov-report=", + *tests, + env={"COVERAGE_FILE": ".coverage.{}".format(session.python)} ) session.notify("cover") @@ -65,6 +71,7 @@ def cover(session): return session.install("coverage") + session.run("coverage", "combine") session.run("coverage", "report", "--fail-under=100", "--show-missing") session.run("coverage", "erase") @@ -94,7 +101,7 @@ def lint(session): session.run("flake8", "nox", *files) -@nox.session(python="3.8") +@nox.session(python="3.7") def docs(session): """Build the documentation.""" output_dir = os.path.join(session.create_tmp(), "output") diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 7200d2d1..75598d60 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -181,13 +181,18 @@ def test_condaenv_create(make_conda): assert dir_.join("test.txt").check() -@pytest.mark.skipif(IS_WINDOWS, reason="Not testing multiple interpreters on Windows.") @pytest.mark.skipif(not HAS_CONDA, reason="Missing conda command.") def test_condaenv_create_interpreter(make_conda): venv, dir_ = make_conda(interpreter="3.7") venv.create() - assert dir_.join("bin", "python").check() - assert dir_.join("bin", "python3.7").check() + if IS_WINDOWS: + assert dir_.join("python.exe").check() + assert dir_.join("python37.dll").check() + assert dir_.join("python37.pdb").check() + assert not dir_.join("python37.exe").check() + else: + assert dir_.join("bin", "python").check() + assert dir_.join("bin", "python3.7").check() @mock.patch("nox.virtualenv._SYSTEM", new="Windows") From 1228981158010130606d9fbf0edd2384372780b2 Mon Sep 17 00:00:00 2001 From: "Paulo S. Costa" Date: Thu, 11 Mar 2021 00:24:41 -0800 Subject: [PATCH 43/91] Remove Travis references (#403) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f911bd1e..a8a4069b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,7 +45,7 @@ To run against a particular Python version: nox --session tests-3.9 -When you send a pull request Travis will handle running everything, but it is +When you send a pull request the CI will handle running everything, but it is recommended to test as much as possible locally before pushing. ## Getting a sticker From 55258326dd43d8d98b715d8bfb06b07e5b440040 Mon Sep 17 00:00:00 2001 From: "Paulo S. Costa" Date: Thu, 11 Mar 2021 19:01:04 -0800 Subject: [PATCH 44/91] Ignore mypy configuration file (#402) --- noxfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/noxfile.py b/noxfile.py index 24e41a7b..802f6dcb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -90,6 +90,7 @@ def lint(session): session.install("flake8==3.7.8", "black==19.3b0", "isort==4.3.21", "mypy==0.720") session.run( "mypy", + "--config-file=", "--disallow-untyped-defs", "--warn-unused-ignores", "--ignore-missing-imports", From de3d9c5bfee5cb13ad826b79c2834bb7a49284c4 Mon Sep 17 00:00:00 2001 From: "Paulo S. Costa" Date: Thu, 11 Mar 2021 19:02:03 -0800 Subject: [PATCH 45/91] Preserve the order of parameterized arguments (#401) * Swap param order in multiple args test * Remove arg sorting in Param repr Co-authored-by: Claudio Jolowicz --- nox/_parametrize.py | 3 +-- tests/test__parametrize.py | 14 +++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nox/_parametrize.py b/nox/_parametrize.py index 9f2cd631..be226e46 100644 --- a/nox/_parametrize.py +++ b/nox/_parametrize.py @@ -51,8 +51,7 @@ def __str__(self) -> str: return self.id else: call_spec = self.call_spec - keys = sorted(call_spec.keys(), key=str) - args = ["{}={}".format(k, repr(call_spec[k])) for k in keys] + args = ["{}={}".format(k, repr(call_spec[k])) for k in call_spec.keys()] return ", ".join(args) __repr__ = __str__ diff --git a/tests/test__parametrize.py b/tests/test__parametrize.py index 2a7f69b3..d130174a 100644 --- a/tests/test__parametrize.py +++ b/tests/test__parametrize.py @@ -200,19 +200,19 @@ def test_generate_calls_multiple_args(): f = mock.Mock() f.__name__ = "f" - arg_names = ("abc", "foo") + arg_names = ("foo", "abc") call_specs = [ - _parametrize.Param(1, "a", arg_names=arg_names), - _parametrize.Param(2, "b", arg_names=arg_names), - _parametrize.Param(3, "c", arg_names=arg_names), + _parametrize.Param("a", 1, arg_names=arg_names), + _parametrize.Param("b", 2, arg_names=arg_names), + _parametrize.Param("c", 3, arg_names=arg_names), ] calls = _decorators.Call.generate_calls(f, call_specs) assert len(calls) == 3 - assert calls[0].session_signature == "(abc=1, foo='a')" - assert calls[1].session_signature == "(abc=2, foo='b')" - assert calls[2].session_signature == "(abc=3, foo='c')" + assert calls[0].session_signature == "(foo='a', abc=1)" + assert calls[1].session_signature == "(foo='b', abc=2)" + assert calls[2].session_signature == "(foo='c', abc=3)" calls[0]() f.assert_called_with(abc=1, foo="a") From 324044192052e7c87846b4680f30becd8bad34c0 Mon Sep 17 00:00:00 2001 From: Ryan Morshead Date: Thu, 11 Mar 2021 19:04:12 -0800 Subject: [PATCH 46/91] set local session posargs via notify (#397) * add local session posargs via notify * add docs for notify(posargs=...) Also SessionRunner.notify did not need to be a property. * fix tests after notify(posargs=...) addition * add test for notified posargs * Add posargs to docstring for Manifest.notify * Fix typo in docstring --- nox/manifest.py | 23 +++++++++++++++++++++-- nox/sessions.py | 22 ++++++++++++++++------ tests/test_manifest.py | 22 ++++++++++++++++++---- tests/test_sessions.py | 6 +++++- 4 files changed, 60 insertions(+), 13 deletions(-) diff --git a/nox/manifest.py b/nox/manifest.py index 616de130..1334fa9d 100644 --- a/nox/manifest.py +++ b/nox/manifest.py @@ -16,7 +16,18 @@ import collections.abc import itertools from collections import OrderedDict -from typing import Any, Iterable, Iterator, List, Mapping, Sequence, Set, Tuple, Union +from typing import ( + Any, + Iterable, + Iterator, + List, + Mapping, + Optional, + Sequence, + Set, + Tuple, + Union, +) from nox._decorators import Call, Func from nox.sessions import Session, SessionRunner @@ -257,7 +268,9 @@ def make_session( def next(self) -> SessionRunner: return self.__next__() - def notify(self, session: Union[str, SessionRunner]) -> bool: + def notify( + self, session: Union[str, SessionRunner], posargs: Optional[List[str]] = None + ) -> bool: """Enqueue the specified session in the queue. If the session is already in the queue, or has been run already, @@ -266,6 +279,10 @@ def notify(self, session: Union[str, SessionRunner]) -> bool: Args: session (Union[str, ~nox.session.Session]): The session to be enqueued. + posargs (Optional[List[str]]): If given, sets the positional + arguments *only* for the queued session. Otherwise, the + standard globally available positional arguments will be + used instead. Returns: bool: Whether the session was added to the queue. @@ -282,6 +299,8 @@ def notify(self, session: Union[str, SessionRunner]) -> bool: # the end of the queue. for s in self._all_sessions: if s == session or s.name == session or session in s.signatures: + if posargs is not None: + s.posargs = posargs self._queue.append(s) return True diff --git a/nox/sessions.py b/nox/sessions.py index 78cdcbba..52f647a1 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -149,9 +149,8 @@ def env(self) -> dict: @property def posargs(self) -> List[str]: - """This is set to any extra arguments - passed to ``nox`` on the commandline.""" - return self._runner.global_config.posargs + """Any extra arguments from the ``nox`` commandline or :class:`Session.notify`.""" + return self._runner.posargs @property def virtualenv(self) -> ProcessEnv: @@ -432,7 +431,11 @@ def install(self, *args: str, **kwargs: Any) -> None: self._run("python", "-m", "pip", "install", *args, external="error", **kwargs) - def notify(self, target: "Union[str, SessionRunner]") -> None: + def notify( + self, + target: "Union[str, SessionRunner]", + posargs: Optional[Iterable[str]] = None, + ) -> None: """Place the given session at the end of the queue. This method is idempotent; multiple notifications to the same session @@ -442,8 +445,14 @@ def notify(self, target: "Union[str, SessionRunner]") -> None: target (Union[str, Callable]): The session to be notified. This may be specified as the appropriate string (same as used for ``nox -s``) or using the function object. + posargs (Optional[Iterable[str]]): If given, sets the positional + arguments *only* for the queued session. Otherwise, the + standard globally available positional arguments will be + used instead. """ - self._runner.manifest.notify(target) + if posargs is not None: + posargs = list(posargs) + self._runner.manifest.notify(target, posargs) def log(self, *args: Any, **kwargs: Any) -> None: """Outputs a log during the session.""" @@ -472,7 +481,8 @@ def __init__( self.func = func self.global_config = global_config self.manifest = manifest - self.venv = None # type: Optional[ProcessEnv] + self.venv: Optional[ProcessEnv] = None + self.posargs: List[str] = global_config.posargs @property def description(self) -> Optional[str]: diff --git a/tests/test_manifest.py b/tests/test_manifest.py index a3631511..35eb775d 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -34,10 +34,11 @@ def create_mock_sessions(): def create_mock_config(): - cfg = mock.sentinel.CONFIG + cfg = mock.sentinel.MOCKED_CONFIG cfg.force_venv_backend = None cfg.default_venv_backend = None cfg.extra_pythons = None + cfg.posargs = [] return cfg @@ -223,9 +224,7 @@ def session_func(): ], ) def test_extra_pythons(python, extra_pythons, expected): - cfg = mock.sentinel.CONFIG - cfg.force_venv_backend = None - cfg.default_venv_backend = None + cfg = create_mock_config() cfg.extra_pythons = extra_pythons manifest = Manifest({}, cfg) @@ -345,6 +344,21 @@ def my_session(session): assert len(manifest) == 1 +def test_notify_with_posargs(): + cfg = create_mock_config() + manifest = Manifest({}, cfg) + + session = manifest.make_session("my_session", Func(lambda session: None))[0] + manifest.add_session(session) + + # delete my_session from the queue + manifest.filter_by_name(()) + + assert session.posargs is cfg.posargs + assert manifest.notify("my_session", posargs=["--an-arg"]) + assert session.posargs == ["--an-arg"] + + def test_notify_error(): manifest = Manifest({}, create_mock_config()) with pytest.raises(ValueError): diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 8bb9b787..150c15ec 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -505,7 +505,11 @@ def test_notify(self): session.notify("other") - runner.manifest.notify.assert_called_once_with("other") + runner.manifest.notify.assert_called_once_with("other", None) + + session.notify("other", posargs=["--an-arg"]) + + runner.manifest.notify.assert_called_with("other", ["--an-arg"]) def test_log(self, caplog): caplog.set_level(logging.INFO) From a67c9a291a87f6f03de383b29bb38990b832f15b Mon Sep 17 00:00:00 2001 From: Diego Palma Date: Fri, 23 Apr 2021 22:30:44 -0400 Subject: [PATCH 47/91] Check whether interpreter changed and type (#123) (#418) Co-authored-by: Diego --- nox/virtualenv.py | 30 +++++++++++++++++++++++++----- tests/test_virtualenv.py | 21 ++++++++++++++++++++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 0e3bbca3..67aee8ac 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -313,12 +313,19 @@ def _clean_location(self) -> bool: """Deletes any existing virtual environment""" if os.path.exists(self.location): if self.reuse_existing: + self._check_reused_environment() return False else: shutil.rmtree(self.location) return True + def _check_reused_environment(self) -> None: + """Check if reused environment type is the same.""" + with open(os.path.join(self.location, "pyvenv.cfg")) as fp: + old_env = "virtualenv" if "virtualenv" in fp.read() else "venv" + assert old_env == self.venv_or_virtualenv + @property def _resolved_interpreter(self) -> str: """Return the interpreter, appropriately resolved for the platform. @@ -394,12 +401,25 @@ def bin_paths(self) -> List[str]: def create(self) -> bool: """Create the virtualenv or venv.""" if not self._clean_location(): - logger.debug( - "Re-using existing virtual environment at {}.".format( - self.location_name - ) + original = nox.command.run( + [self._resolved_interpreter, "-c", "import sys; print(sys.prefix)"], + silent=True, ) - return False + created = nox.command.run( + [ + os.path.join(self.location, "bin", "python"), + "-c", + "import sys; print(sys.real_prefix)", + ], + silent=True, + ) + if original == created: + logger.debug( + "Re-using existing virtual environment at {}.".format( + self.location_name + ) + ) + return False if self.venv_or_virtualenv == "virtualenv": cmd = [sys.executable, "-m", "virtualenv", self.location] diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 75598d60..fb624cbd 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -244,6 +244,9 @@ def test__clean_location(monkeypatch, make_one): # Don't re-use existing, but doesn't currently exist. # Should return True indicating that the venv needs to be created. + monkeypatch.setattr( + nox.virtualenv.VirtualEnv, "_check_reused_environment", mock.MagicMock() + ) monkeypatch.delattr(nox.virtualenv.shutil, "rmtree") assert not dir_.check() assert venv._clean_location() @@ -289,7 +292,7 @@ def test_bin_windows(make_one): assert dir_.join("Scripts").strpath == venv.bin -def test_create(make_one): +def test_create(monkeypatch, make_one): venv, dir_ = make_one() venv.create() @@ -312,10 +315,26 @@ def test_create(make_one): dir_.ensure("test.txt") assert dir_.join("test.txt").check() venv.reuse_existing = True + monkeypatch.setattr(nox.virtualenv.nox.command, "run", mock.MagicMock()) venv.create() assert dir_.join("test.txt").check() +def test_create_check_interpreter(make_one, monkeypatch, tmpdir): + cmd_mock = mock.MagicMock( + side_effect=["python-1", "python-1", "python-2", "python-3", "python-4"] + ) + monkeypatch.setattr(nox.virtualenv.nox.command, "run", cmd_mock) + test_dir = tmpdir.mkdir("pytest") + fp = test_dir.join("pyvenv.cfg") + fp.write("virtualenv") + venv, dir_ = make_one() + venv.reuse_existing = True + venv.location = test_dir.strpath + assert not venv.create() + assert venv.create() + + def test_create_venv_backend(make_one): venv, dir_ = make_one(venv=True) venv.create() From 7f3b8da5f19b2d1b179c68330f81b9bbec3827f3 Mon Sep 17 00:00:00 2001 From: Will Holtz Date: Fri, 23 Apr 2021 19:31:18 -0700 Subject: [PATCH 48/91] fix position of venv_params within conda create (#420) Fixes #419 by moving 'pip' to after venv_params --- nox/virtualenv.py | 13 ++++--------- tests/test_virtualenv.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 67aee8ac..78322e08 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -226,18 +226,13 @@ def create(self) -> bool: ) return False - cmd = [ - "conda", - "create", - "--yes", - "--prefix", - self.location, - # Ensure the pip package is installed. - "pip", - ] + cmd = ["conda", "create", "--yes", "--prefix", self.location] cmd.extend(self.venv_params) + # Ensure the pip package is installed. + cmd.append("pip") + if self.interpreter: python_dep = "python={}".format(self.interpreter) else: diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index fb624cbd..cb76a877 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -181,6 +181,18 @@ def test_condaenv_create(make_conda): assert dir_.join("test.txt").check() +@pytest.mark.skipif(not HAS_CONDA, reason="Missing conda command.") +def test_condaenv_create_with_params(make_conda): + venv, dir_ = make_conda(venv_params=["--verbose"]) + venv.create() + if IS_WINDOWS: + assert dir_.join("python.exe").check() + assert dir_.join("Scripts", "pip.exe").check() + else: + assert dir_.join("bin", "python").check() + assert dir_.join("bin", "pip").check() + + @pytest.mark.skipif(not HAS_CONDA, reason="Missing conda command.") def test_condaenv_create_interpreter(make_conda): venv, dir_ = make_conda(interpreter="3.7") From a9bd87298c47bf723fc327d7ac9bba9402013ea3 Mon Sep 17 00:00:00 2001 From: Diego Palma Date: Fri, 23 Apr 2021 22:32:49 -0400 Subject: [PATCH 49/91] Add normalized comparison to check sessions (#396) (#417) Co-authored-by: Diego --- nox/manifest.py | 42 ++++++++++++++++++++++++++++++++++-------- tests/test_manifest.py | 18 ++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/nox/manifest.py b/nox/manifest.py index 1334fa9d..8265de2d 100644 --- a/nox/manifest.py +++ b/nox/manifest.py @@ -13,6 +13,7 @@ # limitations under the License. import argparse +import ast import collections.abc import itertools from collections import OrderedDict @@ -133,21 +134,26 @@ def filter_by_name(self, specified_sessions: Iterable[str]) -> None: queue = [] for session_name in specified_sessions: for session in self._queue: - if session_name == session.name or session_name in set( - session.signatures - ): + if _normalized_session_match(session_name, session): queue.append(session) - self._queue = queue # If a session was requested and was not found, complain loudly. all_sessions = set( - itertools.chain( - [x.name for x in self._all_sessions if x.name], - *[x.signatures for x in self._all_sessions], + map( + _normalize_arg, + ( + itertools.chain( + [x.name for x in self._all_sessions if x.name], + *[x.signatures for x in self._all_sessions], + ) + ), ) ) - missing_sessions = set(specified_sessions) - all_sessions + normalized_specified_sessions = [ + _normalize_arg(session_name) for session_name in specified_sessions + ] + missing_sessions = set(normalized_specified_sessions) - all_sessions if missing_sessions: raise KeyError("Sessions not found: {}".format(", ".join(missing_sessions))) @@ -344,4 +350,24 @@ def _null_session_func_(session: Session) -> None: session.skip("This session had no parameters available.") +def _normalized_session_match(session_name: str, session: SessionRunner) -> bool: + """Checks if session_name matches session.""" + if session_name == session.name or session_name in session.signatures: + return True + for name in session.signatures: + equal_rep = _normalize_arg(session_name) == _normalize_arg(name) + if equal_rep: + return True + # Exhausted + return False + + +def _normalize_arg(arg: str) -> Union[str]: + """Normalize arg for comparison.""" + try: + return str(ast.dump(ast.parse(arg))) + except (TypeError, SyntaxError): + return arg + + _null_session_func = Func(_null_session_func_, python=False) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 35eb775d..9fc0578e 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -22,6 +22,8 @@ WARN_PYTHONS_IGNORED, KeywordLocals, Manifest, + _normalize_arg, + _normalized_session_match, _null_session_func, ) @@ -42,6 +44,22 @@ def create_mock_config(): return cfg +def test__normalize_arg(): + assert _normalize_arg('test(foo="bar")') == _normalize_arg('test(foo="bar")') + + # In the case of SyntaxError it should fallback to strng + assert ( + _normalize_arg("datetime.datetime(1990; 2, 18),") + == "datetime.datetime(1990; 2, 18)," + ) + + +def test__normalized_session_match(): + session_mock = mock.MagicMock() + session_mock.signatures = ['test(foo="bar")'] + assert _normalized_session_match("test(foo='bar')", session_mock) + + def test_init(): sessions = create_mock_sessions() manifest = Manifest(sessions, create_mock_config()) From 9cd5b7801e0c198855aa02eb6ce6f0ae55d4dc9d Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sat, 24 Apr 2021 04:33:57 +0200 Subject: [PATCH 50/91] Allow nox.parametrize to select the session Python (#413) * Enable nox.parametrize to determine the session Python * Add tests * Add documentation --- docs/config.rst | 38 +++++++++++++++++++++ nox/_decorators.py | 24 +++++++++++--- tests/test__parametrize.py | 68 +++++++++++++++++++++++++++++++++++++- 3 files changed, 125 insertions(+), 5 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index a8da73db..d64d2f88 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -339,6 +339,44 @@ Produces these sessions when running ``nox --list``: * tests(mysql, new) +Parametrizing the session Python +-------------------------------- + +You can use parametrization to select the Python interpreter for a session. +These two examples are equivalent: + +.. code-block:: python + + @nox.session + @nox.parametrize("python", ["3.6", "3.7", "3.8"]) + def tests(session): + ... + + @nox.session(python=["3.6", "3.7", "3.8"]) + def tests(session): + ... + +The first form can be useful if you need to exclude some combinations of Python +versions with other parameters. For example, you may want to test against +multiple versions of a dependency, but the latest version doesn't run on older +Pythons: + +.. code-block:: python + + @nox.session + @nox.parametrize( + "python,dependency", + [ + (python, dependency) + for python in ("3.6", "3.7", "3.8") + for dependency in ("1.0", "2.0") + if (python, dependency) != ("3.6", "2.0") + ], + ) + def tests(session, dependency): + ... + + The session object ------------------ diff --git a/nox/_decorators.py b/nox/_decorators.py index 03878deb..b04e48ab 100644 --- a/nox/_decorators.py +++ b/nox/_decorators.py @@ -1,5 +1,6 @@ import copy import functools +import inspect import types from typing import Any, Callable, Dict, Iterable, List, Optional, cast @@ -66,20 +67,35 @@ def copy(self, name: str = None) -> "Func": class Call(Func): def __init__(self, func: Func, param_spec: "Param") -> None: + call_spec = param_spec.call_spec + session_signature = "({})".format(param_spec) + + # Determine the Python interpreter for the session using either @session + # or @parametrize. For backwards compatibility, we only use a "python" + # parameter in @parametrize if the session function does not expect it + # as a normal argument, and if the @session decorator does not already + # specify `python`. + + python = func.python + if python is None and "python" in call_spec: + signature = inspect.signature(func.func) + if "python" not in signature.parameters: + python = call_spec.pop("python") + super().__init__( func, - func.python, + python, func.reuse_venv, None, func.venv_backend, func.venv_params, func.should_warn, ) - self.param_spec = param_spec - self.session_signature = "({})".format(param_spec) + self.call_spec = call_spec + self.session_signature = session_signature def __call__(self, *args: Any, **kwargs: Any) -> Any: - kwargs.update(self.param_spec.call_spec) + kwargs.update(self.call_spec) return super().__call__(*args, **kwargs) @classmethod diff --git a/tests/test__parametrize.py b/tests/test__parametrize.py index d130174a..79fc2e5d 100644 --- a/tests/test__parametrize.py +++ b/tests/test__parametrize.py @@ -15,7 +15,7 @@ from unittest import mock import pytest -from nox import _decorators, _parametrize +from nox import _decorators, _parametrize, parametrize, session @pytest.mark.parametrize( @@ -242,3 +242,69 @@ def test_generate_calls_ids(): f.assert_called_with(foo=1) calls[1]() f.assert_called_with(foo=2) + + +def test_generate_calls_session_python(): + called_with = [] + + @session + @parametrize("python,dependency", [("3.8", "0.9"), ("3.9", "0.9"), ("3.9", "1.0")]) + def f(session, dependency): + called_with.append((session, dependency)) + + calls = _decorators.Call.generate_calls(f, f.parametrize) + + assert len(calls) == 3 + + assert calls[0].python == "3.8" + assert calls[1].python == "3.9" + assert calls[2].python == "3.9" + + assert calls[0].session_signature == "(python='3.8', dependency='0.9')" + assert calls[1].session_signature == "(python='3.9', dependency='0.9')" + assert calls[2].session_signature == "(python='3.9', dependency='1.0')" + + session_ = () + + calls[0](session_) + calls[1](session_) + calls[2](session_) + + assert len(called_with) == 3 + + assert called_with[0] == (session_, "0.9") + assert called_with[1] == (session_, "0.9") + assert called_with[2] == (session_, "1.0") + + +def test_generate_calls_python_compatibility(): + called_with = [] + + @session + @parametrize("python,dependency", [("3.8", "0.9"), ("3.9", "0.9"), ("3.9", "1.0")]) + def f(session, python, dependency): + called_with.append((session, python, dependency)) + + calls = _decorators.Call.generate_calls(f, f.parametrize) + + assert len(calls) == 3 + + assert calls[0].python is None + assert calls[1].python is None + assert calls[2].python is None + + assert calls[0].session_signature == "(python='3.8', dependency='0.9')" + assert calls[1].session_signature == "(python='3.9', dependency='0.9')" + assert calls[2].session_signature == "(python='3.9', dependency='1.0')" + + session_ = () + + calls[0](session_) + calls[1](session_) + calls[2](session_) + + assert len(called_with) == 3 + + assert called_with[0] == (session_, "3.8", "0.9") + assert called_with[1] == (session_, "3.9", "0.9") + assert called_with[2] == (session_, "3.9", "1.0") From 99c5910786bba13a7f74f7285b81f82cd5ab99f6 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sun, 23 May 2021 19:51:03 +0200 Subject: [PATCH 51/91] Fix regression in check for stale environments (#425) * Add test case for reusing environments * Fix crash due to unconditional use of `sys.real_prefix` - `sys.real_prefix` was set by virtualenv < 20.0. - `sys.base_prefix` is used by virtualenv >= 20.0 and python -m venv (PEP 405) * Fix disabled reuse due to comparing sys.prefix with sys.base_prefix When comparing the resolved interpreter to the interpreter in an existing virtualenv, we need to use sys.base_prefix (or sys.real_prefix) for both. The resolved interpreter is located in the virtualenv, so its sys.prefix would point into the virtualenv instead of the base installation. This bug results in virtualenvs never being reused. * Add test for reusing stale venv-style environment * Add test for reusing stale virtualenv-style environment * Fix crash when reusing environment of unexpected type * Use realistic pyvenv.cfg syntax in test for interpreter check * Add test for venv-style pyvenv.cfg with spurious "virtualenv" string * Fix false positive for venv-style environment * Fix crash on Windows due to hardcoded POSIX-style interpreter path * Add test for virtualenv without pyvenv.cfg file * Fix crash when virtualenv does not contain a pyvenv.cfg file Environments created by virtualenv < 20.0.0 do not have pyvenv.cfg files. * Rename Virtualenv._check_reused_environment{ => _type} * Extract function Virtualenv._check_reused_environment_interpreter * Refactor test reusing environments with different interpreter * Extend test to check that the stale environment is removed * Fix missing directory removal when re-creating environment * Reformat with Black --- nox/virtualenv.py | 57 ++++++++++-------- tests/test_virtualenv.py | 122 ++++++++++++++++++++++++++++++++++----- 2 files changed, 140 insertions(+), 39 deletions(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 78322e08..cd329457 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -293,7 +293,7 @@ def __init__( reuse_existing: bool = False, *, venv: bool = False, - venv_params: Any = None + venv_params: Any = None, ): self.location_name = location self.location = os.path.abspath(location) @@ -307,19 +307,39 @@ def __init__( def _clean_location(self) -> bool: """Deletes any existing virtual environment""" if os.path.exists(self.location): - if self.reuse_existing: - self._check_reused_environment() + if ( + self.reuse_existing + and self._check_reused_environment_type() + and self._check_reused_environment_interpreter() + ): return False else: shutil.rmtree(self.location) return True - def _check_reused_environment(self) -> None: + def _check_reused_environment_type(self) -> bool: """Check if reused environment type is the same.""" - with open(os.path.join(self.location, "pyvenv.cfg")) as fp: - old_env = "virtualenv" if "virtualenv" in fp.read() else "venv" - assert old_env == self.venv_or_virtualenv + path = os.path.join(self.location, "pyvenv.cfg") + if not os.path.isfile(path): + # virtualenv < 20.0 does not create pyvenv.cfg + old_env = "virtualenv" + else: + pattern = re.compile(f"virtualenv[ \t]*=") + with open(path) as fp: + old_env = ( + "virtualenv" if any(pattern.match(line) for line in fp) else "venv" + ) + return old_env == self.venv_or_virtualenv + + def _check_reused_environment_interpreter(self) -> bool: + """Check if reused environment interpreter is the same.""" + program = "import sys; print(getattr(sys, 'real_prefix', sys.base_prefix))" + original = nox.command.run( + [self._resolved_interpreter, "-c", program], silent=True + ) + created = nox.command.run(["python", "-c", program], silent=True) + return original == created @property def _resolved_interpreter(self) -> str: @@ -396,25 +416,12 @@ def bin_paths(self) -> List[str]: def create(self) -> bool: """Create the virtualenv or venv.""" if not self._clean_location(): - original = nox.command.run( - [self._resolved_interpreter, "-c", "import sys; print(sys.prefix)"], - silent=True, - ) - created = nox.command.run( - [ - os.path.join(self.location, "bin", "python"), - "-c", - "import sys; print(sys.real_prefix)", - ], - silent=True, - ) - if original == created: - logger.debug( - "Re-using existing virtual environment at {}.".format( - self.location_name - ) + logger.debug( + "Re-using existing virtual environment at {}.".format( + self.location_name ) - return False + ) + return False if self.venv_or_virtualenv == "virtualenv": cmd = [sys.executable, "-m", "virtualenv", self.location] diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index cb76a877..59dd1804 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -15,6 +15,7 @@ import os import shutil import sys +from textwrap import dedent from unittest import mock import nox.virtualenv @@ -257,7 +258,12 @@ def test__clean_location(monkeypatch, make_one): # Don't re-use existing, but doesn't currently exist. # Should return True indicating that the venv needs to be created. monkeypatch.setattr( - nox.virtualenv.VirtualEnv, "_check_reused_environment", mock.MagicMock() + nox.virtualenv.VirtualEnv, "_check_reused_environment_type", mock.MagicMock() + ) + monkeypatch.setattr( + nox.virtualenv.VirtualEnv, + "_check_reused_environment_interpreter", + mock.MagicMock(), ) monkeypatch.delattr(nox.virtualenv.shutil, "rmtree") assert not dir_.check() @@ -332,19 +338,107 @@ def test_create(monkeypatch, make_one): assert dir_.join("test.txt").check() -def test_create_check_interpreter(make_one, monkeypatch, tmpdir): - cmd_mock = mock.MagicMock( - side_effect=["python-1", "python-1", "python-2", "python-3", "python-4"] - ) - monkeypatch.setattr(nox.virtualenv.nox.command, "run", cmd_mock) - test_dir = tmpdir.mkdir("pytest") - fp = test_dir.join("pyvenv.cfg") - fp.write("virtualenv") - venv, dir_ = make_one() - venv.reuse_existing = True - venv.location = test_dir.strpath - assert not venv.create() - assert venv.create() +def test_create_reuse_environment(make_one): + venv, location = make_one(reuse_existing=True) + venv.create() + + reused = not venv.create() + + assert reused + + +def test_create_reuse_environment_with_different_interpreter(make_one, monkeypatch): + venv, location = make_one(reuse_existing=True) + venv.create() + + # Pretend that the environment was created with a different interpreter. + monkeypatch.setattr(venv, "_check_reused_environment_interpreter", lambda: False) + + # Create a marker file. It should be gone after the environment is re-created. + location.join("marker").ensure() + + reused = not venv.create() + + assert not reused + assert not location.join("marker").check() + + +def test_create_reuse_stale_venv_environment(make_one): + venv, location = make_one(reuse_existing=True) + venv.create() + + # Drop a venv-style pyvenv.cfg into the environment. + pyvenv_cfg = """\ + home = /usr/bin + include-system-site-packages = false + version = 3.9.6 + """ + location.join("pyvenv.cfg").write(dedent(pyvenv_cfg)) + + reused = not venv.create() + + # The environment is not reused because it does not look like a + # virtualenv-style environment. + assert not reused + + +def test_create_reuse_stale_virtualenv_environment(make_one): + venv, location = make_one(reuse_existing=True, venv=True) + venv.create() + + # Drop a virtualenv-style pyvenv.cfg into the environment. + pyvenv_cfg = """\ + home = /usr + implementation = CPython + version_info = 3.9.6.final.0 + virtualenv = 20.4.6 + include-system-site-packages = false + base-prefix = /usr + base-exec-prefix = /usr + base-executable = /usr/bin/python3.9 + """ + location.join("pyvenv.cfg").write(dedent(pyvenv_cfg)) + + reused = not venv.create() + + # The environment is not reused because it does not look like a + # venv-style environment. + assert not reused + + +def test_create_reuse_venv_environment(make_one): + venv, location = make_one(reuse_existing=True, venv=True) + venv.create() + + # Use a spurious occurrence of "virtualenv" in the pyvenv.cfg. + pyvenv_cfg = """\ + home = /opt/virtualenv/bin + include-system-site-packages = false + version = 3.9.6 + """ + location.join("pyvenv.cfg").write(dedent(pyvenv_cfg)) + + reused = not venv.create() + + # The environment should be detected as venv-style and reused. + assert reused + + +def test_create_reuse_oldstyle_virtualenv_environment(make_one): + venv, location = make_one(reuse_existing=True) + venv.create() + + pyvenv_cfg = location.join("pyvenv.cfg") + if not pyvenv_cfg.check(): + pytest.skip("Requires virtualenv >= 20.0.0.") + + # virtualenv < 20.0.0 does not create a pyvenv.cfg file. + pyvenv_cfg.remove() + + reused = not venv.create() + + # The environment is detected as virtualenv-style and reused. + assert reused def test_create_venv_backend(make_one): From a6539a73ea171b7dc5b842b029c042f6dbf59daa Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sun, 23 May 2021 20:56:29 +0200 Subject: [PATCH 52/91] Drop contexter from test requirements (#426) The contexter package is a third-party backport of contextlib (with some extensions). It is no longer required because Nox does not support Python 2 anymore, and we only ever used the standard ExitStack interface. The contexter package was used only in the test suite. --- noxfile.py | 1 - requirements-test.txt | 1 - tests/test_main.py | 6 +++--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/noxfile.py b/noxfile.py index 802f6dcb..67d13bfc 100644 --- a/noxfile.py +++ b/noxfile.py @@ -58,7 +58,6 @@ def conda_tests(session): session.conda_install( "--file", "requirements-conda-test.txt", "--channel", "conda-forge" ) - session.install("contexter", "--no-deps") session.install("-e", ".", "--no-deps") tests = session.posargs or ["tests/"] session.run("pytest", *tests) diff --git a/requirements-test.txt b/requirements-test.txt index d49e0d07..6a0bc991 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,7 +1,6 @@ flask pytest pytest-cov -contexter sphinx sphinx-autobuild recommonmark diff --git a/tests/test_main.py b/tests/test_main.py index 0a1322a0..be7c8413 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import os import sys from pathlib import Path from unittest import mock -import contexter import nox import nox.__main__ import nox._options @@ -296,7 +296,7 @@ def test_main_positional_flag_like_with_double_hyphen(monkeypatch): def test_main_version(capsys, monkeypatch): monkeypatch.setattr(sys, "argv", [sys.executable, "--version"]) - with contexter.ExitStack() as stack: + with contextlib.ExitStack() as stack: execute = stack.enter_context(mock.patch("nox.workflow.execute")) exit_mock = stack.enter_context(mock.patch("sys.exit")) nox.__main__.main() @@ -309,7 +309,7 @@ def test_main_version(capsys, monkeypatch): def test_main_help(capsys, monkeypatch): monkeypatch.setattr(sys, "argv", [sys.executable, "--help"]) - with contexter.ExitStack() as stack: + with contextlib.ExitStack() as stack: execute = stack.enter_context(mock.patch("nox.workflow.execute")) exit_mock = stack.enter_context(mock.patch("sys.exit")) nox.__main__.main() From e012b603a3d03a0ef74adc1c4430da79604072d7 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Mon, 24 May 2021 19:03:50 +0200 Subject: [PATCH 53/91] Fix environments not being reused due to wrong Python lookup (#428) --- nox/virtualenv.py | 6 ++++-- tests/test_virtualenv.py | 11 ++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index cd329457..cd231b32 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -336,9 +336,11 @@ def _check_reused_environment_interpreter(self) -> bool: """Check if reused environment interpreter is the same.""" program = "import sys; print(getattr(sys, 'real_prefix', sys.base_prefix))" original = nox.command.run( - [self._resolved_interpreter, "-c", program], silent=True + [self._resolved_interpreter, "-c", program], silent=True, log=False + ) + created = nox.command.run( + ["python", "-c", program], silent=True, log=False, paths=self.bin_paths ) - created = nox.command.run(["python", "-c", program], silent=True) return original == created @property diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 59dd1804..58cf0b6b 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -410,13 +410,9 @@ def test_create_reuse_venv_environment(make_one): venv, location = make_one(reuse_existing=True, venv=True) venv.create() - # Use a spurious occurrence of "virtualenv" in the pyvenv.cfg. - pyvenv_cfg = """\ - home = /opt/virtualenv/bin - include-system-site-packages = false - version = 3.9.6 - """ - location.join("pyvenv.cfg").write(dedent(pyvenv_cfg)) + # Place a spurious occurrence of "virtualenv" in the pyvenv.cfg. + pyvenv_cfg = location.join("pyvenv.cfg") + pyvenv_cfg.write(pyvenv_cfg.read() + "bogus = virtualenv\n") reused = not venv.create() @@ -424,6 +420,7 @@ def test_create_reuse_venv_environment(make_one): assert reused +@pytest.mark.skipif(IS_WINDOWS, reason="Avoid 'No pyvenv.cfg file' error on Windows.") def test_create_reuse_oldstyle_virtualenv_environment(make_one): venv, location = make_one(reuse_existing=True) venv.create() From 1d0ff6f783c93e0c02fe4630e14e02bf90552742 Mon Sep 17 00:00:00 2001 From: Stargirl Flowers Date: Wed, 26 May 2021 17:09:56 -0700 Subject: [PATCH 54/91] Allow colorlog <7.0.0 (#431) 5 is the latest release, and 6.x just drops support for Python 2, so we should be good. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c8de26b1..d146981c 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ zip_safe=False, install_requires=[ "argcomplete>=1.9.4,<2.0", - "colorlog>=2.6.1,<5.0.0", + "colorlog>=2.6.1,<7.0.0", "packaging>=20.9", "py>=1.4.0,<2.0.0", "virtualenv>=14.0.0", From e9945efa654bf5a9d6af10016fded9b5b3f76fe9 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Thu, 27 May 2021 16:55:19 +0200 Subject: [PATCH 55/91] Add --force-python option as shorthand for --python and --extra-python (#427) * Add test for --force-python option * Add --force-python option as shorthand for --python and --extra-python * Document the --force-python option --- docs/usage.rst | 11 ++++++++--- nox/_options.py | 21 +++++++++++++++++++++ tests/test_main.py | 9 +++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index bca73f39..55675510 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -163,22 +163,27 @@ If the Noxfile sets ``nox.options.reuse_existing_virtualenvs``, you can override Running additional Python versions ---------------------------------- -In addition to Nox supporting executing single sessions, it also supports runnings python versions that aren't specified using ``--extra-pythons``. + +In addition to Nox supporting executing single sessions, it also supports running Python versions that aren't specified using ``--extra-pythons``. .. code-block:: console nox --extra-pythons 3.8 3.9 -This will, in addition to specified python versions in the Noxfile, also create sessions for the specified versions. +This will, in addition to specified Python versions in the Noxfile, also create sessions for the specified versions. This option can be combined with ``--python`` to replace, instead of appending, the Python interpreter for a given session:: nox --python 3.10 --extra-python 3.10 -s lint +Instead of passing both options, you can use the ``--force-python`` shorthand:: + + nox --force-python 3.10 -s lint + Also, you can specify ``python`` in place of a specific version. This will run the session using the ``python`` specified for the current ``PATH``:: - nox --python python --extra-python python -s lint + nox --force-python python -s lint .. _opt-stop-on-first-error: diff --git a/nox/_options.py b/nox/_options.py index 8f66b1aa..4fbd8f20 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -148,6 +148,15 @@ def _color_finalizer(value: bool, args: argparse.Namespace) -> bool: return sys.stdout.isatty() +def _force_pythons_finalizer( + value: Sequence[str], args: argparse.Namespace +) -> Sequence[str]: + """Propagate ``--force-python`` to ``--python`` and ``--extra-python``.""" + if value: + args.pythons = args.extra_pythons = value + return value + + def _posargs_finalizer( value: Sequence[Any], args: argparse.Namespace ) -> Union[Sequence[Any], List[Any]]: @@ -335,6 +344,18 @@ def _session_completer( nargs="*", help="Additionally, run sessions using the given python interpreter versions.", ), + _option_set.Option( + "force_pythons", + "--force-pythons", + "--force-python", + group=options.groups["secondary"], + nargs="*", + help=( + "Run sessions with the given interpreters instead of those listed in the Noxfile." + " This is a shorthand for ``--python=X.Y --extra-python=X.Y``." + ), + finalizer_func=_force_pythons_finalizer, + ), *_option_set.make_flag_pair( "stop_on_first_error", ("-x", "--stop-on-first-error"), diff --git a/tests/test_main.py b/tests/test_main.py index be7c8413..90120c37 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -578,3 +578,12 @@ def test_main_color_conflict(capsys, monkeypatch): _, err = capsys.readouterr() assert "color" in err + + +def test_main_force_python(monkeypatch): + monkeypatch.setattr(sys, "argv", ["nox", "--force-python=3.10"]) + with mock.patch("nox.workflow.execute", return_value=0) as execute: + with mock.patch.object(sys, "exit"): + nox.__main__.main() + config = execute.call_args[1]["global_config"] + assert config.pythons == config.extra_pythons == ["3.10"] From 56c7d56f656c032a98ad17a0924893c338a842e6 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Tue, 1 Jun 2021 12:15:58 +0200 Subject: [PATCH 56/91] Add option --no-install to skip install commands in reused environments (#432) Use either of these to reuse a virtualenv without re-installing packages: ``` nox -R nox --reuse-existing-virtualenvs --no-install ``` The `--no-install` option causes the following session methods to return early: - `session.install` - `session.conda_install` - `session.run_always` This option has no effect if the virtualenv is not being reused. Co-authored-by: Jam * Add option --no-install * Add test for VirtualEnv._reused * Add test for CondaEnv._reused * Add {Virtual,Conda,Process}Env._reused * Add test for session.install with --no-install * Skip session.install when --no-install is given * Add test for session.conda_install with --no-install * Skip session.conda_install when --no-install is given * Add test for session.run_always with --no-install * Skip session.run_always when --no-install is given * Add test for short option -R * Add short option -R for `--reuse-existing-virtualenvs --no-install` * Document the --no-install and -R options * Update broken link to pip documentation * Clarify documentation of session.run_always --- docs/usage.rst | 17 ++++++++++- nox/_options.py | 31 ++++++++++++++++++++ nox/sessions.py | 29 ++++++++++++++++-- nox/virtualenv.py | 7 +++++ tests/test_main.py | 9 ++++++ tests/test_sessions.py | 63 ++++++++++++++++++++++++++++++++++++++++ tests/test_virtualenv.py | 4 +++ 7 files changed, 156 insertions(+), 4 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 55675510..8103cb2a 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -149,7 +149,7 @@ Finally note that the ``--no-venv`` flag is a shortcut for ``--force-venv-backen Re-using virtualenvs -------------------- -By default, Nox deletes and recreates virtualenvs every time it is run. This is usually fine for most projects and continuous integration environments as `pip's caching `_ makes re-install rather quick. However, there are some situations where it is advantageous to re-use the virtualenvs between runs. Use ``-r`` or ``--reuse-existing-virtualenvs``: +By default, Nox deletes and recreates virtualenvs every time it is run. This is usually fine for most projects and continuous integration environments as `pip's caching `_ makes re-install rather quick. However, there are some situations where it is advantageous to re-use the virtualenvs between runs. Use ``-r`` or ``--reuse-existing-virtualenvs``: .. code-block:: console @@ -159,6 +159,21 @@ By default, Nox deletes and recreates virtualenvs every time it is run. This is If the Noxfile sets ``nox.options.reuse_existing_virtualenvs``, you can override the Noxfile setting from the command line by using ``--no-reuse-existing-virtualenvs``. +Additionally, you can skip the re-installation of packages when a virtualenv is reused. Use ``-R`` or ``--reuse-existing-virtualenvs --no-install``: + +.. code-block:: console + + nox -R + nox --reuse-existing-virtualenvs --no-install + +The ``--no-install`` option causes the following session methods to return early: + +- :func:`session.install ` +- :func:`session.conda_install ` +- :func:`session.run_always ` + +This option has no effect if the virtualenv is not being reused. + .. _opt-running-extra-pythons: Running additional Python versions diff --git a/nox/_options.py b/nox/_options.py index 4fbd8f20..770b8438 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -157,6 +157,13 @@ def _force_pythons_finalizer( return value +def _R_finalizer(value: bool, args: argparse.Namespace) -> bool: + """Propagate -R to --reuse-existing-virtualenvs and --no-install.""" + if value: + args.reuse_existing_virtualenvs = args.no_install = value + return value + + def _posargs_finalizer( value: Sequence[Any], args: argparse.Namespace ) -> Union[Sequence[Any], List[Any]]: @@ -320,6 +327,18 @@ def _session_completer( group=options.groups["secondary"], help="Re-use existing virtualenvs instead of recreating them.", ), + _option_set.Option( + "R", + "-R", + default=False, + group=options.groups["secondary"], + action="store_true", + help=( + "Re-use existing virtualenvs and skip package re-installation." + " This is an alias for '--reuse-existing-virtualenvs --no-install'." + ), + finalizer_func=_R_finalizer, + ), _option_set.Option( "noxfile", "-f", @@ -384,6 +403,18 @@ def _session_completer( action="store_true", help="Skip session.run invocations in the Noxfile.", ), + _option_set.Option( + "no_install", + "--no-install", + default=False, + group=options.groups["secondary"], + action="store_true", + help=( + "Skip invocations of session methods for installing packages" + " (session.install, session.conda_install, session.run_always)" + " when a virtualenv is being reused." + ), + ), _option_set.Option( "report", "--report", diff --git a/nox/sessions.py b/nox/sessions.py index 52f647a1..6ed894a6 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -272,8 +272,16 @@ def run_always( ) -> Optional[Any]: """Run a command **always**. - This is a variant of :meth:`run` that runs in all cases, including in - the presence of ``--install-only``. + This is a variant of :meth:`run` that runs even in the presence of + ``--install-only``. This method returns early if ``--no-install`` is + specified and the virtualenv is being reused. + + Here are some cases where this method is useful: + + - You need to install packages using a command other than ``pip + install`` or ``conda install``. + - You need to run a command as a prerequisite of package installation, + such as building a package or compiling a binary extension. :param env: A dictionary of environment variables to expose to the command. By default, all environment variables are passed. @@ -290,6 +298,13 @@ def run_always( do not have a virtualenv. :type external: bool """ + if ( + self._runner.global_config.no_install + and self._runner.venv is not None + and self._runner.venv._reused + ): + return None + if not args: raise ValueError("At least one argument required to run_always().") @@ -368,6 +383,9 @@ def conda_install( if not args: raise ValueError("At least one argument required to install().") + if self._runner.global_config.no_install and venv._reused: + return None + # Escape args that should be (conda-specific; pip install does not need this) args = _dblquote_pkg_install_args(args) @@ -417,8 +435,10 @@ def install(self, *args: str, **kwargs: Any) -> None: .. _pip: https://pip.readthedocs.org """ + venv = self._runner.venv + if not isinstance( - self._runner.venv, (CondaEnv, VirtualEnv, PassthroughEnv) + venv, (CondaEnv, VirtualEnv, PassthroughEnv) ): # pragma: no cover raise ValueError( "A session without a virtualenv can not install dependencies." @@ -426,6 +446,9 @@ def install(self, *args: str, **kwargs: Any) -> None: if not args: raise ValueError("At least one argument required to install().") + if self._runner.global_config.no_install and venv._reused: + return None + if "silent" not in kwargs: kwargs["silent"] = True diff --git a/nox/virtualenv.py b/nox/virtualenv.py index cd231b32..f8ff176c 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -54,6 +54,7 @@ class ProcessEnv: def __init__(self, bin_paths: None = None, env: Mapping[str, str] = None) -> None: self._bin_paths = bin_paths self.env = os.environ.copy() + self._reused = False if env is not None: self.env.update(env) @@ -224,6 +225,9 @@ def create(self) -> bool: logger.debug( "Re-using existing conda env at {}.".format(self.location_name) ) + + self._reused = True + return False cmd = ["conda", "create", "--yes", "--prefix", self.location] @@ -423,6 +427,9 @@ def create(self) -> bool: self.location_name ) ) + + self._reused = True + return False if self.venv_or_virtualenv == "virtualenv": diff --git a/tests/test_main.py b/tests/test_main.py index 90120c37..35395819 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -587,3 +587,12 @@ def test_main_force_python(monkeypatch): nox.__main__.main() config = execute.call_args[1]["global_config"] assert config.pythons == config.extra_pythons == ["3.10"] + + +def test_main_reuse_existing_virtualenvs_no_install(monkeypatch): + monkeypatch.setattr(sys, "argv", ["nox", "-R"]) + with mock.patch("nox.workflow.execute", return_value=0) as execute: + with mock.patch.object(sys, "exit"): + nox.__main__.main() + config = execute.call_args[1]["global_config"] + assert config.reuse_existing_virtualenvs and config.no_install diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 150c15ec..93b9b155 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -310,6 +310,25 @@ def test_run_always_install_only(self, caplog): assert session.run_always(operator.add, 23, 19) == 42 + @pytest.mark.parametrize( + ("no_install", "reused", "run_called"), + [ + (True, True, False), + (True, False, True), + (False, True, True), + (False, False, True), + ], + ) + def test_run_always_no_install(self, no_install, reused, run_called): + session, runner = self.make_session_and_runner() + runner.global_config.no_install = no_install + runner.venv._reused = reused + + with mock.patch.object(nox.command, "run") as run: + session.run_always("python", "-m", "pip", "install", "requests") + + assert run.called is run_called + def test_conda_install_bad_args(self): session, runner = self.make_session_and_runner() runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) @@ -380,6 +399,31 @@ class SessionNoSlots(nox.sessions.Session): external="error", ) + @pytest.mark.parametrize( + ("no_install", "reused", "run_called"), + [ + (True, True, False), + (True, False, True), + (False, True, True), + (False, False, True), + ], + ) + def test_conda_venv_reused_with_no_install(self, no_install, reused, run_called): + session, runner = self.make_session_and_runner() + + runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) + runner.venv.location = "/path/to/conda/env" + runner.venv.env = {} + runner.venv.is_offline = lambda: True + + runner.global_config.no_install = no_install + runner.venv._reused = reused + + with mock.patch.object(nox.command, "run") as run: + session.conda_install("baked beans", "eggs", "spam") + + assert run.called is run_called + @pytest.mark.parametrize( "version_constraint", ["no", "yes", "already_dbl_quoted"], @@ -538,6 +582,25 @@ def test_skip_no_log(self): with pytest.raises(nox.sessions._SessionSkip): session.skip() + @pytest.mark.parametrize( + ("no_install", "reused", "run_called"), + [ + (True, True, False), + (True, False, True), + (False, True, True), + (False, False, True), + ], + ) + def test_session_venv_reused_with_no_install(self, no_install, reused, run_called): + session, runner = self.make_session_and_runner() + runner.global_config.no_install = no_install + runner.venv._reused = reused + + with mock.patch.object(nox.command, "run") as run: + session.install("eggs", "spam") + + assert run.called is run_called + def test___slots__(self): session, _ = self.make_session_and_runner() with pytest.raises(AttributeError): diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 58cf0b6b..1ff8b436 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -180,6 +180,7 @@ def test_condaenv_create(make_conda): venv.reuse_existing = True venv.create() assert dir_.join("test.txt").check() + assert venv._reused @pytest.mark.skipif(not HAS_CONDA, reason="Missing conda command.") @@ -334,7 +335,10 @@ def test_create(monkeypatch, make_one): assert dir_.join("test.txt").check() venv.reuse_existing = True monkeypatch.setattr(nox.virtualenv.nox.command, "run", mock.MagicMock()) + venv.create() + + assert venv._reused assert dir_.join("test.txt").check() From 4ddd05565b94b7e13cad6714da911093c42189cb Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Thu, 3 Jun 2021 18:33:54 +0200 Subject: [PATCH 57/91] Fix garbled error message when session not found (#434) * Add Noxfile for testing normalized session names * Add tests for normalized session names * Add failing test for regression when session not found * Fix garbled error message when session not found --- nox/manifest.py | 7 ++- tests/resources/noxfile_normalization.py | 16 ++++++ tests/test_main.py | 72 ++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 tests/resources/noxfile_normalization.py diff --git a/nox/manifest.py b/nox/manifest.py index 8265de2d..dbc3304e 100644 --- a/nox/manifest.py +++ b/nox/manifest.py @@ -150,10 +150,11 @@ def filter_by_name(self, specified_sessions: Iterable[str]) -> None: ), ) ) - normalized_specified_sessions = [ - _normalize_arg(session_name) for session_name in specified_sessions + missing_sessions = [ + session_name + for session_name in specified_sessions + if _normalize_arg(session_name) not in all_sessions ] - missing_sessions = set(normalized_specified_sessions) - all_sessions if missing_sessions: raise KeyError("Sessions not found: {}".format(", ".join(missing_sessions))) diff --git a/tests/resources/noxfile_normalization.py b/tests/resources/noxfile_normalization.py new file mode 100644 index 00000000..d1a07fe7 --- /dev/null +++ b/tests/resources/noxfile_normalization.py @@ -0,0 +1,16 @@ +import datetime + +import nox + + +class Foo: + pass + + +@nox.session +@nox.parametrize( + "arg", + ["Jane", "Joe's", '"hello world"', datetime.datetime(1980, 1, 1), [42], Foo()], +) +def test(session, arg): + pass diff --git a/tests/test_main.py b/tests/test_main.py index 35395819..9f41ac5d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -370,6 +370,78 @@ def test_main_session_with_names(capsys, monkeypatch): sys_exit.assert_called_once_with(0) +@pytest.fixture +def run_nox(capsys, monkeypatch): + def _run_nox(*args): + monkeypatch.setattr(sys, "argv", ["nox", *args]) + + with mock.patch("sys.exit") as sys_exit: + nox.__main__.main() + stdout, stderr = capsys.readouterr() + returncode = sys_exit.call_args[0][0] + + return returncode, stdout, stderr + + return _run_nox + + +@pytest.mark.parametrize( + ("normalized_name", "session"), + [ + ("test(arg='Jane')", "test(arg='Jane')"), + ("test(arg='Jane')", 'test(arg="Jane")'), + ("test(arg='Jane')", 'test(arg = "Jane")'), + ("test(arg='Jane')", 'test ( arg = "Jane" )'), + ('test(arg="Joe\'s")', 'test(arg="Joe\'s")'), + ('test(arg="Joe\'s")', "test(arg='Joe\\'s')"), + ("test(arg='\"hello world\"')", "test(arg='\"hello world\"')"), + ("test(arg='\"hello world\"')", 'test(arg="\\"hello world\\"")'), + ("test(arg=[42])", "test(arg=[42])"), + ("test(arg=[42])", "test(arg=[42,])"), + ("test(arg=[42])", "test(arg=[ 42 ])"), + ("test(arg=[42])", "test(arg=[0x2a])"), + ( + "test(arg=datetime.datetime(1980, 1, 1, 0, 0))", + "test(arg=datetime.datetime(1980, 1, 1, 0, 0))", + ), + ( + "test(arg=datetime.datetime(1980, 1, 1, 0, 0))", + "test(arg=datetime.datetime(1980,1,1,0,0))", + ), + ( + "test(arg=datetime.datetime(1980, 1, 1, 0, 0))", + "test(arg=datetime.datetime(1980, 1, 1, 0, 0x0))", + ), + ], +) +def test_main_with_normalized_session_names(run_nox, normalized_name, session): + noxfile = os.path.join(RESOURCES, "noxfile_normalization.py") + returncode, _, stderr = run_nox(f"--noxfile={noxfile}", f"--session={session}") + assert returncode == 0 + assert normalized_name in stderr + + +@pytest.mark.parametrize( + "session", + [ + "syntax error", + "test(arg=Jane)", + "test(arg='Jane ')", + "_test(arg='Jane')", + "test(arg=42)", + "test(arg=[42.0])", + "test(arg=[43])", + "test(arg=)", + "test(arg=datetime.datetime(1980, 1, 1))", + ], +) +def test_main_with_bad_session_names(run_nox, session): + noxfile = os.path.join(RESOURCES, "noxfile_normalization.py") + returncode, _, stderr = run_nox(f"--noxfile={noxfile}", f"--session={session}") + assert returncode != 0 + assert session in stderr + + def test_main_noxfile_options(monkeypatch): monkeypatch.setattr( sys, From 41b9c798e93dae8dc27256a07622c32bc5b9f0fd Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sat, 5 Jun 2021 10:49:43 +0200 Subject: [PATCH 58/91] Upgrade linters to the latest version (#438) --- .isort.cfg | 6 +----- nox/_decorators.py | 2 +- nox/_version.py | 4 ++-- nox/command.py | 1 + nox/popen.py | 4 ++-- nox/sessions.py | 3 ++- nox/tasks.py | 3 ++- nox/virtualenv.py | 5 +++-- noxfile.py | 10 +++++----- tests/test__option_set.py | 1 + tests/test__parametrize.py | 1 + tests/test__version.py | 1 + tests/test_command.py | 5 +++-- tests/test_logger.py | 1 + tests/test_main.py | 3 ++- tests/test_manifest.py | 3 ++- tests/test_registry.py | 1 + tests/test_sessions.py | 3 ++- tests/test_tasks.py | 3 ++- tests/test_tox_to_nox.py | 1 + tests/test_virtualenv.py | 3 ++- 21 files changed, 38 insertions(+), 26 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index ba2778dc..b9fb3f3e 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,2 @@ [settings] -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -line_length=88 +profile=black diff --git a/nox/_decorators.py b/nox/_decorators.py index b04e48ab..153e331b 100644 --- a/nox/_decorators.py +++ b/nox/_decorators.py @@ -27,7 +27,7 @@ def _copy_func(src: Callable, name: str = None) -> Callable: closure=src.__closure__, # type: ignore ) dst.__dict__.update(copy.deepcopy(src.__dict__)) - dst = functools.update_wrapper(dst, src) # type: ignore + dst = functools.update_wrapper(dst, src) dst.__kwdefaults__ = src.__kwdefaults__ # type: ignore return dst diff --git a/nox/_version.py b/nox/_version.py index 296f2e67..740b7fd3 100644 --- a/nox/_version.py +++ b/nox/_version.py @@ -20,9 +20,9 @@ from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.version import InvalidVersion, Version -try: +if sys.version_info >= (3, 8): # pragma: no cover import importlib.metadata as metadata -except ImportError: # pragma: no cover +else: # pragma: no cover import importlib_metadata as metadata diff --git a/nox/command.py b/nox/command.py index a9a14a82..8d915054 100644 --- a/nox/command.py +++ b/nox/command.py @@ -17,6 +17,7 @@ from typing import Any, Iterable, List, Optional, Sequence, Union import py + from nox.logger import logger from nox.popen import popen diff --git a/nox/popen.py b/nox/popen.py index 8e981e1b..9bd1c115 100644 --- a/nox/popen.py +++ b/nox/popen.py @@ -16,10 +16,10 @@ import locale import subprocess import sys -from typing import IO, Mapping, Optional, Sequence, Tuple, Union +from typing import IO, Mapping, Sequence, Tuple, Union -def shutdown_process(proc: subprocess.Popen) -> Tuple[Optional[bytes], Optional[bytes]]: +def shutdown_process(proc: subprocess.Popen) -> Tuple[bytes, bytes]: """Gracefully shutdown a child process.""" with contextlib.suppress(subprocess.TimeoutExpired): diff --git a/nox/sessions.py b/nox/sessions.py index 6ed894a6..31a28e75 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -32,8 +32,9 @@ Union, ) -import nox.command import py + +import nox.command from nox import _typing from nox._decorators import Func from nox.logger import logger diff --git a/nox/tasks.py b/nox/tasks.py index c8d99b0b..c23b118d 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -20,8 +20,9 @@ from argparse import Namespace from typing import List, Union -import nox from colorlog.escape_codes import parse_colors + +import nox from nox import _options, registry from nox._version import InvalidVersionSpecifier, VersionCheckFailed, check_nox_version from nox.logger import logger diff --git a/nox/virtualenv.py b/nox/virtualenv.py index f8ff176c..ab015268 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -20,8 +20,9 @@ from socket import gethostbyname from typing import Any, List, Mapping, Optional, Tuple, Union -import nox.command import py + +import nox.command from nox.logger import logger from . import _typing @@ -329,7 +330,7 @@ def _check_reused_environment_type(self) -> bool: # virtualenv < 20.0 does not create pyvenv.cfg old_env = "virtualenv" else: - pattern = re.compile(f"virtualenv[ \t]*=") + pattern = re.compile("virtualenv[ \t]*=") with open(path) as fp: old_env = ( "virtualenv" if any(pattern.match(line) for line in fp) else "venv" diff --git a/noxfile.py b/noxfile.py index 67d13bfc..9e247bb2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -78,15 +78,15 @@ def cover(session): @nox.session(python="3.8") def blacken(session): """Run black code formatter.""" - session.install("black==19.3b0", "isort==4.3.21") + session.install("black==21.5b2", "isort==5.8.0") files = ["nox", "tests", "noxfile.py", "setup.py"] session.run("black", *files) - session.run("isort", "--recursive", *files) + session.run("isort", *files) @nox.session(python="3.8") def lint(session): - session.install("flake8==3.7.8", "black==19.3b0", "isort==4.3.21", "mypy==0.720") + session.install("flake8==3.9.2", "black==21.5b2", "isort==5.8.0", "mypy==0.812") session.run( "mypy", "--config-file=", @@ -97,8 +97,8 @@ def lint(session): ) files = ["nox", "tests", "noxfile.py", "setup.py"] session.run("black", "--check", *files) - session.run("isort", "--check", "--recursive", *files) - session.run("flake8", "nox", *files) + session.run("isort", "--check", *files) + session.run("flake8", *files) @nox.session(python="3.7") diff --git a/tests/test__option_set.py b/tests/test__option_set.py index 3ad8b593..ae238f5e 100644 --- a/tests/test__option_set.py +++ b/tests/test__option_set.py @@ -13,6 +13,7 @@ # limitations under the License. import pytest + from nox import _option_set, _options # The vast majority of _option_set is tested by test_main, but the test helper diff --git a/tests/test__parametrize.py b/tests/test__parametrize.py index 79fc2e5d..e581380b 100644 --- a/tests/test__parametrize.py +++ b/tests/test__parametrize.py @@ -15,6 +15,7 @@ from unittest import mock import pytest + from nox import _decorators, _parametrize, parametrize, session diff --git a/tests/test__version.py b/tests/test__version.py index 2606952c..b4497d02 100644 --- a/tests/test__version.py +++ b/tests/test__version.py @@ -17,6 +17,7 @@ from typing import Optional import pytest + from nox import needs_version from nox._version import ( InvalidVersionSpecifier, diff --git a/tests/test_command.py b/tests/test_command.py index fec272b8..db49a874 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -23,9 +23,10 @@ from textwrap import dedent from unittest import mock +import pytest + import nox.command import nox.popen -import pytest PYTHON = sys.executable @@ -474,7 +475,7 @@ def test_output_decoding_utf8_only_fail(monkeypatch: pytest.MonkeyPatch) -> None def test_output_decoding_utf8_fail_cp1252_success( - monkeypatch: pytest.MonkeyPatch + monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(nox.popen.locale, "getpreferredencoding", lambda: "cp1252") diff --git a/tests/test_logger.py b/tests/test_logger.py index 50563bd5..eef9dcf3 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -16,6 +16,7 @@ from unittest import mock import pytest + from nox import logger diff --git a/tests/test_main.py b/tests/test_main.py index 9f41ac5d..15f7b506 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -18,12 +18,13 @@ from pathlib import Path from unittest import mock +import pytest + import nox import nox.__main__ import nox._options import nox.registry import nox.sessions -import pytest try: import importlib.metadata as metadata diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 9fc0578e..e807619e 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -15,8 +15,9 @@ import collections from unittest import mock -import nox import pytest + +import nox from nox._decorators import Func from nox.manifest import ( WARN_PYTHONS_IGNORED, diff --git a/tests/test_registry.py b/tests/test_registry.py index 652633e1..3c36f9a1 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -13,6 +13,7 @@ # limitations under the License. import pytest + from nox import registry diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 93b9b155..3573b45e 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -21,12 +21,13 @@ from pathlib import Path from unittest import mock +import pytest + import nox.command import nox.manifest import nox.registry import nox.sessions import nox.virtualenv -import pytest from nox import _options from nox.logger import logger diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 65f3112d..967f605e 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -21,8 +21,9 @@ from textwrap import dedent from unittest import mock -import nox import pytest + +import nox from nox import _options, sessions, tasks from nox.manifest import WARN_PYTHONS_IGNORED, Manifest diff --git a/tests/test_tox_to_nox.py b/tests/test_tox_to_nox.py index 1e5ff7a0..57ec2bf7 100644 --- a/tests/test_tox_to_nox.py +++ b/tests/test_tox_to_nox.py @@ -16,6 +16,7 @@ import textwrap import pytest + from nox import tox_to_nox diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 1ff8b436..ac0fa598 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -18,10 +18,11 @@ from textwrap import dedent from unittest import mock -import nox.virtualenv import py import pytest +import nox.virtualenv + IS_WINDOWS = nox.virtualenv._SYSTEM == "Windows" HAS_CONDA = shutil.which("conda") is not None RAISE_ERROR = "RAISE_ERROR" From bcaa5ffa42173037030f67e2ae58d10fbe659945 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sun, 6 Jun 2021 14:51:55 +0200 Subject: [PATCH 59/91] Prevent sessions from modifying each other's posargs (#439) * test: Do not compare posargs using `is` * test: Do not set posargs to mock sentinel This breaks tests because sentinels cannot be subscripted. * test: Initialize posargs to an empty list, not None * test: Add failing test for bleed-through of posargs between sessions * Give each session its own copy of posargs --- nox/sessions.py | 2 +- tests/test__option_set.py | 6 ++++-- tests/test_manifest.py | 2 +- tests/test_sessions.py | 37 +++++++++++++++++++++++++++---------- tests/test_tasks.py | 20 ++++++++++++++------ 5 files changed, 47 insertions(+), 20 deletions(-) diff --git a/nox/sessions.py b/nox/sessions.py index 31a28e75..cabaae12 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -506,7 +506,7 @@ def __init__( self.global_config = global_config self.manifest = manifest self.venv: Optional[ProcessEnv] = None - self.posargs: List[str] = global_config.posargs + self.posargs: List[str] = global_config.posargs[:] @property def description(self) -> Optional[str]: diff --git a/tests/test__option_set.py b/tests/test__option_set.py index ae238f5e..202473ea 100644 --- a/tests/test__option_set.py +++ b/tests/test__option_set.py @@ -56,7 +56,7 @@ def test_namespace_non_existant_options_with_values(self): optionset.namespace(non_existant_option="meep") def test_session_completer(self): - parsed_args = _options.options.namespace(sessions=(), keywords=()) + parsed_args = _options.options.namespace(sessions=(), keywords=(), posargs=[]) all_nox_sessions = _options._session_completer( prefix=None, parsed_args=parsed_args ) @@ -66,7 +66,9 @@ def test_session_completer(self): assert len(set(some_expected_sessions) - set(all_nox_sessions)) == 0 def test_session_completer_invalid_sessions(self): - parsed_args = _options.options.namespace(sessions=("baz",), keywords=()) + parsed_args = _options.options.namespace( + sessions=("baz",), keywords=(), posargs=[] + ) all_nox_sessions = _options._session_completer( prefix=None, parsed_args=parsed_args ) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index e807619e..4c2cf151 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -373,7 +373,7 @@ def test_notify_with_posargs(): # delete my_session from the queue manifest.filter_by_name(()) - assert session.posargs is cfg.posargs + assert session.posargs == cfg.posargs assert manifest.notify("my_session", posargs=["--an-arg"]) assert session.posargs == ["--an-arg"] diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 3573b45e..89df734e 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -67,9 +67,7 @@ def make_session_and_runner(self): signatures=["test"], func=func, global_config=_options.options.namespace( - posargs=mock.sentinel.posargs, - error_on_external_run=False, - install_only=False, + posargs=[], error_on_external_run=False, install_only=False ), manifest=mock.create_autospec(nox.manifest.Manifest), ) @@ -101,7 +99,7 @@ def test_properties(self): assert session.name is runner.friendly_name assert session.env is runner.venv.env - assert session.posargs is runner.global_config.posargs + assert session.posargs == runner.global_config.posargs assert session.virtualenv is runner.venv assert session.bin_paths is runner.venv.bin_paths assert session.bin is runner.venv.bin_paths[0] @@ -371,7 +369,7 @@ def test_conda_install(self, auto_offline, offline): name="test", signatures=["test"], func=mock.sentinel.func, - global_config=_options.options.namespace(posargs=mock.sentinel.posargs), + global_config=_options.options.namespace(posargs=[]), manifest=mock.create_autospec(nox.manifest.Manifest), ) runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) @@ -435,7 +433,7 @@ def test_conda_install_non_default_kwargs(self, version_constraint): name="test", signatures=["test"], func=mock.sentinel.func, - global_config=_options.options.namespace(posargs=mock.sentinel.posargs), + global_config=_options.options.namespace(posargs=[]), manifest=mock.create_autospec(nox.manifest.Manifest), ) runner.venv = mock.create_autospec(nox.virtualenv.CondaEnv) @@ -492,7 +490,7 @@ def test_install(self): name="test", signatures=["test"], func=mock.sentinel.func, - global_config=_options.options.namespace(posargs=mock.sentinel.posargs), + global_config=_options.options.namespace(posargs=[]), manifest=mock.create_autospec(nox.manifest.Manifest), ) runner.venv = mock.create_autospec(nox.virtualenv.VirtualEnv) @@ -521,7 +519,7 @@ def test_install_non_default_kwargs(self): name="test", signatures=["test"], func=mock.sentinel.func, - global_config=_options.options.namespace(posargs=mock.sentinel.posargs), + global_config=_options.options.namespace(posargs=[]), manifest=mock.create_autospec(nox.manifest.Manifest), ) runner.venv = mock.create_autospec(nox.virtualenv.VirtualEnv) @@ -556,6 +554,25 @@ def test_notify(self): runner.manifest.notify.assert_called_with("other", ["--an-arg"]) + def test_posargs_are_not_shared_between_sessions(self, monkeypatch, tmp_path): + registry = {} + monkeypatch.setattr("nox.registry._REGISTRY", registry) + + @nox.session(venv_backend="none") + def test(session): + session.posargs.extend(["-x"]) + + @nox.session(venv_backend="none") + def lint(session): + if "-x" in session.posargs: + raise RuntimeError("invalid option: -x") + + config = _options.options.namespace(posargs=[]) + manifest = nox.manifest.Manifest(registry, config) + + assert manifest["test"].execute() + assert manifest["lint"].execute() + def test_log(self, caplog): caplog.set_level(logging.INFO) session, _ = self.make_session_and_runner() @@ -628,7 +645,7 @@ def make_runner(self): global_config=_options.options.namespace( noxfile=os.path.join(os.getcwd(), "noxfile.py"), envdir="envdir", - posargs=mock.sentinel.posargs, + posargs=[], reuse_existing_virtualenvs=False, error_on_missing_interpreters=False, ), @@ -644,7 +661,7 @@ def test_properties(self): assert runner.func is not None assert callable(runner.func) assert isinstance(runner.description, str) - assert runner.global_config.posargs == mock.sentinel.posargs + assert runner.global_config.posargs == [] assert isinstance(runner.manifest, nox.manifest.Manifest) def test_str_and_friendly_name(self): diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 967f605e..9827cfc8 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -140,7 +140,7 @@ def notasession(): mock_module = argparse.Namespace( __name__=foo.__module__, foo=foo, bar=bar, notasession=notasession ) - config = _options.options.namespace(sessions=(), keywords=()) + config = _options.options.namespace(sessions=(), keywords=(), posargs=[]) # Get the manifest and establish that it looks like what we expect. manifest = tasks.discover_manifest(mock_module, config) @@ -150,7 +150,9 @@ def notasession(): def test_filter_manifest(): - config = _options.options.namespace(sessions=(), pythons=(), keywords=()) + config = _options.options.namespace( + sessions=(), pythons=(), keywords=(), posargs=[] + ) manifest = Manifest({"foo": session_func, "bar": session_func}, config) return_value = tasks.filter_manifest(manifest, config) assert return_value is manifest @@ -158,14 +160,18 @@ def test_filter_manifest(): def test_filter_manifest_not_found(): - config = _options.options.namespace(sessions=("baz",), pythons=(), keywords=()) + config = _options.options.namespace( + sessions=("baz",), pythons=(), keywords=(), posargs=[] + ) manifest = Manifest({"foo": session_func, "bar": session_func}, config) return_value = tasks.filter_manifest(manifest, config) assert return_value == 3 def test_filter_manifest_pythons(): - config = _options.options.namespace(sessions=(), pythons=("3.8",), keywords=()) + config = _options.options.namespace( + sessions=(), pythons=("3.8",), keywords=(), posargs=[] + ) manifest = Manifest( {"foo": session_func_with_python, "bar": session_func, "baz": session_func}, config, @@ -176,7 +182,9 @@ def test_filter_manifest_pythons(): def test_filter_manifest_keywords(): - config = _options.options.namespace(sessions=(), pythons=(), keywords="foo or bar") + config = _options.options.namespace( + sessions=(), pythons=(), keywords="foo or bar", posargs=[] + ) manifest = Manifest( {"foo": session_func, "bar": session_func, "baz": session_func}, config ) @@ -231,7 +239,7 @@ def test_verify_manifest_empty(): def test_verify_manifest_nonempty(): - config = _options.options.namespace(sessions=(), keywords=()) + config = _options.options.namespace(sessions=(), keywords=(), posargs=[]) manifest = Manifest({"session": session_func}, config) return_value = tasks.verify_manifest_nonempty(manifest, global_config=config) assert return_value == manifest From da48d3c7a034dffdd21e0141ca499a8eb329f759 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sun, 6 Jun 2021 15:14:18 +0200 Subject: [PATCH 60/91] Release 2021.6.6 (#440) --- CHANGELOG.md | 27 ++++++++++++++++++++++++++- setup.py | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a6675d5..1e6c9d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,32 @@ # Changelog +## 2021.6.6 + +- Add option `--no-install` to skip install commands in reused environments. (#432) +- Add option `--force-python` as shorthand for `--python` and `--extra-python`. (#427) +- Do not reuse environments if the interpreter or the environment type has changed. (#418, #425, #428) +- Allow common variations in session names with parameters, such as double quotes instead of single quotes. Session names are considered equal if they produce the same Python AST. (#417, #434) +- Preserve the order of parameters in session names. (#401) +- Allow `@nox.parametrize` to select the session Python. (#413) +- Allow passing `posargs` when scheduling another session via `session.notify`. (#397) +- Prevent sessions from modifying each other's posargs. (#439) +- Add `nox.needs_version` to specify Nox version requirements. (#388) +- Add `session.name` to get the session name. (#386) +- Gracefully shutdown child processes. (#393) +- Decode command output using the system locale if UTF-8 decoding fails. (#380) +- Fix creation of Conda environments when `venv_params` is used. (#420) +- Various improvements to Nox's type annotations. (#376, #377, #378) +- Remove outdated notes on Windows compatibility from the documentation. (#382) +- Increase Nox's test coverage on Windows. (#300) +- Avoid mypy searching for configuration files in other directories. (#402) +- Replace AppVeyor and Travis CI by GitHub Actions. (#389, #390, #403) +- Allow colorlog <7.0.0. (#431) +- Drop contexter from test requirements. (#426) +- Upgrade linters to the latest version. (#438) + ## 2020.12.31 -- Fix `NoxColoredFormatter.format`(#374) + +- Fix `NoxColoredFormatter.format` (#374) - Use conda remove to clean up existing conda environments (#373) - Support users specifying an undeclared parametrization of python via `--extra-python` (#361) - Support double-digit minor version in `python` keyword (#367) diff --git a/setup.py b/setup.py index d146981c..d3db6370 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="nox", - version="2020.12.31", + version="2021.6.6", description="Flexible test automation.", long_description=long_description, url="https://nox.thea.codes", From 24fc2639ccd0e4a7284f70a08a2b46ac053e16d1 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Thu, 10 Jun 2021 14:38:58 +0200 Subject: [PATCH 61/91] Avoid polluting tests/resources with a .nox directory (#445) --- tests/resources/noxfile_normalization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/resources/noxfile_normalization.py b/tests/resources/noxfile_normalization.py index d1a07fe7..36a24880 100644 --- a/tests/resources/noxfile_normalization.py +++ b/tests/resources/noxfile_normalization.py @@ -7,7 +7,7 @@ class Foo: pass -@nox.session +@nox.session(venv_backend="none") @nox.parametrize( "arg", ["Jane", "Joe's", '"hello world"', datetime.datetime(1980, 1, 1), [42], Foo()], From 0fb841d05cf3f9ffc99a6fa8993d641e4be548a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20de=20la=20Bru=C3=A8re-T?= Date: Fri, 11 Jun 2021 02:21:16 -0400 Subject: [PATCH 62/91] Group CLI arguments in functional groups (#305) (#442) --- nox/_options.py | 88 ++++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 34 deletions(-) diff --git a/nox/_options.py b/nox/_options.py index 770b8438..bf5d044b 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -29,14 +29,34 @@ options.add_groups( _option_set.OptionGroup( - "primary", - "Primary arguments", - "These are the most common arguments used when invoking Nox.", + "general", + "General options", + "These are general arguments used when invoking Nox.", ), _option_set.OptionGroup( - "secondary", - "Additional arguments & flags", - "These arguments are used to control Nox's behavior or control advanced features.", + "sessions", + "Sessions options", + "These arguments are used to control which Nox session(s) to execute.", + ), + _option_set.OptionGroup( + "python", + "Python options", + "These arguments are used to control which Python version(s) to use.", + ), + _option_set.OptionGroup( + "environment", + "Environment options", + "These arguments are used to control Nox's creation and usage of virtual environments.", + ), + _option_set.OptionGroup( + "execution", + "Execution options", + "These arguments are used to control execution of sessions.", + ), + _option_set.OptionGroup( + "reporting", + "Reporting options", + "These arguments are used to control Nox's reporting during execution.", ), ) @@ -209,14 +229,14 @@ def _session_completer( "help", "-h", "--help", - group=options.groups["primary"], + group=options.groups["general"], action="store_true", help="Show this help message and exit.", ), _option_set.Option( "version", "--version", - group=options.groups["primary"], + group=options.groups["general"], action="store_true", help="Show the Nox version and exit.", ), @@ -225,7 +245,7 @@ def _session_completer( "-l", "--list-sessions", "--list", - group=options.groups["primary"], + group=options.groups["sessions"], action="store_true", help="List all available sessions and exit.", ), @@ -235,7 +255,7 @@ def _session_completer( "-e", "--sessions", "--session", - group=options.groups["primary"], + group=options.groups["sessions"], noxfile=True, merge_func=functools.partial(_sessions_and_keywords_merge_func, "sessions"), nargs="*", @@ -248,7 +268,7 @@ def _session_completer( "-p", "--pythons", "--python", - group=options.groups["primary"], + group=options.groups["python"], noxfile=True, nargs="*", help="Only run sessions that use the given python interpreter versions.", @@ -257,7 +277,7 @@ def _session_completer( "keywords", "-k", "--keywords", - group=options.groups["primary"], + group=options.groups["sessions"], noxfile=True, merge_func=functools.partial(_sessions_and_keywords_merge_func, "keywords"), help="Only run sessions that match the given expression.", @@ -265,7 +285,7 @@ def _session_completer( _option_set.Option( "posargs", "posargs", - group=options.groups["primary"], + group=options.groups["general"], nargs=argparse.REMAINDER, help="Arguments following ``--`` that are passed through to the session(s).", finalizer_func=_posargs_finalizer, @@ -274,7 +294,7 @@ def _session_completer( "verbose", "-v", "--verbose", - group=options.groups["secondary"], + group=options.groups["reporting"], action="store_true", help="Logs the output of all commands run including commands marked silent.", noxfile=True, @@ -283,7 +303,7 @@ def _session_completer( "add_timestamp", "-ts", "--add-timestamp", - group=options.groups["secondary"], + group=options.groups["reporting"], action="store_true", help="Adds a timestamp to logged output.", noxfile=True, @@ -292,7 +312,7 @@ def _session_completer( "default_venv_backend", "-db", "--default-venv-backend", - group=options.groups["secondary"], + group=options.groups["environment"], noxfile=True, merge_func=_default_venv_backend_merge_func, help="Virtual environment backend to use by default for nox sessions, this is ``'virtualenv'`` by default but " @@ -303,7 +323,7 @@ def _session_completer( "force_venv_backend", "-fb", "--force-venv-backend", - group=options.groups["secondary"], + group=options.groups["environment"], noxfile=True, merge_func=_force_venv_backend_merge_func, help="Virtual environment backend to force-use for all nox sessions in this run, overriding any other venv " @@ -314,7 +334,7 @@ def _session_completer( _option_set.Option( "no_venv", "--no-venv", - group=options.groups["secondary"], + group=options.groups["environment"], default=False, action="store_true", help="Runs the selected sessions directly on the current interpreter, without creating a venv. This is an alias " @@ -324,14 +344,14 @@ def _session_completer( "reuse_existing_virtualenvs", ("-r", "--reuse-existing-virtualenvs"), ("--no-reuse-existing-virtualenvs",), - group=options.groups["secondary"], + group=options.groups["environment"], help="Re-use existing virtualenvs instead of recreating them.", ), _option_set.Option( "R", "-R", default=False, - group=options.groups["secondary"], + group=options.groups["environment"], action="store_true", help=( "Re-use existing virtualenvs and skip package re-installation." @@ -343,7 +363,7 @@ def _session_completer( "noxfile", "-f", "--noxfile", - group=options.groups["secondary"], + group=options.groups["general"], default="noxfile.py", help="Location of the Python file containing nox sessions.", ), @@ -352,14 +372,14 @@ def _session_completer( "--envdir", noxfile=True, merge_func=_envdir_merge_func, - group=options.groups["secondary"], + group=options.groups["environment"], help="Directory where nox will store virtualenvs, this is ``.nox`` by default.", ), _option_set.Option( "extra_pythons", "--extra-pythons", "--extra-python", - group=options.groups["secondary"], + group=options.groups["python"], nargs="*", help="Additionally, run sessions using the given python interpreter versions.", ), @@ -367,7 +387,7 @@ def _session_completer( "force_pythons", "--force-pythons", "--force-python", - group=options.groups["secondary"], + group=options.groups["python"], nargs="*", help=( "Run sessions with the given interpreters instead of those listed in the Noxfile." @@ -379,27 +399,27 @@ def _session_completer( "stop_on_first_error", ("-x", "--stop-on-first-error"), ("--no-stop-on-first-error",), - group=options.groups["secondary"], + group=options.groups["execution"], help="Stop after the first error.", ), *_option_set.make_flag_pair( "error_on_missing_interpreters", ("--error-on-missing-interpreters",), ("--no-error-on-missing-interpreters",), - group=options.groups["secondary"], + group=options.groups["execution"], help="Error instead of skipping sessions if an interpreter can not be located.", ), *_option_set.make_flag_pair( "error_on_external_run", ("--error-on-external-run",), ("--no-error-on-external-run",), - group=options.groups["secondary"], + group=options.groups["execution"], help="Error if run() is used to execute a program that isn't installed in a session's virtualenv.", ), _option_set.Option( "install_only", "--install-only", - group=options.groups["secondary"], + group=options.groups["execution"], action="store_true", help="Skip session.run invocations in the Noxfile.", ), @@ -407,7 +427,7 @@ def _session_completer( "no_install", "--no-install", default=False, - group=options.groups["secondary"], + group=options.groups["execution"], action="store_true", help=( "Skip invocations of session methods for installing packages" @@ -418,14 +438,14 @@ def _session_completer( _option_set.Option( "report", "--report", - group=options.groups["secondary"], + group=options.groups["reporting"], noxfile=True, help="Output a report of all sessions to the given filename.", ), _option_set.Option( "non_interactive", "--non-interactive", - group=options.groups["secondary"], + group=options.groups["execution"], action="store_true", help="Force session.interactive to always be False, even in interactive sessions.", ), @@ -433,7 +453,7 @@ def _session_completer( "nocolor", "--nocolor", "--no-color", - group=options.groups["secondary"], + group=options.groups["reporting"], default=lambda: "NO_COLOR" in os.environ, action="store_true", help="Disable all color output.", @@ -442,7 +462,7 @@ def _session_completer( "forcecolor", "--forcecolor", "--force-color", - group=options.groups["secondary"], + group=options.groups["reporting"], default=False, action="store_true", help="Force color output, even if stdout is not an interactive terminal.", @@ -450,7 +470,7 @@ def _session_completer( _option_set.Option( "color", "--color", - group=options.groups["secondary"], + group=options.groups["reporting"], hidden=True, finalizer_func=_color_finalizer, ), From 2791b70fabe1110117c74f630d1937cfa9e329ff Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sat, 12 Jun 2021 22:29:29 +0200 Subject: [PATCH 63/91] Fix crash on Python 2 when reusing environments (#450) * Remove redundant mock in test for environment reuse This breaks tests comparing a string from pyvenv.cfg to command output, because the latter will be a mock. There is no inherent need for this mock. It was added recently, and probably speculatively or for performance reasons. * Add test case for reusing Python 2 environments * Read base prefix from pyvenv.cfg if present * Remove trailing newline when querying interpreter base prefix --- nox/virtualenv.py | 26 +++++++++++++++++++++++--- tests/test_virtualenv.py | 14 +++++++++++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index ab015268..cc59eebb 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -339,15 +339,35 @@ def _check_reused_environment_type(self) -> bool: def _check_reused_environment_interpreter(self) -> bool: """Check if reused environment interpreter is the same.""" - program = "import sys; print(getattr(sys, 'real_prefix', sys.base_prefix))" - original = nox.command.run( - [self._resolved_interpreter, "-c", program], silent=True, log=False + original = self._read_base_prefix_from_pyvenv_cfg() + program = ( + "import sys; sys.stdout.write(getattr(sys, 'real_prefix', sys.base_prefix))" ) + + if original is None: + output = nox.command.run( + [self._resolved_interpreter, "-c", program], silent=True, log=False + ) + assert isinstance(output, str) + original = output + created = nox.command.run( ["python", "-c", program], silent=True, log=False, paths=self.bin_paths ) + return original == created + def _read_base_prefix_from_pyvenv_cfg(self) -> Optional[str]: + """Return the base-prefix entry from pyvenv.cfg, if present.""" + path = os.path.join(self.location, "pyvenv.cfg") + if os.path.isfile(path): + with open(path) as io: + for line in io: + key, _, value = line.partition("=") + if key.strip() == "base-prefix": + return value.strip() + return None + @property def _resolved_interpreter(self) -> str: """Return the interpreter, appropriately resolved for the platform. diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index ac0fa598..5240b524 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -335,7 +335,6 @@ def test_create(monkeypatch, make_one): dir_.ensure("test.txt") assert dir_.join("test.txt").check() venv.reuse_existing = True - monkeypatch.setattr(nox.virtualenv.nox.command, "run", mock.MagicMock()) venv.create() @@ -443,6 +442,19 @@ def test_create_reuse_oldstyle_virtualenv_environment(make_one): assert reused +def test_create_reuse_python2_environment(make_one): + venv, location = make_one(reuse_existing=True, interpreter="2.7") + + try: + venv.create() + except nox.virtualenv.InterpreterNotFound: + pytest.skip("Requires Python 2.7 installation.") + + reused = not venv.create() + + assert reused + + def test_create_venv_backend(make_one): venv, dir_ = make_one(venv=True) venv.create() From 950015e1ec9013bf7199dc2c3e3563d7218a868b Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sat, 12 Jun 2021 23:10:43 +0200 Subject: [PATCH 64/91] Hide staleness check behind a feature flag (#451) * Hide staleness check behind a feature flag * Set feature flag for staleness check in unit tests * Set feature flag for staleness check in Python 2 test --- nox/virtualenv.py | 3 +++ tests/test_virtualenv.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index cc59eebb..13f3fbc3 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -33,6 +33,7 @@ ["PIP_RESPECT_VIRTUALENV", "PIP_REQUIRE_VIRTUALENV", "__PYVENV_LAUNCHER__"] ) _SYSTEM = platform.system() +_ENABLE_STALENESS_CHECK = "NOX_ENABLE_STALENESS_CHECK" in os.environ class InterpreterNotFound(OSError): @@ -312,6 +313,8 @@ def __init__( def _clean_location(self) -> bool: """Deletes any existing virtual environment""" if os.path.exists(self.location): + if self.reuse_existing and not _ENABLE_STALENESS_CHECK: + return False if ( self.reuse_existing and self._check_reused_environment_type() diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 5240b524..c85c08bc 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -351,6 +351,15 @@ def test_create_reuse_environment(make_one): assert reused +@pytest.fixture +def _enable_staleness_check(monkeypatch): + monkeypatch.setattr("nox.virtualenv._ENABLE_STALENESS_CHECK", True) + + +enable_staleness_check = pytest.mark.usefixtures("_enable_staleness_check") + + +@enable_staleness_check def test_create_reuse_environment_with_different_interpreter(make_one, monkeypatch): venv, location = make_one(reuse_existing=True) venv.create() @@ -367,6 +376,7 @@ def test_create_reuse_environment_with_different_interpreter(make_one, monkeypat assert not location.join("marker").check() +@enable_staleness_check def test_create_reuse_stale_venv_environment(make_one): venv, location = make_one(reuse_existing=True) venv.create() @@ -386,6 +396,7 @@ def test_create_reuse_stale_venv_environment(make_one): assert not reused +@enable_staleness_check def test_create_reuse_stale_virtualenv_environment(make_one): venv, location = make_one(reuse_existing=True, venv=True) venv.create() @@ -410,6 +421,7 @@ def test_create_reuse_stale_virtualenv_environment(make_one): assert not reused +@enable_staleness_check def test_create_reuse_venv_environment(make_one): venv, location = make_one(reuse_existing=True, venv=True) venv.create() @@ -424,6 +436,7 @@ def test_create_reuse_venv_environment(make_one): assert reused +@enable_staleness_check @pytest.mark.skipif(IS_WINDOWS, reason="Avoid 'No pyvenv.cfg file' error on Windows.") def test_create_reuse_oldstyle_virtualenv_environment(make_one): venv, location = make_one(reuse_existing=True) @@ -442,6 +455,7 @@ def test_create_reuse_oldstyle_virtualenv_environment(make_one): assert reused +@enable_staleness_check def test_create_reuse_python2_environment(make_one): venv, location = make_one(reuse_existing=True, interpreter="2.7") From 787dfaa87eb47c590e407acb0b5a63922725c291 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sun, 13 Jun 2021 07:04:18 +0200 Subject: [PATCH 65/91] Release 2021.6.12 (#452) --- CHANGELOG.md | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e6c9d50..8d66f263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 2021.6.12 + +- Fix crash on Python 2 when reusing environments. (#450) +- Hide staleness check behind a feature flag. (#451) +- Group command-line options in `--help` message by function. (#442) +- Avoid polluting tests with a .nox directory. (#445) + ## 2021.6.6 - Add option `--no-install` to skip install commands in reused environments. (#432) diff --git a/setup.py b/setup.py index d3db6370..28d99cd4 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="nox", - version="2021.6.6", + version="2021.6.12", description="Flexible test automation.", long_description=long_description, url="https://nox.thea.codes", From d977535a791e199fcac9e0b807b412b1ea12ebfa Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 20 Jun 2021 13:29:26 -0400 Subject: [PATCH 66/91] chore: Use PEP 517 build system (#456) --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..f6c16894 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel", +] +build-backend = "setuptools.build_meta" From 4ea20511bb6362bd4dd7a35e7943e9c1f67b2d62 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 20 Jun 2021 13:39:31 -0400 Subject: [PATCH 67/91] chore: upgrade to mypy 0.902 (#455) * chore: upgrade to mypy 0.902 Uses the new pyproject.toml configuration and much tighter checking, nearly --strict * fix: adding pragma no cover for static if Co-authored-by: Claudio Jolowicz --- nox/_decorators.py | 6 +++--- nox/_option_set.py | 3 ++- nox/_version.py | 2 +- nox/command.py | 9 +++++++-- nox/popen.py | 6 +++--- nox/sessions.py | 8 +++++--- nox/tasks.py | 2 +- nox/virtualenv.py | 4 +++- noxfile.py | 17 +++++++++-------- pyproject.toml | 23 +++++++++++++++++++++++ setup.py | 1 + 11 files changed, 58 insertions(+), 23 deletions(-) diff --git a/nox/_decorators.py b/nox/_decorators.py index 153e331b..f94cef9a 100644 --- a/nox/_decorators.py +++ b/nox/_decorators.py @@ -18,7 +18,7 @@ def __new__( return cast("FunctionDecorator", functools.wraps(func)(obj)) -def _copy_func(src: Callable, name: str = None) -> Callable: +def _copy_func(src: Callable, name: Optional[str] = None) -> Callable: dst = types.FunctionType( src.__code__, src.__globals__, # type: ignore @@ -41,7 +41,7 @@ def __init__( name: Optional[str] = None, venv_backend: Any = None, venv_params: Any = None, - should_warn: Dict[str, Any] = None, + should_warn: Optional[Dict[str, Any]] = None, ): self.func = func self.python = python @@ -53,7 +53,7 @@ def __init__( def __call__(self, *args: Any, **kwargs: Any) -> Any: return self.func(*args, **kwargs) - def copy(self, name: str = None) -> "Func": + def copy(self, name: Optional[str] = None) -> "Func": return Func( _copy_func(self.func, name), self.python, diff --git a/nox/_option_set.py b/nox/_option_set.py index 4bcbb357..0da46fb0 100644 --- a/nox/_option_set.py +++ b/nox/_option_set.py @@ -20,7 +20,8 @@ import argparse import collections import functools -from argparse import ArgumentError, ArgumentParser, Namespace +from argparse import ArgumentError as ArgumentError +from argparse import ArgumentParser, Namespace from typing import Any, Callable, List, Optional, Tuple, Union import argcomplete diff --git a/nox/_version.py b/nox/_version.py index 740b7fd3..2e247534 100644 --- a/nox/_version.py +++ b/nox/_version.py @@ -36,7 +36,7 @@ class InvalidVersionSpecifier(Exception): def get_nox_version() -> str: """Return the version of the installed Nox package.""" - return metadata.version("nox") + return metadata.version("nox") # type: ignore def _parse_string_constant(node: ast.AST) -> Optional[str]: # pragma: no cover diff --git a/nox/command.py b/nox/command.py index 8d915054..2c3f1095 100644 --- a/nox/command.py +++ b/nox/command.py @@ -21,11 +21,16 @@ from nox.logger import logger from nox.popen import popen +if sys.version_info < (3, 8): # pragma: no cover + from typing_extensions import Literal +else: # pragma: no cover + from typing import Literal + class CommandFailed(Exception): """Raised when an executed command returns a non-success status code.""" - def __init__(self, reason: str = None) -> None: + def __init__(self, reason: Optional[str] = None) -> None: super(CommandFailed, self).__init__(reason) self.reason = reason @@ -70,7 +75,7 @@ def run( paths: Optional[List[str]] = None, success_codes: Optional[Iterable[int]] = None, log: bool = True, - external: bool = False, + external: Union[Literal["error"], bool] = False, **popen_kws: Any ) -> Union[str, bool]: """Run a command-line program.""" diff --git a/nox/popen.py b/nox/popen.py index 9bd1c115..010dd321 100644 --- a/nox/popen.py +++ b/nox/popen.py @@ -16,7 +16,7 @@ import locale import subprocess import sys -from typing import IO, Mapping, Sequence, Tuple, Union +from typing import IO, Mapping, Optional, Sequence, Tuple, Union def shutdown_process(proc: subprocess.Popen) -> Tuple[bytes, bytes]: @@ -54,9 +54,9 @@ def decode_output(output: bytes) -> str: def popen( args: Sequence[str], - env: Mapping[str, str] = None, + env: Optional[Mapping[str, str]] = None, silent: bool = False, - stdout: Union[int, IO] = None, + stdout: Optional[Union[int, IO]] = None, stderr: Union[int, IO] = subprocess.STDOUT, ) -> Tuple[int, str]: if silent and stdout is not None: diff --git a/nox/sessions.py b/nox/sessions.py index cabaae12..0cdbe78e 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -211,7 +211,7 @@ def _run_func( raise nox.command.CommandFailed() def run( - self, *args: str, env: Mapping[str, str] = None, **kwargs: Any + self, *args: str, env: Optional[Mapping[str, str]] = None, **kwargs: Any ) -> Optional[Any]: """Run a command. @@ -269,7 +269,7 @@ def run( return self._run(*args, env=env, **kwargs) def run_always( - self, *args: str, env: Mapping[str, str] = None, **kwargs: Any + self, *args: str, env: Optional[Mapping[str, str]] = None, **kwargs: Any ) -> Optional[Any]: """Run a command **always**. @@ -311,7 +311,9 @@ def run_always( return self._run(*args, env=env, **kwargs) - def _run(self, *args: str, env: Mapping[str, str] = None, **kwargs: Any) -> Any: + def _run( + self, *args: str, env: Optional[Mapping[str, str]] = None, **kwargs: Any + ) -> Any: """Like run(), except that it runs even if --install-only is provided.""" # Legacy support - run a function given. if callable(args[0]): diff --git a/nox/tasks.py b/nox/tasks.py index c23b118d..60b5f9a6 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -63,7 +63,7 @@ def load_nox_module(global_config: Namespace) -> Union[types.ModuleType, int]: os.chdir(os.path.realpath(os.path.dirname(global_config.noxfile))) return importlib.machinery.SourceFileLoader( "user_nox_module", global_config.noxfile - ).load_module() # type: ignore + ).load_module() except (VersionCheckFailed, InvalidVersionSpecifier) as error: logger.error(str(error)) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 13f3fbc3..a6bf8448 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -53,7 +53,9 @@ class ProcessEnv: # Special programs that aren't included in the environment. allowed_globals = () # type: _typing.ClassVar[Tuple[Any, ...]] - def __init__(self, bin_paths: None = None, env: Mapping[str, str] = None) -> None: + def __init__( + self, bin_paths: None = None, env: Optional[Mapping[str, str]] = None + ) -> None: self._bin_paths = bin_paths self.env = os.environ.copy() self._reused = False diff --git a/noxfile.py b/noxfile.py index 9e247bb2..58967c2c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -86,15 +86,16 @@ def blacken(session): @nox.session(python="3.8") def lint(session): - session.install("flake8==3.9.2", "black==21.5b2", "isort==5.8.0", "mypy==0.812") - session.run( - "mypy", - "--config-file=", - "--disallow-untyped-defs", - "--warn-unused-ignores", - "--ignore-missing-imports", - "nox", + session.install( + "flake8==3.9.2", + "black==21.6b0", + "isort==5.8.0", + "mypy==0.902", + "types-jinja2", + "packaging", + "importlib_metadata", ) + session.run("mypy") files = ["nox", "tests", "noxfile.py", "setup.py"] session.run("black", "--check", *files) session.run("isort", "--check", *files) diff --git a/pyproject.toml b/pyproject.toml index f6c16894..90bcd11d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,3 +4,26 @@ requires = [ "wheel", ] build-backend = "setuptools.build_meta" + + +[tool.mypy] +files = ["nox"] +python_version = "3.6" +warn_unused_configs = true +disallow_any_generics = false +disallow_subclassing_any = false +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_return_any = false +no_implicit_reexport = true +strict_equality = true + +[[tool.mypy.overrides]] +module = [ "argcomplete", "colorlog.*", "py", "tox.*" ] +ignore_missing_imports = true \ No newline at end of file diff --git a/setup.py b/setup.py index 28d99cd4..92e3d6a1 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ "colorlog>=2.6.1,<7.0.0", "packaging>=20.9", "py>=1.4.0,<2.0.0", + "typing_extensions>=3.7.4; python_version < '3.8'", "virtualenv>=14.0.0", "importlib_metadata; python_version < '3.8'", ], From 0ba939a13f9c9c3d019db8662ed0f368199b382d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 20 Jun 2021 13:58:17 -0400 Subject: [PATCH 68/91] chore: use setup.cfg (#457) --- setup.cfg | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 59 +---------------------------------------------------- 2 files changed, 62 insertions(+), 58 deletions(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..5d720982 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,61 @@ +[metadata] +name = nox +version = 2021.6.12 +description = Flexible test automation. +long_description = file: README.rst +long_description_content_type = text/x-rst +url = https://nox.thea.codes +author = Alethea Katherine Flowers +author_email = me@thea.codes +license = Apache-2.0 +license_file = LICENSE +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Console + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Operating System :: MacOS + Operating System :: Microsoft :: Windows + Operating System :: POSIX + Operating System :: Unix + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Topic :: Software Development :: Testing +keywords = testing automation tox +project_urls = + Documentation = https://nox.thea.codes + Source Code = https://github.com/theacodes/nox + Bug Tracker = https://github.com/theacodes/nox/issues + +[options] +packages = + nox +install_requires = + argcomplete>=1.9.4,<2.0 + colorlog>=2.6.1,<7.0.0 + packaging>=20.9 + py>=1.4.0,<2.0.0 + typing_extensions>=3.7.4;python_version < '3.8' + virtualenv>=14.0.0 + importlib_metadata;python_version < '3.8' +python_requires = >=3.6 +include_package_data = True +zip_safe = False + +[options.entry_points] +console_scripts = + nox = nox.__main__:main + tox-to-nox = nox.tox_to_nox:main [tox_to_nox] + +[options.extras_require] +tox_to_nox = + jinja2 + tox + +[options.package_data] +nox = py.typed diff --git a/setup.py b/setup.py index 92e3d6a1..6df7270c 100644 --- a/setup.py +++ b/setup.py @@ -12,63 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from codecs import open - from setuptools import setup -long_description = open("README.rst", "r", encoding="utf-8").read() - -setup( - name="nox", - version="2021.6.12", - description="Flexible test automation.", - long_description=long_description, - url="https://nox.thea.codes", - author="Alethea Katherine Flowers", - author_email="me@thea.codes", - license="Apache Software License", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Topic :: Software Development :: Testing", - "Environment :: Console", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Operating System :: POSIX", - "Operating System :: MacOS", - "Operating System :: Unix", - "Operating System :: Microsoft :: Windows", - ], - keywords="testing automation tox", - packages=["nox"], - package_data={"nox": ["py.typed"]}, - include_package_data=True, - zip_safe=False, - install_requires=[ - "argcomplete>=1.9.4,<2.0", - "colorlog>=2.6.1,<7.0.0", - "packaging>=20.9", - "py>=1.4.0,<2.0.0", - "typing_extensions>=3.7.4; python_version < '3.8'", - "virtualenv>=14.0.0", - "importlib_metadata; python_version < '3.8'", - ], - extras_require={"tox_to_nox": ["jinja2", "tox"]}, - entry_points={ - "console_scripts": [ - "nox=nox.__main__:main", - "tox-to-nox=nox.tox_to_nox:main [tox_to_nox]", - ] - }, - project_urls={ - "Documentation": "https://nox.thea.codes", - "Source Code": "https://github.com/theacodes/nox", - "Bug Tracker": "https://github.com/theacodes/nox/issues", - }, - python_requires=">=3.6", -) +setup() From 6cb03768fe77a06776e0ce535394e7efc6fd1f3a Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 2 Jul 2021 15:30:05 -0400 Subject: [PATCH 69/91] docs: mention more projects that use Nox (#460) --- docs/index.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 0071cbf6..59e51542 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -51,17 +51,23 @@ Projects that use Nox Nox is lucky to have several wonderful projects that use it and provide feedback and contributions. - `Bézier `__ +- `cibuildwheel `__ - `gapic-generator-python `__ - `gdbgui `__ - `Google Assistant SDK `__ - `google-cloud-python `__ - `google-resumable-media-python `__ - `Hydra `__ +- `manylinux `__ - `OmegaConf `__ - `OpenCensus Python `__ -- `packaging.python.org `__ -- `pipx `__ +- `packaging `__ +- `packaging.python.org `__ +- `pip `__ +- `pipx `__ - `Salt `__ +- `Scikit-build `__ +- `Scikit-HEP `__ - `Subpar `__ - `Urllib3 `__ - `Zazo `__ From 5ac156fc44a286d3c147271ae0e157c5bfbe0767 Mon Sep 17 00:00:00 2001 From: Claudio Jolowicz Date: Sun, 4 Jul 2021 18:10:22 +0200 Subject: [PATCH 70/91] Remove setup.py (#458) * Remove setup.py Editable installs for projects with a setup.cfg no longer require a setup.py shim. This requires pip >= 21.1 (2021-04-24). * Remove setup.py from blacken and lint sessions * CI: Use python -m build in deploy job --- .github/workflows/ci.yml | 6 +++--- noxfile.py | 4 ++-- setup.py | 17 ----------------- 3 files changed, 5 insertions(+), 22 deletions(-) delete mode 100644 setup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d9d7b42..f7c33c30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,10 +59,10 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Install setuptools and wheel - run: python -m pip install --upgrade --user setuptools wheel + - name: Install build + run: python -m pip install --user build - name: Build sdist and wheel - run: python setup.py sdist bdist_wheel + run: python -m build - name: Publish distribution PyPI uses: pypa/gh-action-pypi-publish@master with: diff --git a/noxfile.py b/noxfile.py index 58967c2c..6aed0571 100644 --- a/noxfile.py +++ b/noxfile.py @@ -79,7 +79,7 @@ def cover(session): def blacken(session): """Run black code formatter.""" session.install("black==21.5b2", "isort==5.8.0") - files = ["nox", "tests", "noxfile.py", "setup.py"] + files = ["nox", "tests", "noxfile.py"] session.run("black", *files) session.run("isort", *files) @@ -96,7 +96,7 @@ def lint(session): "importlib_metadata", ) session.run("mypy") - files = ["nox", "tests", "noxfile.py", "setup.py"] + files = ["nox", "tests", "noxfile.py"] session.run("black", "--check", *files) session.run("isort", "--check", *files) session.run("flake8", *files) diff --git a/setup.py b/setup.py deleted file mode 100644 index 6df7270c..00000000 --- a/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2016 Alethea Katherine Flowers -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import setup - -setup() From 0d4764e0780275d5ee842a026002b3a5b0930c76 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 4 Jul 2021 17:25:46 +0100 Subject: [PATCH 71/91] Add header from noxfile.py docstring #454 (#459) * Add header from noxfile.py docstring #454 This commit adds the functionality for using the `noxfile.py` module docstring as a header description for the `nox -l` option. The module docstring is now an attribute in `Manifest` which is now populated in `discover_manifest` which is in-turn passed to `honor_list_request` which will print the docstring if it is present, and do nothing if it is not present. I've also added two tests which cover these conditions (existent and non-existent module docstring) and added to an existing parametrized test. * Standardise printed docstring whitespace Co-authored-by: Claudio Jolowicz --- nox/manifest.py | 8 ++++++- nox/tasks.py | 9 ++++++-- noxfile.py | 1 + tests/test_tasks.py | 52 +++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 65 insertions(+), 5 deletions(-) diff --git a/nox/manifest.py b/nox/manifest.py index dbc3304e..4f6912b9 100644 --- a/nox/manifest.py +++ b/nox/manifest.py @@ -55,15 +55,21 @@ class Manifest: session_functions (Mapping[str, function]): The registry of discovered session functions. global_config (.nox.main.GlobalConfig): The global configuration. + module_docstring (Optional[str]): The user noxfile.py docstring. + Defaults to `None`. """ def __init__( - self, session_functions: Mapping[str, "Func"], global_config: argparse.Namespace + self, + session_functions: Mapping[str, "Func"], + global_config: argparse.Namespace, + module_docstring: Optional[str] = None, ) -> None: self._all_sessions = [] # type: List[SessionRunner] self._queue = [] # type: List[SessionRunner] self._consumed = [] # type: List[SessionRunner] self._config = global_config # type: argparse.Namespace + self.module_docstring = module_docstring # type: Optional[str] # Create the sessions based on the provided session functions. for name, func in session_functions.items(): diff --git a/nox/tasks.py b/nox/tasks.py index 60b5f9a6..afb50a8d 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -104,8 +104,11 @@ def discover_manifest( # sorted by decorator call time. functions = registry.get() + # Get the docstring from the noxfile + module_docstring = module.__doc__ + # Return the final dictionary of session functions. - return Manifest(functions, global_config) + return Manifest(functions, global_config, module_docstring) def filter_manifest( @@ -166,7 +169,9 @@ def honor_list_request( return manifest # If the user just asked for a list of sessions, print that - # and be done. + # and any docstring specified in noxfile.py and be done. + if manifest.module_docstring: + print(manifest.module_docstring.strip(), end="\n\n") print("Sessions defined in {noxfile}:\n".format(noxfile=global_config.noxfile)) diff --git a/noxfile.py b/noxfile.py index 6aed0571..36150cd8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + import functools import os import platform diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 9827cfc8..4031724c 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -200,12 +200,21 @@ def test_honor_list_request_noop(): assert return_value is manifest -@pytest.mark.parametrize("description", [None, "bar"]) -def test_honor_list_request(description): +@pytest.mark.parametrize( + "description, module_docstring", + [ + (None, None), + (None, "hello docstring"), + ("Bar", None), + ("Bar", "hello docstring"), + ], +) +def test_honor_list_request(description, module_docstring): config = _options.options.namespace( list_sessions=True, noxfile="noxfile.py", color=False ) manifest = mock.create_autospec(Manifest) + manifest.module_docstring = module_docstring manifest.list_all_sessions.return_value = [ (argparse.Namespace(friendly_name="foo", description=description), True) ] @@ -218,6 +227,7 @@ def test_honor_list_request_skip_and_selected(capsys): list_sessions=True, noxfile="noxfile.py", color=False ) manifest = mock.create_autospec(Manifest) + manifest.module_docstring = None manifest.list_all_sessions.return_value = [ (argparse.Namespace(friendly_name="foo", description=None), True), (argparse.Namespace(friendly_name="bar", description=None), False), @@ -231,6 +241,44 @@ def test_honor_list_request_skip_and_selected(capsys): assert "- bar" in out +def test_honor_list_request_prints_docstring_if_present(capsys): + config = _options.options.namespace( + list_sessions=True, noxfile="noxfile.py", color=False + ) + manifest = mock.create_autospec(Manifest) + manifest.module_docstring = "Hello I'm a docstring" + manifest.list_all_sessions.return_value = [ + (argparse.Namespace(friendly_name="foo", description=None), True), + (argparse.Namespace(friendly_name="bar", description=None), False), + ] + + return_value = tasks.honor_list_request(manifest, global_config=config) + assert return_value == 0 + + out = capsys.readouterr().out + + assert "Hello I'm a docstring" in out + + +def test_honor_list_request_doesnt_print_docstring_if_not_present(capsys): + config = _options.options.namespace( + list_sessions=True, noxfile="noxfile.py", color=False + ) + manifest = mock.create_autospec(Manifest) + manifest.module_docstring = None + manifest.list_all_sessions.return_value = [ + (argparse.Namespace(friendly_name="foo", description=None), True), + (argparse.Namespace(friendly_name="bar", description=None), False), + ] + + return_value = tasks.honor_list_request(manifest, global_config=config) + assert return_value == 0 + + out = capsys.readouterr().out + + assert "Hello I'm a docstring" not in out + + def test_verify_manifest_empty(): config = _options.options.namespace(sessions=(), keywords=()) manifest = Manifest({}, config) From 98e8c24cca6626b56937ee2614f0435c7f62ffee Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 8 Jul 2021 04:19:22 -0400 Subject: [PATCH 72/91] ci: simpler build (#461) --- .github/workflows/ci.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7c33c30..a73d7b30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,14 +55,8 @@ jobs: if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install build - run: python -m pip install --user build - name: Build sdist and wheel - run: python -m build + run: pipx run build - name: Publish distribution PyPI uses: pypa/gh-action-pypi-publish@master with: From 8ee3ebed052ed7eeef0151a424929039dd479b94 Mon Sep 17 00:00:00 2001 From: Tom Date: Sat, 10 Jul 2021 10:40:05 +0100 Subject: [PATCH 73/91] Add friendlier message if no noxfile.py (#463) * Add friendlier message if no noxfile.py Fixes #462. Add a friendlier message on the specific case that user is calling nox from within a directory with no noxfile. Existing test for this case modified and two additional tests added to ensure lower level errors are still handled further down. * Standardise no noxfile found message for noxfile with different names Co-authored-by: Claudio Jolowicz --- nox/tasks.py | 8 +++++++- tests/test_tasks.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/nox/tasks.py b/nox/tasks.py index afb50a8d..ad11b5c6 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -52,6 +52,7 @@ def load_nox_module(global_config: Namespace) -> Union[types.ModuleType, int]: # Be sure to expand variables os.path.expandvars(global_config.noxfile) ) + noxfile_parent_dir = os.path.realpath(os.path.dirname(global_config.noxfile)) # Check ``nox.needs_version`` by parsing the AST. check_nox_version(global_config.noxfile) @@ -60,7 +61,7 @@ def load_nox_module(global_config: Namespace) -> Union[types.ModuleType, int]: # This will ensure that the Noxfile's path is on sys.path, and that # import-time path resolutions work the way the Noxfile author would # guess. - os.chdir(os.path.realpath(os.path.dirname(global_config.noxfile))) + os.chdir(noxfile_parent_dir) return importlib.machinery.SourceFileLoader( "user_nox_module", global_config.noxfile ).load_module() @@ -68,6 +69,11 @@ def load_nox_module(global_config: Namespace) -> Union[types.ModuleType, int]: except (VersionCheckFailed, InvalidVersionSpecifier) as error: logger.error(str(error)) return 2 + except FileNotFoundError: + logger.error( + f"Failed to load Noxfile {global_config.noxfile}, no such file exists." + ) + return 2 except (IOError, OSError): logger.exception("Failed to load Noxfile {}".format(global_config.noxfile)) return 2 diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 4031724c..c8edb8a7 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -18,6 +18,7 @@ import json import os import platform +from pathlib import Path from textwrap import dedent from unittest import mock @@ -76,9 +77,48 @@ def test_load_nox_module_expandvars(): assert noxfile_module.SIGIL == "123" -def test_load_nox_module_not_found(): - config = _options.options.namespace(noxfile="bogus.py") +def test_load_nox_module_not_found(caplog, tmp_path): + bogus_noxfile = tmp_path / "bogus.py" + config = _options.options.namespace(noxfile=str(bogus_noxfile)) + assert tasks.load_nox_module(config) == 2 + assert ( + f"Failed to load Noxfile {bogus_noxfile}, no such file exists." in caplog.text + ) + + +def test_load_nox_module_IOError(caplog): + + # Need to give it a noxfile that exists so load_nox_module can progress + # past FileNotFoundError + # use our own noxfile.py for this + our_noxfile = Path(__file__).parent.parent.joinpath("noxfile.py") + config = _options.options.namespace(noxfile=str(our_noxfile)) + + with mock.patch( + "nox.tasks.importlib.machinery.SourceFileLoader.load_module" + ) as mock_load: + mock_load.side_effect = IOError + + assert tasks.load_nox_module(config) == 2 + assert "Failed to load Noxfile" in caplog.text + + +def test_load_nox_module_OSError(caplog): + + # Need to give it a noxfile that exists so load_nox_module can progress + # past FileNotFoundError + # use our own noxfile.py for this + our_noxfile = Path(__file__).parent.parent.joinpath("noxfile.py") + config = _options.options.namespace(noxfile=str(our_noxfile)) + + with mock.patch( + "nox.tasks.importlib.machinery.SourceFileLoader.load_module" + ) as mock_load: + mock_load.side_effect = OSError + + assert tasks.load_nox_module(config) == 2 + assert "Failed to load Noxfile" in caplog.text @pytest.fixture From d537c5559a29a97216ca934bcfe95f94cb1055db Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 9 Sep 2021 21:16:57 +0100 Subject: [PATCH 74/91] Add python 3.10.0-rc2 to GitHub Actions (#475) * Add python 3.10.0-rc.2 to GitHub Actions * Make separate CI job for 3.10 to avoid conda failure * Fix docs job in CI so that it now executes correctly --- .github/workflows/ci.yml | 22 ++++++++++++++++++++++ noxfile.py | 6 ++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a73d7b30..59cefcd1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,28 @@ jobs: python -m pip install --disable-pip-version-check . - name: Run tests on ${{ matrix.os }} run: nox --non-interactive --session "tests-${{ matrix.python-version }}" -- --full-trace + + build-py310: + name: Build python 3.10 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04, windows-2019] + python-version: ["3.10.0-rc.2"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + # Conda does not support 3.10 yet, hence why it's skipped here + # TODO: Merge the two build jobs when 3.10 is released for conda + - name: Install Nox-under-test + run: | + python -m pip install --disable-pip-version-check . + - name: Run tests on ${{ matrix.os }} + run: nox --non-interactive --session "tests-${{ matrix.python-version }}" -- --full-trace + lint: runs-on: ubuntu-20.04 steps: diff --git a/noxfile.py b/noxfile.py index 36150cd8..6b31e022 100644 --- a/noxfile.py +++ b/noxfile.py @@ -30,7 +30,9 @@ def is_python_version(session, version): return py_version.startswith(version) -@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) +# TODO: When 3.10 is released, change the version below to 3.10 +# this is here so GitHub actions can pick up on the session name +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10.0-rc.2"]) def tests(session): """Run test suite with pytest.""" session.create_tmp() @@ -103,7 +105,7 @@ def lint(session): session.run("flake8", *files) -@nox.session(python="3.7") +@nox.session(python="3.8") def docs(session): """Build the documentation.""" output_dir = os.path.join(session.create_tmp(), "output") From 5f9cffae0649ad62ba791fb036a2cdf3315e0cf7 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 9 Sep 2021 21:25:48 +0100 Subject: [PATCH 75/91] Add session.notify example to docs (#467) --- nox/sessions.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nox/sessions.py b/nox/sessions.py index 0cdbe78e..ef20e832 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -467,6 +467,20 @@ def notify( This method is idempotent; multiple notifications to the same session have no effect. + A common use case is to notify a code coverage analysis session + from a test session:: + + @nox.session + def test(session): + session.run("pytest") + session.notify("coverage") + + @nox.session + def coverage(session): + session.run("coverage") + + Now if you run `nox -s test`, the coverage session will run afterwards. + Args: target (Union[str, Callable]): The session to be notified. This may be specified as the appropriate string (same as used for From 2821216d6a282e760690a4dce3b2dddbbe4df40f Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 9 Sep 2021 21:26:33 +0100 Subject: [PATCH 76/91] Conda logs now respect nox.options.verbose. (#466) Fixes #345 --- nox/virtualenv.py | 5 +++-- tests/test_virtualenv.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index a6bf8448..9bf80927 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -22,6 +22,7 @@ import py +import nox import nox.command from nox.logger import logger @@ -250,7 +251,7 @@ def create(self) -> bool: logger.info( "Creating conda env in {} with {}".format(self.location_name, python_dep) ) - nox.command.run(cmd, silent=True, log=False) + nox.command.run(cmd, silent=True, log=nox.options.verbose or False) return True @@ -473,6 +474,6 @@ def create(self) -> bool: self.location_name, ) ) - nox.command.run(cmd, silent=True, log=False) + nox.command.run(cmd, silent=True, log=nox.options.verbose or False) return True diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index c85c08bc..8fe1c410 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -210,6 +210,23 @@ def test_condaenv_create_interpreter(make_conda): assert dir_.join("bin", "python3.7").check() +@pytest.mark.skipif(not HAS_CONDA, reason="Missing conda command.") +def test_conda_env_create_verbose(make_conda): + venv, dir_ = make_conda() + with mock.patch("nox.virtualenv.nox.command.run") as mock_run: + venv.create() + + args, kwargs = mock_run.call_args + assert kwargs["log"] is False + + nox.options.verbose = True + with mock.patch("nox.virtualenv.nox.command.run") as mock_run: + venv.create() + + args, kwargs = mock_run.call_args + assert kwargs["log"] + + @mock.patch("nox.virtualenv._SYSTEM", new="Windows") def test_condaenv_bin_windows(make_conda): venv, dir_ = make_conda() From 9c696e2bc2e47ec0f2acd322c7e4be941dab0518 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 9 Sep 2021 22:00:04 +0100 Subject: [PATCH 77/91] Run Flynt to convert str.format to f-strings (#464) * chore: Convert old style str.format to f strings Ran `flynt`: https://github.com/ikamensh/flynt against nox to automatically convert `"{}".format` style strings to the newer `f"{}"` style strings. * Fix Lint failure by running black * Improve readibility of some converted strings * Run Flynt string conversion against tests/ * Change remaining str.format calls to f-strings --- nox/_decorators.py | 2 +- nox/_option_set.py | 14 +++++------ nox/_options.py | 4 ++-- nox/_parametrize.py | 4 ++-- nox/command.py | 27 +++++++++------------ nox/manifest.py | 14 +++++------ nox/sessions.py | 38 +++++++++++++----------------- nox/tasks.py | 29 +++++++++-------------- nox/tox_to_nox.py | 2 +- nox/virtualenv.py | 28 ++++++++-------------- noxfile.py | 2 +- tests/resources/noxfile_nested.py | 2 +- tests/resources/noxfile_pythons.py | 2 +- tests/resources/noxfile_spaces.py | 2 +- tests/test_main.py | 8 +++---- tests/test_sessions.py | 2 +- 16 files changed, 75 insertions(+), 105 deletions(-) diff --git a/nox/_decorators.py b/nox/_decorators.py index f94cef9a..cedd1fb9 100644 --- a/nox/_decorators.py +++ b/nox/_decorators.py @@ -68,7 +68,7 @@ def copy(self, name: Optional[str] = None) -> "Func": class Call(Func): def __init__(self, func: Func, param_spec: "Param") -> None: call_spec = param_spec.call_spec - session_signature = "({})".format(param_spec) + session_signature = f"({param_spec})" # Determine the Python interpreter for the session using either @session # or @parametrize. For backwards compatibility, we only use a "python" diff --git a/nox/_option_set.py b/nox/_option_set.py index 0da46fb0..1e81443a 100644 --- a/nox/_option_set.py +++ b/nox/_option_set.py @@ -85,7 +85,7 @@ def __init__( default: Union[Any, Callable[[], Any]] = None, hidden: bool = False, completer: Optional[Callable[..., List[str]]] = None, - **kwargs: Any + **kwargs: Any, ) -> None: self.name = name self.flags = flags @@ -156,14 +156,14 @@ def make_flag_pair( name: str, enable_flags: Union[Tuple[str, str], Tuple[str]], disable_flags: Tuple[str], - **kwargs: Any + **kwargs: Any, ) -> Tuple[Option, Option]: """Returns two options - one to enable a behavior and another to disable it. The positive option is considered to be available to the noxfile, as there isn't much point in doing flag pairs without it. """ - disable_name = "no_{}".format(name) + disable_name = f"no_{name}" kwargs["action"] = "store_true" enable_option = Option( @@ -171,12 +171,10 @@ def make_flag_pair( *enable_flags, noxfile=True, merge_func=functools.partial(flag_pair_merge_func, name, disable_name), - **kwargs + **kwargs, ) - kwargs["help"] = "Disables {} if it is enabled in the Noxfile.".format( - enable_flags[-1] - ) + kwargs["help"] = f"Disables {enable_flags[-1]} if it is enabled in the Noxfile." disable_option = Option(disable_name, *disable_flags, **kwargs) return enable_option, disable_option @@ -285,7 +283,7 @@ def namespace(self, **kwargs: Any) -> argparse.Namespace: # used in tests. for key, value in kwargs.items(): if key not in args: - raise KeyError("{} is not an option.".format(key)) + raise KeyError(f"{key} is not an option.") args[key] = value return argparse.Namespace(**args) diff --git a/nox/_options.py b/nox/_options.py index bf5d044b..62515239 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -197,14 +197,14 @@ def _posargs_finalizer( if "--" not in posargs: unexpected_posargs = posargs raise _option_set.ArgumentError( - None, "Unknown argument(s) '{}'.".format(" ".join(unexpected_posargs)) + None, f"Unknown argument(s) '{' '.join(unexpected_posargs)}'." ) dash_index = posargs.index("--") if dash_index != 0: unexpected_posargs = posargs[0:dash_index] raise _option_set.ArgumentError( - None, "Unknown argument(s) '{}'.".format(" ".join(unexpected_posargs)) + None, f"Unknown argument(s) '{' '.join(unexpected_posargs)}'." ) return posargs[dash_index + 1 :] diff --git a/nox/_parametrize.py b/nox/_parametrize.py index be226e46..82022402 100644 --- a/nox/_parametrize.py +++ b/nox/_parametrize.py @@ -32,7 +32,7 @@ def __init__( self, *args: Any, arg_names: Optional[Sequence[str]] = None, - id: Optional[str] = None + id: Optional[str] = None, ) -> None: self.args = tuple(args) self.id = id @@ -51,7 +51,7 @@ def __str__(self) -> str: return self.id else: call_spec = self.call_spec - args = ["{}={}".format(k, repr(call_spec[k])) for k in call_spec.keys()] + args = [f"{k}={call_spec[k]!r}" for k in call_spec.keys()] return ", ".join(args) __repr__ = __str__ diff --git a/nox/command.py b/nox/command.py index 2c3f1095..5f215096 100644 --- a/nox/command.py +++ b/nox/command.py @@ -50,8 +50,8 @@ def which(program: str, paths: Optional[List[str]]) -> str: if full_path: return full_path.strpath - logger.error("Program {} not found.".format(program)) - raise CommandFailed("Program {} not found".format(program)) + logger.error(f"Program {program} not found.") + raise CommandFailed(f"Program {program} not found") def _clean_env(env: Optional[dict]) -> Optional[dict]: @@ -76,7 +76,7 @@ def run( success_codes: Optional[Iterable[int]] = None, log: bool = True, external: Union[Literal["error"], bool] = False, - **popen_kws: Any + **popen_kws: Any, ) -> Union[str, bool]: """Run a command-line program.""" @@ -84,7 +84,7 @@ def run( success_codes = [0] cmd, args = args[0], args[1:] - full_cmd = "{} {}".format(cmd, " ".join(args)) + full_cmd = f"{cmd} {' '.join(args)}" cmd_path = which(cmd, paths) @@ -97,18 +97,14 @@ def run( if is_external_tool: if external == "error": logger.error( - "Error: {} is not installed into the virtualenv, it is located at {}. " - "Pass external=True into run() to explicitly allow this.".format( - cmd, cmd_path - ) + f"Error: {cmd} is not installed into the virtualenv, it is located at {cmd_path}. " + "Pass external=True into run() to explicitly allow this." ) raise CommandFailed("External program disallowed.") elif external is False: logger.warning( - "Warning: {} is not installed into the virtualenv, it is located at {}. This might cause issues! " - "Pass external=True into run() to silence this message.".format( - cmd, cmd_path - ) + f"Warning: {cmd} is not installed into the virtualenv, it is located at {cmd_path}. This might cause issues! " + "Pass external=True into run() to silence this message." ) env = _clean_env(env) @@ -119,16 +115,15 @@ def run( ) if return_code not in success_codes: + suffix = ":" if silent else "" logger.error( - "Command {} failed with exit code {}{}".format( - full_cmd, return_code, ":" if silent else "" - ) + f"Command {full_cmd} failed with exit code {return_code}{suffix}" ) if silent: sys.stderr.write(output) - raise CommandFailed("Returned code {}".format(return_code)) + raise CommandFailed(f"Returned code {return_code}") if output: logger.output(output) diff --git a/nox/manifest.py b/nox/manifest.py index 4f6912b9..72c3d05b 100644 --- a/nox/manifest.py +++ b/nox/manifest.py @@ -162,7 +162,7 @@ def filter_by_name(self, specified_sessions: Iterable[str]) -> None: if _normalize_arg(session_name) not in all_sessions ] if missing_sessions: - raise KeyError("Sessions not found: {}".format(", ".join(missing_sessions))) + raise KeyError(f"Sessions not found: {', '.join(missing_sessions)}") def filter_by_python_interpreter(self, specified_pythons: Sequence[str]) -> None: """Filter sessions in the queue based on the user-specified @@ -247,7 +247,7 @@ def make_session( if not multi: long_names.append(name) if func.python: - long_names.append("{}-{}".format(name, func.python)) + long_names.append(f"{name}-{func.python}") return [SessionRunner(name, long_names, func, self._config, self)] @@ -258,13 +258,11 @@ def make_session( for call in calls: long_names = [] if not multi: - long_names.append("{}{}".format(name, call.session_signature)) + long_names.append(f"{name}{call.session_signature}") if func.python: - long_names.append( - "{}-{}{}".format(name, func.python, call.session_signature) - ) + long_names.append(f"{name}-{func.python}{call.session_signature}") # Ensure that specifying session-python will run all parameterizations. - long_names.append("{}-{}".format(name, func.python)) + long_names.append(f"{name}-{func.python}") sessions.append(SessionRunner(name, long_names, call, self._config, self)) @@ -318,7 +316,7 @@ def notify( return True # The session was not found in the list of sessions. - raise ValueError("Session {} not found.".format(session)) + raise ValueError(f"Session {session} not found.") class KeywordLocals(collections.abc.Mapping): diff --git a/nox/sessions.py b/nox/sessions.py index ef20e832..93a33dd2 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -63,9 +63,9 @@ def _normalize_path(envdir: str, path: Union[str, bytes]) -> str: logger.warning("The virtualenv name was hashed to avoid being too long.") else: logger.error( - "The virtualenv path {} is too long and will cause issues on " + f"The virtualenv path {full_path} is too long and will cause issues on " "some environments. Use the --envdir path to modify where " - "nox stores virtualenvs.".format(full_path) + "nox stores virtualenvs." ) return full_path @@ -79,7 +79,7 @@ def _dblquote_pkg_install_arg(pkg_req_str: str) -> str: # sanity check: we need an even number of double-quotes if pkg_req_str.count('"') % 2 != 0: raise ValueError( - "ill-formated argument with odd number of quotes: %s" % pkg_req_str + f"ill-formated argument with odd number of quotes: {pkg_req_str}" ) if "<" in pkg_req_str or ">" in pkg_req_str: @@ -89,10 +89,8 @@ def _dblquote_pkg_install_arg(pkg_req_str: str) -> str: else: # need to double-quote string if '"' in pkg_req_str: - raise ValueError( - "Cannot escape requirement string: %s" % pkg_req_str - ) - return '"%s"' % pkg_req_str + raise ValueError(f"Cannot escape requirement string: {pkg_req_str}") + return f'"{pkg_req_str}"' else: # no dangerous char: no need to double-quote string return pkg_req_str @@ -193,7 +191,7 @@ def interactive(self) -> bool: def chdir(self, dir: Union[str, os.PathLike]) -> None: """Change the current working directory.""" - self.log("cd {}".format(dir)) + self.log(f"cd {dir}") os.chdir(dir) cd = chdir @@ -203,11 +201,11 @@ def _run_func( self, func: Callable, args: Iterable[Any], kwargs: Mapping[str, Any] ) -> Any: """Legacy support for running a function through :func`run`.""" - self.log("{}(args={!r}, kwargs={!r})".format(func, args, kwargs)) + self.log(f"{func}(args={args!r}, kwargs={kwargs!r})") try: return func(*args, **kwargs) except Exception as e: - logger.exception("Function {!r} raised {!r}.".format(func, e)) + logger.exception(f"Function {func!r} raised {e!r}.") raise nox.command.CommandFailed() def run( @@ -263,7 +261,7 @@ def run( raise ValueError("At least one argument required to run().") if self._runner.global_config.install_only: - logger.info("Skipping {} run, as --install-only is set.".format(args[0])) + logger.info(f"Skipping {args[0]} run, as --install-only is set.") return None return self._run(*args, env=env, **kwargs) @@ -410,7 +408,7 @@ def conda_install( *prefix_args, *args, external="error", - **kwargs + **kwargs, ) def install(self, *args: str, **kwargs: Any) -> None: @@ -534,7 +532,7 @@ def description(self) -> Optional[str]: def __str__(self) -> str: sigs = ", ".join(self.signatures) - return "Session(name={}, signatures={})".format(self.name, sigs) + return f"Session(name={self.name}, signatures={sigs})" @property def friendly_name(self) -> str: @@ -583,15 +581,13 @@ def _create_venv(self) -> None: ) else: raise ValueError( - "Expected venv_backend one of ('virtualenv', 'conda', 'venv'), but got '{}'.".format( - backend - ) + f"Expected venv_backend one of ('virtualenv', 'conda', 'venv'), but got '{backend}'." ) self.venv.create() def execute(self) -> "Result": - logger.warning("Running session {}".format(self.friendly_name)) + logger.warning(f"Running session {self.friendly_name}") try: # By default, nox should quietly change to the directory where @@ -624,13 +620,11 @@ def execute(self) -> "Result": return Result(self, Status.FAILED) except KeyboardInterrupt: - logger.error("Session {} interrupted.".format(self.friendly_name)) + logger.error(f"Session {self.friendly_name} interrupted.") raise except Exception as exc: - logger.exception( - "Session {} raised exception {!r}".format(self.friendly_name, exc) - ) + logger.exception(f"Session {self.friendly_name} raised exception {exc!r}") return Result(self, Status.FAILED) @@ -669,7 +663,7 @@ def imperfect(self) -> str: return "was successful" status = self.status.name.lower() if self.reason: - return "{}: {}".format(status, self.reason) + return f"{status}: {self.reason}" else: return status diff --git a/nox/tasks.py b/nox/tasks.py index ad11b5c6..7e55100d 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -75,7 +75,7 @@ def load_nox_module(global_config: Namespace) -> Union[types.ModuleType, int]: ) return 2 except (IOError, OSError): - logger.exception("Failed to load Noxfile {}".format(global_config.noxfile)) + logger.exception(f"Failed to load Noxfile {global_config.noxfile}") return 2 @@ -179,7 +179,7 @@ def honor_list_request( if manifest.module_docstring: print(manifest.module_docstring.strip(), end="\n\n") - print("Sessions defined in {noxfile}:\n".format(noxfile=global_config.noxfile)) + print(f"Sessions defined in {global_config.noxfile}:\n") reset = parse_colors("reset") if global_config.color else "" selected_color = parse_colors("cyan") if global_config.color else "" @@ -209,9 +209,7 @@ def honor_list_request( ) print( - "\nsessions marked with {selected_color}*{reset} are selected, sessions marked with {skipped_color}-{reset} are skipped.".format( - selected_color=selected_color, skipped_color=skipped_color, reset=reset - ) + f"\nsessions marked with {selected_color}*{reset} are selected, sessions marked with {skipped_color}-{reset} are skipped." ) return 0 @@ -255,17 +253,14 @@ def run_manifest(manifest: Manifest, global_config: Namespace) -> List[Result]: # possibly raise warnings associated with this session if WARN_PYTHONS_IGNORED in session.func.should_warn: logger.warning( - "Session {} is set to run with venv_backend='none', IGNORING its python={} parametrization. ".format( - session.name, session.func.should_warn[WARN_PYTHONS_IGNORED] - ) + f"Session {session.name} is set to run with venv_backend='none', " + f"IGNORING its python={session.func.should_warn[WARN_PYTHONS_IGNORED]} parametrization. " ) result = session.execute() - result.log( - "Session {name} {status}.".format( - name=session.friendly_name, status=result.imperfect - ) - ) + name = session.friendly_name + status = result.imperfect + result.log(f"Session {name} {status}.") results.append(result) # Sanity check: If we are supposed to stop on the first error case, @@ -296,11 +291,9 @@ def print_summary(results: List[Result], global_config: Namespace) -> List[Resul # human-readable way. logger.warning("Ran multiple sessions:") for result in results: - result.log( - "* {name}: {status}".format( - name=result.session.friendly_name, status=result.status.name.lower() - ) - ) + name = result.session.friendly_name + status = result.status.name.lower() + result.log(f"* {name}: {status}") # Return the results that were sent to this function. return results diff --git a/nox/tox_to_nox.py b/nox/tox_to_nox.py index 222f5fb9..7e121901 100644 --- a/nox/tox_to_nox.py +++ b/nox/tox_to_nox.py @@ -29,7 +29,7 @@ def wrapjoin(seq: Iterator[Any]) -> str: - return ", ".join(["'{}'".format(item) for item in seq]) + return ", ".join([f"'{item}'" for item in seq]) def main() -> None: diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 9bf80927..93b700ef 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -39,7 +39,7 @@ class InterpreterNotFound(OSError): def __init__(self, interpreter: str) -> None: - super().__init__("Python interpreter {} not found".format(interpreter)) + super().__init__(f"Python interpreter {interpreter} not found") self.interpreter = interpreter @@ -139,7 +139,7 @@ def locate_using_path_and_version(version: str) -> Optional[str]: path_python = py.path.local.sysfind("python") if path_python: try: - prefix = "{}.".format(version) + prefix = f"{version}." version_string = path_python.sysexec("-c", script).strip() if version_string.startswith(prefix): return str(path_python) @@ -227,9 +227,7 @@ def bin_paths(self) -> List[str]: def create(self) -> bool: """Create the conda env.""" if not self._clean_location(): - logger.debug( - "Re-using existing conda env at {}.".format(self.location_name) - ) + logger.debug(f"Re-using existing conda env at {self.location_name}.") self._reused = True @@ -243,14 +241,12 @@ def create(self) -> bool: cmd.append("pip") if self.interpreter: - python_dep = "python={}".format(self.interpreter) + python_dep = f"python={self.interpreter}" else: python_dep = "python" cmd.append(python_dep) - logger.info( - "Creating conda env in {} with {}".format(self.location_name, python_dep) - ) + logger.info(f"Creating conda env in {self.location_name} with {python_dep}") nox.command.run(cmd, silent=True, log=nox.options.verbose or False) return True @@ -403,7 +399,7 @@ def _resolved_interpreter(self) -> str: match = re.match(r"^(?P\d(\.\d+)?)(\.\d+)?$", self.interpreter) if match: xy_version = match.group("xy_ver") - cleaned_interpreter = "python{}".format(xy_version) + cleaned_interpreter = f"python{xy_version}" # If the cleaned interpreter is on the PATH, go ahead and return it. if py.path.local.sysfind(cleaned_interpreter): @@ -450,9 +446,7 @@ def create(self) -> bool: """Create the virtualenv or venv.""" if not self._clean_location(): logger.debug( - "Re-using existing virtual environment at {}.".format( - self.location_name - ) + f"Re-using existing virtual environment at {self.location_name}." ) self._reused = True @@ -467,12 +461,10 @@ def create(self) -> bool: cmd = [self._resolved_interpreter, "-m", "venv", self.location] cmd.extend(self.venv_params) + resolved_interpreter_name = os.path.basename(self._resolved_interpreter) + logger.info( - "Creating virtual environment ({}) using {} in {}".format( - self.venv_or_virtualenv, - os.path.basename(self._resolved_interpreter), - self.location_name, - ) + f"Creating virtual environment ({self.venv_or_virtualenv}) using {resolved_interpreter_name} in {self.location_name}" ) nox.command.run(cmd, silent=True, log=nox.options.verbose or False) diff --git a/noxfile.py b/noxfile.py index 6b31e022..bda4abed 100644 --- a/noxfile.py +++ b/noxfile.py @@ -49,7 +49,7 @@ def tests(session): ".coveragerc", "--cov-report=", *tests, - env={"COVERAGE_FILE": ".coverage.{}".format(session.python)} + env={"COVERAGE_FILE": f".coverage.{session.python}"}, ) session.notify("cover") diff --git a/tests/resources/noxfile_nested.py b/tests/resources/noxfile_nested.py index b7e70327..7a254acd 100644 --- a/tests/resources/noxfile_nested.py +++ b/tests/resources/noxfile_nested.py @@ -18,4 +18,4 @@ @nox.session(py=False) @nox.parametrize("cheese", ["cheddar", "jack", "brie"]) def snack(unused_session, cheese): - print("Noms, {} so good!".format(cheese)) + print(f"Noms, {cheese} so good!") diff --git a/tests/resources/noxfile_pythons.py b/tests/resources/noxfile_pythons.py index bfba5a08..a9a03d30 100644 --- a/tests/resources/noxfile_pythons.py +++ b/tests/resources/noxfile_pythons.py @@ -4,4 +4,4 @@ @nox.session(python=["3.6"]) @nox.parametrize("cheese", ["cheddar", "jack", "brie"]) def snack(unused_session, cheese): - print("Noms, {} so good!".format(cheese)) + print(f"Noms, {cheese} so good!") diff --git a/tests/resources/noxfile_spaces.py b/tests/resources/noxfile_spaces.py index 8c006f74..f11ca9a4 100644 --- a/tests/resources/noxfile_spaces.py +++ b/tests/resources/noxfile_spaces.py @@ -18,4 +18,4 @@ @nox.session(py=False, name="cheese list") @nox.parametrize("cheese", ["cheddar", "jack", "brie"]) def snack(unused_session, cheese): - print("Noms, {} so good!".format(cheese)) + print(f"Noms, {cheese} so good!") diff --git a/tests/test_main.py b/tests/test_main.py index 15f7b506..0b61e804 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -542,8 +542,8 @@ def generate_noxfile(default_session, default_python, alternate_python): return generate_noxfile -python_current_version = "{}.{}".format(sys.version_info.major, sys.version_info.minor) -python_next_version = "{}.{}".format(sys.version_info.major, sys.version_info.minor + 1) +python_current_version = f"{sys.version_info.major}.{sys.version_info.minor}" +python_next_version = f"{sys.version_info.major}.{sys.version_info.minor + 1}" def test_main_noxfile_options_with_pythons_override( @@ -566,7 +566,7 @@ def test_main_noxfile_options_with_pythons_override( for python_version in [python_current_version, python_next_version]: for session in ["test", "launch_rocket"]: - line = "Running session {}-{}".format(session, python_version) + line = f"Running session {session}-{python_version}" if session == "test" and python_version == python_current_version: assert line in stderr else: @@ -593,7 +593,7 @@ def test_main_noxfile_options_with_sessions_override( for python_version in [python_current_version, python_next_version]: for session in ["test", "launch_rocket"]: - line = "Running session {}-{}".format(session, python_version) + line = f"Running session {session}-{python_version}" if session == "launch_rocket" and python_version == python_current_version: assert line in stderr else: diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 89df734e..4408bec3 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -450,7 +450,7 @@ class SessionNoSlots(nox.sessions.Session): pkg_requirement = passed_arg = "urllib3" elif version_constraint == "yes": pkg_requirement = "urllib3<1.25" - passed_arg = '"%s"' % pkg_requirement + passed_arg = f'"{pkg_requirement}"' elif version_constraint == "already_dbl_quoted": pkg_requirement = passed_arg = '"urllib3<1.25"' else: From 74f87936bd755bbaf698d86893325ebd77384ddb Mon Sep 17 00:00:00 2001 From: Franek Magiera Date: Sat, 11 Sep 2021 23:04:17 +0200 Subject: [PATCH 78/91] Add session.invoked_from (#472) * Add original cwd to session properties * Fix test * Rename original_wd to invoked_from * Hoist invoked_from up to the options parser * Add test case for hidden options * Add test case for groupless options Co-authored-by: Thea Flowers --- nox/_option_set.py | 10 ++++++++-- nox/_options.py | 8 ++++++++ nox/sessions.py | 12 ++++++++++++ nox/tasks.py | 3 ++- tests/test__option_set.py | 23 +++++++++++++++++++++++ tests/test_sessions.py | 17 ++++++++++++++++- 6 files changed, 69 insertions(+), 4 deletions(-) diff --git a/nox/_option_set.py b/nox/_option_set.py index 1e81443a..91dcc153 100644 --- a/nox/_option_set.py +++ b/nox/_option_set.py @@ -77,7 +77,7 @@ def __init__( self, name: str, *flags: str, - group: OptionGroup, + group: Optional[OptionGroup], help: Optional[str] = None, noxfile: bool = False, merge_func: Optional[Callable[[Namespace, Namespace], Any]] = None, @@ -230,9 +230,15 @@ def parser(self) -> ArgumentParser: } for option in self.options.values(): - if option.hidden: + if option.hidden is True: continue + # Every option must have a group (except for hidden options) + if option.group is None: + raise ValueError( + f"Option {option.name} must either have a group or be hidden." + ) + argument = groups[option.group.name].add_argument( *option.flags, help=option.help, default=option.default, **option.kwargs ) diff --git a/nox/_options.py b/nox/_options.py index 62515239..353ed647 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -474,6 +474,14 @@ def _session_completer( hidden=True, finalizer_func=_color_finalizer, ), + # Stores the original working directory that Nox was invoked from, + # since it could be different from the Noxfile's directory. + _option_set.Option( + "invoked_from", + group=None, + hidden=True, + default=lambda: os.getcwd(), + ), ) diff --git a/nox/sessions.py b/nox/sessions.py index 93a33dd2..c97c7eb2 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -189,6 +189,18 @@ def interactive(self) -> bool: """Returns True if Nox is being run in an interactive session or False otherwise.""" return not self._runner.global_config.non_interactive and sys.stdin.isatty() + @property + def invoked_from(self) -> str: + """The directory that Nox was originally invoked from. + + Since you can use the ``--noxfile / -f`` command-line + argument to run a Noxfile in a location different from your shell's + current working directory, Nox automatically changes the working directory + to the Noxfile's directory before running any sessions. This gives + you the original working directory that Nox was invoked form. + """ + return self._runner.global_config.invoked_from + def chdir(self, dir: Union[str, os.PathLike]) -> None: """Change the current working directory.""" self.log(f"cd {dir}") diff --git a/nox/tasks.py b/nox/tasks.py index 7e55100d..52263321 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -60,7 +60,8 @@ def load_nox_module(global_config: Namespace) -> Union[types.ModuleType, int]: # Move to the path where the Noxfile is. # This will ensure that the Noxfile's path is on sys.path, and that # import-time path resolutions work the way the Noxfile author would - # guess. + # guess. The original working directory (the directory that Nox was + # invoked from) gets stored by the .invoke_from "option" in _options. os.chdir(noxfile_parent_dir) return importlib.machinery.SourceFileLoader( "user_nox_module", global_config.noxfile diff --git a/tests/test__option_set.py b/tests/test__option_set.py index 202473ea..3107a54c 100644 --- a/tests/test__option_set.py +++ b/tests/test__option_set.py @@ -55,6 +55,29 @@ def test_namespace_non_existant_options_with_values(self): with pytest.raises(KeyError): optionset.namespace(non_existant_option="meep") + def test_parser_hidden_option(self): + optionset = _option_set.OptionSet() + optionset.add_options( + _option_set.Option( + "oh_boy_i_am_hidden", hidden=True, group=None, default="meep" + ) + ) + + parser = optionset.parser() + namespace = parser.parse_args([]) + optionset._finalize_args(namespace) + + assert namespace.oh_boy_i_am_hidden == "meep" + + def test_parser_groupless_option(self): + optionset = _option_set.OptionSet() + optionset.add_options( + _option_set.Option("oh_no_i_have_no_group", group=None, default="meep") + ) + + with pytest.raises(ValueError): + optionset.parser() + def test_session_completer(self): parsed_args = _options.options.namespace(sessions=(), keywords=(), posargs=[]) all_nox_sessions = _options._session_completer( diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 4408bec3..f12f02c7 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -67,7 +67,10 @@ def make_session_and_runner(self): signatures=["test"], func=func, global_config=_options.options.namespace( - posargs=[], error_on_external_run=False, install_only=False + posargs=[], + error_on_external_run=False, + install_only=False, + invoked_from=os.getcwd(), ), manifest=mock.create_autospec(nox.manifest.Manifest), ) @@ -104,6 +107,7 @@ def test_properties(self): assert session.bin_paths is runner.venv.bin_paths assert session.bin is runner.venv.bin_paths[0] assert session.python is runner.func.python + assert session.invoked_from is runner.global_config.invoked_from def test_no_bin_paths(self): session, runner = self.make_session_and_runner() @@ -155,6 +159,17 @@ def test_chdir(self, tmpdir): assert os.getcwd() == cdto os.chdir(current_cwd) + def test_invoked_from(self, tmpdir): + cdto = str(tmpdir.join("cdbby").ensure(dir=True)) + current_cwd = os.getcwd() + + session, _ = self.make_session_and_runner() + + session.chdir(cdto) + + assert session.invoked_from == current_cwd + os.chdir(current_cwd) + def test_chdir_pathlib(self, tmpdir): cdto = str(tmpdir.join("cdbby").ensure(dir=True)) current_cwd = os.getcwd() From 9ebae0ac400a3542a8e4e5ac1a6746e9f08419a8 Mon Sep 17 00:00:00 2001 From: Diego Ramirez Date: Wed, 15 Sep 2021 08:50:44 -0500 Subject: [PATCH 79/91] Add a "shared cache" directory (#476) * Add a "shared cache" directory Add an "{envdir}/.shared" directory, and create an API for accesing to it. * Don't fail if the ".shared" directory exists Don't raise an exception. * Update sessions.py Move the "shared cache" API inside of nox.sessions.Session * Update sessions.py Convert the "shared cache" path into a property. Now, it returns a "pathlib.Path". * Update test_sessions.py Make a test for the recent changes. * Update sessions.py Use the "pathlib.Path" methods to reduce the variable usage. * Update test_sessions.py Make some modifications to the tests. * Update test_sessions.py Fix an import error. * Update sessions.py Use the parent directory to create the cache dir. * Update test_sessions.py Use a tempfile to test the session properties. * Fix test indent * Update nox/sessions.py This avoids some unnecessary path manipulations, and using the parent directory of a virtualenv which does not necessarily exist (`PassthroughEnv` does not create a virtualenv). Co-authored-by: Claudio Jolowicz * Update sessions.py Use ".cache" instead of ".shared". * Update test_sessions.py Use ".cache" instead of ".shared". Co-authored-by: Tom Fleet Co-authored-by: Claudio Jolowicz --- nox/sessions.py | 8 ++++++++ tests/test_sessions.py | 21 +++++++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/nox/sessions.py b/nox/sessions.py index c97c7eb2..5d1b35c0 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -16,6 +16,7 @@ import enum import hashlib import os +import pathlib import re import sys import unicodedata @@ -184,6 +185,13 @@ def create_tmp(self) -> str: self.env["TMPDIR"] = tmpdir return tmpdir + @property + def cache_dir(self) -> pathlib.Path: + """Create and return a 'shared cache' directory to be used across sessions.""" + path = pathlib.Path(self._runner.global_config.envdir).joinpath(".cache") + path.mkdir(exist_ok=True) + return path + @property def interactive(self) -> bool: """Returns True if Nox is being run in an interactive session or False otherwise.""" diff --git a/tests/test_sessions.py b/tests/test_sessions.py index f12f02c7..3c949e64 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -99,15 +99,20 @@ def test_create_tmp_twice(self): def test_properties(self): session, runner = self.make_session_and_runner() + with tempfile.TemporaryDirectory() as root: + runner.global_config.envdir = root - assert session.name is runner.friendly_name - assert session.env is runner.venv.env - assert session.posargs == runner.global_config.posargs - assert session.virtualenv is runner.venv - assert session.bin_paths is runner.venv.bin_paths - assert session.bin is runner.venv.bin_paths[0] - assert session.python is runner.func.python - assert session.invoked_from is runner.global_config.invoked_from + assert session.name is runner.friendly_name + assert session.env is runner.venv.env + assert session.posargs == runner.global_config.posargs + assert session.virtualenv is runner.venv + assert session.bin_paths is runner.venv.bin_paths + assert session.bin is runner.venv.bin_paths[0] + assert session.python is runner.func.python + assert session.invoked_from is runner.global_config.invoked_from + assert session.cache_dir == Path(runner.global_config.envdir).joinpath( + ".cache" + ) def test_no_bin_paths(self): session, runner = self.make_session_and_runner() From f584aed846b19fbf4cdad075c6f4f552839ae5e8 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 16 Sep 2021 08:55:55 +0100 Subject: [PATCH 80/91] CI: Python 3.10.0-rc.2 coverage fix (#479) * Hopeful python3.10.0-rc.2 CI fix * Lets get the correct syntax this time * Lower coverage requirement if 3.10 * Fix failing test coupled to now modified noxfile.py * Explicitly run cover session in GHA * Remove parametrisation in favour of explicit sys.version_info check * Change CI workflow now cover is no longer parametrised * Remove redundant cover session in GHA --- .github/workflows/ci.yml | 5 ++--- noxfile.py | 15 +++++++++++---- tests/test__option_set.py | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59cefcd1..481f6aca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,20 +30,19 @@ jobs: strategy: matrix: os: [ubuntu-20.04, windows-2019] - python-version: ["3.10.0-rc.2"] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: "3.10.0-rc.2" # Conda does not support 3.10 yet, hence why it's skipped here # TODO: Merge the two build jobs when 3.10 is released for conda - name: Install Nox-under-test run: | python -m pip install --disable-pip-version-check . - name: Run tests on ${{ matrix.os }} - run: nox --non-interactive --session "tests-${{ matrix.python-version }}" -- --full-trace + run: nox --non-interactive --session "tests-3.10" -- --full-trace lint: runs-on: ubuntu-20.04 diff --git a/noxfile.py b/noxfile.py index bda4abed..c269ee79 100644 --- a/noxfile.py +++ b/noxfile.py @@ -16,6 +16,7 @@ import functools import os import platform +import sys import nox @@ -30,9 +31,7 @@ def is_python_version(session, version): return py_version.startswith(version) -# TODO: When 3.10 is released, change the version below to 3.10 -# this is here so GitHub actions can pick up on the session name -@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10.0-rc.2"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) def tests(session): """Run test suite with pytest.""" session.create_tmp() @@ -54,6 +53,7 @@ def tests(session): session.notify("cover") +# TODO: When conda supports 3.10 on GHA, add here too @nox.session(python=["3.6", "3.7", "3.8", "3.9"], venv_backend="conda") def conda_tests(session): """Run test suite with pytest.""" @@ -72,9 +72,16 @@ def cover(session): if ON_WINDOWS_CI: return + # 3.10 produces different coverage results for some reason + # see https://github.com/theacodes/nox/issues/478 + fail_under = 100 + py_version = sys.version_info + if py_version.major == 3 and py_version.minor == 10: + fail_under = 99 + session.install("coverage") session.run("coverage", "combine") - session.run("coverage", "report", "--fail-under=100", "--show-missing") + session.run("coverage", "report", f"--fail-under={fail_under}", "--show-missing") session.run("coverage", "erase") diff --git a/tests/test__option_set.py b/tests/test__option_set.py index 3107a54c..f8f14211 100644 --- a/tests/test__option_set.py +++ b/tests/test__option_set.py @@ -85,7 +85,7 @@ def test_session_completer(self): ) # if noxfile.py changes, this will have to change as well since these are # some of the actual sessions found in noxfile.py - some_expected_sessions = ["cover", "blacken", "lint", "docs"] + some_expected_sessions = ["blacken", "lint", "docs"] assert len(set(some_expected_sessions) - set(all_nox_sessions)) == 0 def test_session_completer_invalid_sessions(self): From e7d4f5a299a356ccf8988029796fe296db57536d Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 17 Sep 2021 21:28:25 +0100 Subject: [PATCH 81/91] Decouple `test_session_completer` from project level noxfile (#480) * Decouple test_session_completer from project's root noxfile * Make test variables a bit more intuitive --- tests/resources/noxfile_multiple_sessions.py | 33 ++++++++++++++++++++ tests/test__option_set.py | 19 +++++++---- 2 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 tests/resources/noxfile_multiple_sessions.py diff --git a/tests/resources/noxfile_multiple_sessions.py b/tests/resources/noxfile_multiple_sessions.py new file mode 100644 index 00000000..47a6da75 --- /dev/null +++ b/tests/resources/noxfile_multiple_sessions.py @@ -0,0 +1,33 @@ +# Copyright 2018 Alethea Katherine Flowers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import nox + +# Deliberately giving these silly names so we know this is not confused +# with the projects noxfile + + +@nox.session +def testytest(session): + session.log("Testing") + + +@nox.session +def lintylint(session): + session.log("Linting") + + +@nox.session +def typeytype(session): + session.log("Type Checking") diff --git a/tests/test__option_set.py b/tests/test__option_set.py index f8f14211..99d2d95a 100644 --- a/tests/test__option_set.py +++ b/tests/test__option_set.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path + import pytest from nox import _option_set, _options @@ -20,6 +22,9 @@ # :func:`OptionSet.namespace` needs a bit of help to get to full coverage. +RESOURCES = Path(__file__).parent.joinpath("resources") + + class TestOptionSet: def test_namespace(self): optionset = _option_set.OptionSet() @@ -79,14 +84,16 @@ def test_parser_groupless_option(self): optionset.parser() def test_session_completer(self): - parsed_args = _options.options.namespace(sessions=(), keywords=(), posargs=[]) - all_nox_sessions = _options._session_completer( + parsed_args = _options.options.namespace( + posargs=[], + noxfile=str(RESOURCES.joinpath("noxfile_multiple_sessions.py")), + ) + actual_sessions_from_file = _options._session_completer( prefix=None, parsed_args=parsed_args ) - # if noxfile.py changes, this will have to change as well since these are - # some of the actual sessions found in noxfile.py - some_expected_sessions = ["blacken", "lint", "docs"] - assert len(set(some_expected_sessions) - set(all_nox_sessions)) == 0 + + expected_sessions = ["testytest", "lintylint", "typeytype"] + assert expected_sessions == actual_sessions_from_file def test_session_completer_invalid_sessions(self): parsed_args = _options.options.namespace( From 7f63350ca4c896a77fffeeca40690fb869a8283d Mon Sep 17 00:00:00 2001 From: Diego Ramirez Date: Tue, 21 Sep 2021 13:34:51 -0500 Subject: [PATCH 82/91] Add `session.warn` to output warnings (#482) * Add `session.warn` to output warnings Show warnings during the session. * Update test_sessions.py Add a test for "session.warn" (It is pretty similar to "test_log", but it has changed the logging level to be tested). * Update test_sessions.py Run black to fix linting errors. * Update test_sessions.py Remove an additional whiespace. --- nox/sessions.py | 4 ++++ tests/test_sessions.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/nox/sessions.py b/nox/sessions.py index 5d1b35c0..78a02868 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -516,6 +516,10 @@ def log(self, *args: Any, **kwargs: Any) -> None: """Outputs a log during the session.""" logger.info(*args, **kwargs) + def warn(self, *args: Any, **kwargs: Any) -> None: + """Outputs a warning during the session.""" + logger.warning(*args, **kwargs) + def error(self, *args: Any) -> "_typing.NoReturn": """Immediately aborts the session and optionally logs an error.""" raise _SessionQuit(*args) diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 3c949e64..2d66ba92 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -601,6 +601,14 @@ def test_log(self, caplog): assert "meep" in caplog.text + def test_warn(self, caplog): + caplog.set_level(logging.WARNING) + session, _ = self.make_session_and_runner() + + session.warn("meep") + + assert "meep" in caplog.text + def test_error(self, caplog): caplog.set_level(logging.ERROR) session, _ = self.make_session_and_runner() From e679b77a98fe4b3820a159630e3bccd33b934e4d Mon Sep 17 00:00:00 2001 From: Tom Fleet Date: Sat, 25 Sep 2021 16:03:48 +0100 Subject: [PATCH 83/91] Move configs into pyproject.toml or setup.cfg(flake8) (#484) * Move configs into pyproject.toml or setup.cfg(flake8) * Drop redundant isort multi-line-output setting --- .coveragerc | 10 ---------- .flake8 | 6 ------ .isort.cfg | 2 -- noxfile.py | 4 ++-- pyproject.toml | 15 +++++++++++++++ setup.cfg | 7 +++++++ 6 files changed, 24 insertions(+), 20 deletions(-) delete mode 100644 .coveragerc delete mode 100644 .flake8 delete mode 100644 .isort.cfg diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 92cae43f..00000000 --- a/.coveragerc +++ /dev/null @@ -1,10 +0,0 @@ -[run] -branch = True -omit = - nox/_typing.py - -[report] -exclude_lines = - pragma: no cover - if _typing.TYPE_CHECKING: - @overload diff --git a/.flake8 b/.flake8 deleted file mode 100644 index aa15c81d..00000000 --- a/.flake8 +++ /dev/null @@ -1,6 +0,0 @@ -[flake8] -# Ignore black styles. -ignore = E501, W503, E203 -# Imports -import-order-style = google -application-import-names = nox,tests diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index b9fb3f3e..00000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -profile=black diff --git a/noxfile.py b/noxfile.py index c269ee79..9ffd7405 100644 --- a/noxfile.py +++ b/noxfile.py @@ -45,7 +45,7 @@ def tests(session): "pytest", "--cov=nox", "--cov-config", - ".coveragerc", + "pyproject.toml", "--cov-report=", *tests, env={"COVERAGE_FILE": f".coverage.{session.python}"}, @@ -79,7 +79,7 @@ def cover(session): if py_version.major == 3 and py_version.minor == 10: fail_under = 99 - session.install("coverage") + session.install("coverage[toml]") session.run("coverage", "combine") session.run("coverage", "report", f"--fail-under={fail_under}", "--show-missing") session.run("coverage", "erase") diff --git a/pyproject.toml b/pyproject.toml index 90bcd11d..a163ffe9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,21 @@ requires = [ ] build-backend = "setuptools.build_meta" +[tool.isort] +profile = "black" + +[tool.coverage.run] +branch = true +omit = [ + "nox/_typing.py", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if _typing.TYPE_CHECKING:", + "@overload", +] [tool.mypy] files = ["nox"] diff --git a/setup.cfg b/setup.cfg index 5d720982..f9f05662 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,3 +59,10 @@ tox_to_nox = [options.package_data] nox = py.typed + +[flake8] +# Ignore black styles. +ignore = E501, W503, E203 +# Imports +import-order-style = google +application-import-names = nox,tests From 73d14b11e6f45d3269544811ddcb0000875ce312 Mon Sep 17 00:00:00 2001 From: Thea Flowers Date: Fri, 1 Oct 2021 09:02:43 -0400 Subject: [PATCH 84/91] Release 2021.10.1 --- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ setup.cfg | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d66f263..de973122 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,35 @@ # Changelog +## 2021.10.1 + +New features: +- Add `session.warn` to output warnings (#482) +- Add a shared session cache directory (#476) +- Add `session.invoked_from` (#472) + +Improvements: +- Conda logs now respect `nox.options.verbose` (#466) +- Add `session.notify` example to docs (#467) +- Add friendlier message if no `noxfile.py` is found (#463) +- Show the `noxfile.py` docstring when using `nox -l` (#459) +- Mention more projects that use Nox in the docs (#460) + +Internal changes: +- Move configs into pyproject.toml or setup.cfg (flake8) (#484) +- Decouple `test_session_completer` from project level noxfile (#480) +- Run Flynt to convert str.format to f-strings (#464) +- Add python 3.10.0-rc2 to GitHub Actions (#475, #479) +- Simplify CI build (#461) +- Use PEP 517 build system, remove `setup.py`, use `setup.cfg` (#456, #457, #458) +- Upgrade to mypy 0.902 (#455) + +Special thanks to our contributors: +- @henryiii +- @cjolowicz +- @FollowTheProcess +- @franekmagiera +- @DiddiLeija + ## 2021.6.12 - Fix crash on Python 2 when reusing environments. (#450) diff --git a/setup.cfg b/setup.cfg index f9f05662..2726957b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = nox -version = 2021.6.12 +version = 2021.10.1 description = Flexible test automation. long_description = file: README.rst long_description_content_type = text/x-rst From d95c57cc7891b794b02c6fcb6317e8a71323c223 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Sat, 2 Oct 2021 13:12:39 -0500 Subject: [PATCH 85/91] Using `shlex.join()` when logging a command (#490) * Using `shlex.join()` when logging a command * Backport `shlex.join()` * Fix copy-pasta --- nox/command.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nox/command.py b/nox/command.py index 5f215096..b26f58df 100644 --- a/nox/command.py +++ b/nox/command.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import shlex import sys from typing import Any, Iterable, List, Optional, Sequence, Union @@ -67,6 +68,11 @@ def _clean_env(env: Optional[dict]) -> Optional[dict]: return clean_env +def _shlex_join(args: Sequence[str]) -> str: + # shlex.join() was added in Python 3.8 + return " ".join(shlex.quote(arg) for arg in args) + + def run( args: Sequence[str], *, @@ -84,7 +90,7 @@ def run( success_codes = [0] cmd, args = args[0], args[1:] - full_cmd = f"{cmd} {' '.join(args)}" + full_cmd = f"{cmd} {_shlex_join(args)}" cmd_path = which(cmd, paths) From a2bd140ed9702428d3fc93be9857879388f21ad0 Mon Sep 17 00:00:00 2001 From: Diego Ramirez Date: Tue, 5 Oct 2021 00:09:13 -0500 Subject: [PATCH 86/91] Add `session.debug` to show debug-level messages (#489) * Add `session.debug` to show debug-level messages A logging function to show debug-level messages. * Add `session.debug` to show debug-level messages Add a coverage test. --- nox/sessions.py | 4 ++++ tests/test_sessions.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/nox/sessions.py b/nox/sessions.py index 78a02868..08c9cb42 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -520,6 +520,10 @@ def warn(self, *args: Any, **kwargs: Any) -> None: """Outputs a warning during the session.""" logger.warning(*args, **kwargs) + def debug(self, *args: Any, **kwargs: Any) -> None: + """Outputs a debug-level message during the session.""" + logger.debug(*args, **kwargs) + def error(self, *args: Any) -> "_typing.NoReturn": """Immediately aborts the session and optionally logs an error.""" raise _SessionQuit(*args) diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 2d66ba92..6bcd916d 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -609,6 +609,14 @@ def test_warn(self, caplog): assert "meep" in caplog.text + def test_debug(self, caplog): + caplog.set_level(logging.DEBUG) + session, _ = self.make_session_and_runner() + + session.debug("meep") + + assert "meep" in caplog.text + def test_error(self, caplog): caplog.set_level(logging.ERROR) session, _ = self.make_session_and_runner() From f1b01e2e50141ea3f172d6e76439484715ef0200 Mon Sep 17 00:00:00 2001 From: Tom Fleet Date: Wed, 6 Oct 2021 13:01:06 +0100 Subject: [PATCH 87/91] Add python 3.10.0 to GitHub actions and a 3.10 classifier setup.cfg (#495) --- .github/workflows/ci.yml | 2 +- setup.cfg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 481f6aca..a9ed9ba8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: "3.10.0-rc.2" + python-version: "3.10" # Conda does not support 3.10 yet, hence why it's skipped here # TODO: Merge the two build jobs when 3.10 is released for conda - name: Install Nox-under-test diff --git a/setup.cfg b/setup.cfg index 2726957b..76cbc883 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ classifiers = Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 Topic :: Software Development :: Testing keywords = testing automation tox project_urls = From 536e9d1c1bf0788f8f21ad61bd90c6132dfde1f1 Mon Sep 17 00:00:00 2001 From: Nick Watts <1156625+nawatts@users.noreply.github.com> Date: Wed, 6 Oct 2021 08:13:58 -0400 Subject: [PATCH 88/91] Show error message when keywords expression contains a syntax error (#493) --- nox/tasks.py | 13 +++++++++++-- tests/test_tasks.py | 9 +++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/nox/tasks.py b/nox/tasks.py index 52263321..918f3f18 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ast import importlib.machinery import io import json @@ -150,9 +151,17 @@ def filter_manifest( manifest.filter_by_python_interpreter(global_config.pythons) # Filter by keywords. - # This function never errors, but may cause an empty list of sessions - # (which is an error condition later). if global_config.keywords: + try: + ast.parse(global_config.keywords, mode="eval") + except SyntaxError: + logger.error( + "Error while collecting sessions: keywords argument must be a Python expression." + ) + return 3 + + # This function never errors, but may cause an empty list of sessions + # (which is an error condition later). manifest.filter_by_keywords(global_config.keywords) # Return the modified manifest. diff --git a/tests/test_tasks.py b/tests/test_tasks.py index c8edb8a7..6ce4e503 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -233,6 +233,15 @@ def test_filter_manifest_keywords(): assert len(manifest) == 2 +def test_filter_manifest_keywords_syntax_error(): + config = _options.options.namespace( + sessions=(), pythons=(), keywords="foo:bar", posargs=[] + ) + manifest = Manifest({"foo_bar": session_func, "foo_baz": session_func}, config) + return_value = tasks.filter_manifest(manifest, config) + assert return_value == 3 + + def test_honor_list_request_noop(): config = _options.options.namespace(list_sessions=False) manifest = {"thing": mock.sentinel.THING} From 39b8dd436a8b129ee7a74dab85b715273325fbd0 Mon Sep 17 00:00:00 2001 From: Diego Ramirez Date: Wed, 6 Oct 2021 23:01:14 -0500 Subject: [PATCH 89/91] Bump the ReadTheDocs Python version to 3.8 (#496) Python 3.9 is not supported yet, and 3.6 could reach its EOL soon. --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 85dfcbd3..97ca7a58 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,7 +4,7 @@ build: image: latest python: - version: 3.6 + version: 3.8 pip_install: true requirements_file: requirements-test.txt From e9f7f03934843970ee2583827a2237ea5bce40aa Mon Sep 17 00:00:00 2001 From: Tom Fleet Date: Fri, 8 Oct 2021 22:38:54 +0100 Subject: [PATCH 90/91] Replace deprecated `load_module` with `exec_module` (#498) * Replace to be deprecated load_module * Satisfy mypy (sort of) * Add tests to cover new statements * Factor out the loader function and test seperately --- nox/tasks.py | 44 ++++++++++++++++++++++++++++++++++++++++---- tests/test_tasks.py | 30 ++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/nox/tasks.py b/nox/tasks.py index 918f3f18..e237e4d3 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -13,10 +13,11 @@ # limitations under the License. import ast -import importlib.machinery +import importlib.util import io import json import os +import sys import types from argparse import Namespace from typing import List, Union @@ -31,6 +32,42 @@ from nox.sessions import Result +def _load_and_exec_nox_module(global_config: Namespace) -> types.ModuleType: + """ + Loads, executes, then returns the global_config nox module. + + Args: + global_config (Namespace): The global config. + + Raises: + IOError: If the nox module cannot be loaded. This + exception is chosen such that it will be caught + by load_nox_module and logged appropriately. + + Returns: + types.ModuleType: The initialised nox module. + """ + spec = importlib.util.spec_from_file_location( + "user_nox_module", global_config.noxfile + ) + if not spec: + raise IOError(f"Could not get module spec from {global_config.noxfile}") + + module = importlib.util.module_from_spec(spec) + if not module: + raise IOError(f"Noxfile {global_config.noxfile} is not a valid python module.") + + sys.modules["user_nox_module"] = module + + loader = spec.loader + if not loader: # pragma: no cover + raise IOError(f"Could not get module loader for {global_config.noxfile}") + # See https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly + # unsure why mypy doesn't like this + loader.exec_module(module) # type: ignore + return module + + def load_nox_module(global_config: Namespace) -> Union[types.ModuleType, int]: """Load the user's noxfile and return the module object for it. @@ -64,9 +101,8 @@ def load_nox_module(global_config: Namespace) -> Union[types.ModuleType, int]: # guess. The original working directory (the directory that Nox was # invoked from) gets stored by the .invoke_from "option" in _options. os.chdir(noxfile_parent_dir) - return importlib.machinery.SourceFileLoader( - "user_nox_module", global_config.noxfile - ).load_module() + + return _load_and_exec_nox_module(global_config) except (VersionCheckFailed, InvalidVersionSpecifier) as error: logger.error(str(error)) diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 6ce4e503..2df4cfd5 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -95,9 +95,7 @@ def test_load_nox_module_IOError(caplog): our_noxfile = Path(__file__).parent.parent.joinpath("noxfile.py") config = _options.options.namespace(noxfile=str(our_noxfile)) - with mock.patch( - "nox.tasks.importlib.machinery.SourceFileLoader.load_module" - ) as mock_load: + with mock.patch("nox.tasks.importlib.util.module_from_spec") as mock_load: mock_load.side_effect = IOError assert tasks.load_nox_module(config) == 2 @@ -112,15 +110,35 @@ def test_load_nox_module_OSError(caplog): our_noxfile = Path(__file__).parent.parent.joinpath("noxfile.py") config = _options.options.namespace(noxfile=str(our_noxfile)) - with mock.patch( - "nox.tasks.importlib.machinery.SourceFileLoader.load_module" - ) as mock_load: + with mock.patch("nox.tasks.importlib.util.module_from_spec") as mock_load: mock_load.side_effect = OSError assert tasks.load_nox_module(config) == 2 assert "Failed to load Noxfile" in caplog.text +def test_load_nox_module_invalid_spec(): + our_noxfile = Path(__file__).parent.parent.joinpath("noxfile.py") + config = _options.options.namespace(noxfile=str(our_noxfile)) + + with mock.patch("nox.tasks.importlib.util.spec_from_file_location") as mock_spec: + mock_spec.return_value = None + + with pytest.raises(IOError): + tasks._load_and_exec_nox_module(config) + + +def test_load_nox_module_invalid_module(): + our_noxfile = Path(__file__).parent.parent.joinpath("noxfile.py") + config = _options.options.namespace(noxfile=str(our_noxfile)) + + with mock.patch("nox.tasks.importlib.util.module_from_spec") as mock_spec: + mock_spec.return_value = None + + with pytest.raises(IOError): + tasks._load_and_exec_nox_module(config) + + @pytest.fixture def reset_needs_version(): """Do not leak ``nox.needs_version`` between tests.""" From cbd9d2d8ed792ba8ada911e30fb00b300e7db39d Mon Sep 17 00:00:00 2001 From: Diego Ramirez Date: Wed, 13 Oct 2021 13:20:59 -0500 Subject: [PATCH 91/91] Improve the Sphinx config file (#499) * Remove redundancies on the Sphinx config file - Remove a (duplicated?) commentary. - Remove the "u" prefix on strngs, since they are set to Unicode by default. * Run black * Add `docs/conf.py` to the `blacken` session Run black on that file, too. --- docs/conf.py | 194 +++++++++++++++++++++++++-------------------------- noxfile.py | 2 +- 2 files changed, 97 insertions(+), 99 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a7029aaf..421b6fae 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,48 +26,47 @@ # Note: even though nox is installed when the docs are built, there's a # possibility it's installed as a bytecode-compiled binary (.egg). So, # include the source anyway. -sys.path.insert(0, os.path.abspath('..')) -sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath("..")) +sys.path.insert(0, os.path.abspath(".")) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'recommonmark', + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "recommonmark", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: -# source_suffix = ['.rst', '.md'] -source_suffix = ['.rst', '.md'] +source_suffix = [".rst", ".md"] # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'Nox' -copyright = u'2016, Alethea Katherine Flowers' -author = u'Alethea Katherine Flowers' +project = "Nox" +copyright = "2016, Alethea Katherine Flowers" +author = "Alethea Katherine Flowers" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = metadata.version('nox') +version = metadata.version("nox") # The full version, including alpha/beta/rc tags. release = version @@ -80,20 +79,20 @@ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). @@ -101,16 +100,16 @@ # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'witchhazel.WitchHazelStyle' +pygments_style = "witchhazel.WitchHazelStyle" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -120,179 +119,172 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - 'logo': 'alice.png', - 'logo_name': True, - 'description': 'Flexible test automation', - 'github_user': 'theacodes', - 'github_repo': 'nox', - 'github_banner': True, - 'github_button': False, - 'travis_button': False, - 'codecov_button': False, - 'analytics_id': False, # TODO - 'font_family': "'Roboto', Georgia, sans", - 'head_font_family': "'Roboto', Georgia, serif", - 'code_font_family': "'Roboto Mono', 'Consolas', monospace", - 'pre_bg': '#433e56' + "logo": "alice.png", + "logo_name": True, + "description": "Flexible test automation", + "github_user": "theacodes", + "github_repo": "nox", + "github_banner": True, + "github_button": False, + "travis_button": False, + "codecov_button": False, + "analytics_id": False, # TODO + "font_family": "'Roboto', Georgia, sans", + "head_font_family": "'Roboto', Georgia, serif", + "code_font_family": "'Roboto Mono', 'Consolas', monospace", + "pre_bg": "#433e56", } # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. html_sidebars = { - '**': [ - 'about.html', - 'navigation.html', - 'relations.html', - 'searchbox.html', - 'donate.html', + "**": [ + "about.html", + "navigation.html", + "relations.html", + "searchbox.html", + "donate.html", ] } # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'noxdoc' +htmlhelp_basename = "noxdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', + # Latex figure (float) alignment + #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'nox.tex', u'nox Documentation', - u'Alethea Katherine Flowers', 'manual'), + (master_doc, "nox.tex", "nox Documentation", "Alethea Katherine Flowers", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'nox', u'nox Documentation', - [author], 1) -] +man_pages = [(master_doc, "nox", "nox Documentation", [author], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -301,19 +293,25 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'nox', u'nox Documentation', - author, 'nox', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "nox", + "nox Documentation", + author, + "nox", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False diff --git a/noxfile.py b/noxfile.py index 9ffd7405..ec301bd5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -89,7 +89,7 @@ def cover(session): def blacken(session): """Run black code formatter.""" session.install("black==21.5b2", "isort==5.8.0") - files = ["nox", "tests", "noxfile.py"] + files = ["nox", "tests", "noxfile.py", "docs/conf.py"] session.run("black", *files) session.run("isort", *files)