diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml
index 39c494d142cf..c3555bf6228e 100644
--- a/doc/source/_toc.yml
+++ b/doc/source/_toc.yml
@@ -53,6 +53,7 @@ parts:
- file: ray-air/computer-vision
- file: ray-air/examples/serving_guide
- file: ray-air/deployment
+ - file: ray-air/experimental-features
- file: ray-air/examples/index
sections:
- file: ray-air/examples/opt_deepspeed_batch_inference
diff --git a/doc/source/ray-air/experimental-features.rst b/doc/source/ray-air/experimental-features.rst
new file mode 100644
index 000000000000..b8c338fee290
--- /dev/null
+++ b/doc/source/ray-air/experimental-features.rst
@@ -0,0 +1,133 @@
+.. _air-experimental-features:
+
+================================
+Experimental features in Ray AIR
+================================
+
+The Ray Team is testing a number of experimental features in Ray AIR.
+
+During development, the features
+are disabled per default. You can opt-in by setting a
+feature-specific environment variable.
+
+After some time, the Ray Team enables the feature by default to gather
+more feedback from the community. In that case, you can still
+disable the feature using the same environment variable to
+fully revert to the old behavior.
+
+If you run into issues with experimental features,
+`open an issue `_
+on GitHub. The Ray Team considers feedback before removing
+the old implementation and making the new implementation the
+default.
+
+.. note::
+
+ Experimental features can undergo frequent changes,
+ especially on the master branch and the nightly wheels.
+
+.. _air-experimental-new-output:
+
+Context-aware progress reporting
+--------------------------------
+
+.. note::
+
+ This feature is *disabled by default* in Ray 2.5.
+
+ To enable, set the environment variable ``RAY_AIR_NEW_OUTPUT=1``.
+
+A context-aware output engine is available for Ray Train and Ray Tune runs.
+
+This output engine affects how the training progress
+is printed in the console. The output changes depending on the execution
+context: Ray Tune runs will be displayed differently to Ray Train runs.
+
+The features include:
+
+- Ray Train runs report status relevant to the single training run.
+ It does not use the default Ray Tune table layout from previous versions.
+- The table format has been updated.
+- The format of reporting configurations and observed metrics is different from pervious versions.
+- Significant reduction in the default metrics displayed in the console output for runs (e.g., RLlib runs).
+- Decluttered the output to improve readability.
+
+
+This output feature only works for the regular console.
+It is automatically disabled when you use Jupyter Notebooks
+or Ray client.
+
+
+.. _air-experimental-rich:
+
+Rich layout (sticky status)
+---------------------------
+
+.. note::
+
+ This feature is *disabled by default*.
+
+ To enable, set the environment variable ``RAY_AIR_RICH_LAYOUT=1``.
+
+The :ref:`context-aware output engine `
+exposes an advanced layout using the
+`rich `_ library.
+
+The *rich* layout provides a sticky
+status table: The regular console logs are still printed
+as before, but the trial overview table (in Ray Tune) is stuck to the bottom of the
+screen and periodically updated.
+
+This feature is still in development. You can opt-in to try
+it out.
+
+To opt-in, set the ``RAY_AIR_RICH_LAYOUT=1`` environment variable
+and install rich (``pip install rich``).
+
+.. figure:: images/rich-sticky-status.png
+
+
+.. _air-experimental-execution:
+
+Event-based trial execution engine
+----------------------------------
+
+.. note::
+
+ This feature is *enabled by default* starting Ray 2.5.
+
+ To disable, set the environment variable ``TUNE_NEW_EXECUTION=0``.
+
+
+Ray Tune has an updated trial execution engine.
+Since Ray Tune is also the execution backend for
+Ray Train, the updated engine affects both tuning and training runs.
+
+The update is a refactor of the :ref:`TrialRunner `
+which uses a generic Ray actor and future manager instead of
+the previous ``RayTrialExecutor``. This manager exposes an
+interface to react to scheduling and task execution events, which makes
+it easier to maintain and develop.
+
+This is a drop-in replacement of an internal class, and you shouldn't see
+any change to the previous behavior.
+
+However, if you notice any odd behavior, you can opt out of
+the event-based execution engine and see if it resolves your problem.
+
+In that case, please `open an issue `_
+on GitHub, ideally with a reproducible script.
+
+Things to look out for:
+
+- Less trials are running in parallel than before
+- It takes longer to start new trials (or goes much faster)
+- The tuning run finishes, but the script does not exit
+- The end-to-end runtime is much slower than before
+- The CPU load on the head node is high,
+ even though the training jobs don't
+ require many resources or don't run on the head node
+- Any exceptions are raised that indicate an error in starting or
+ stopping trials or the experiment
+
+Note that some edge cases may not be captured in the regression tests. Your feedback is welcome.
diff --git a/doc/source/ray-air/images/rich-sticky-status.png b/doc/source/ray-air/images/rich-sticky-status.png
new file mode 100644
index 000000000000..e054d2ceeb23
Binary files /dev/null and b/doc/source/ray-air/images/rich-sticky-status.png differ
diff --git a/doc/source/ray-air/user-guides.rst b/doc/source/ray-air/user-guides.rst
index 2c5056689160..ee92e2068367 100644
--- a/doc/source/ray-air/user-guides.rst
+++ b/doc/source/ray-air/user-guides.rst
@@ -110,6 +110,11 @@ Please also see the :ref:`Ray Tune environment variables `.
- **RAY_AIR_FULL_TRACEBACKS**: If set to 1, will print full tracebacks for training functions,
including internal code paths. Otherwise, abbreviated tracebacks that only show user code
are printed. Defaults to 0 (disabled).
+- **RAY_AIR_NEW_OUTPUT**: If set to 0, this disables
+ the :ref:`experimental new console output `.
+- **RAY_AIR_RICH_LAYOUT**: If set to 1, this enables
+ the :ref:`stick table layout `
+ (only available for Ray Tune).
.. _air-multi-tenancy:
@@ -125,3 +130,20 @@ If you still want to do this, refer to
the
:ref:`Ray Tune multi-tenancy docs `
for potential pitfalls.
+
+.. _air-experimental-overview:
+
+Experimental features in Ray 2.5+
+---------------------------------
+Starting in Ray 2.5, some experimental
+features are enabled by default.
+
+Experimental features are enabled to allow for feedback
+from users. Every experimental feature can be disabled
+by setting an environment variable. Some features are
+not ready for general testing and can only be *enabled* using an
+environment variable.
+
+Please see the :ref:`experimental features `
+page for more details on the current features and how to enable
+or disable them.
diff --git a/doc/source/tune/api/env.rst b/doc/source/tune/api/env.rst
index df8b9580f5ae..c6846107484e 100644
--- a/doc/source/tune/api/env.rst
+++ b/doc/source/tune/api/env.rst
@@ -21,6 +21,7 @@ These are the environment variables Ray Tune currently considers:
* **TUNE_DISABLE_DATED_SUBDIR**: Ray Tune automatically adds a date string to experiment
directories when the name is not specified explicitly or the trainable isn't passed
as a string. Setting this environment variable to ``1`` disables adding these date strings.
+* **TUNE_NEW_EXECUTION**: Disable :ref:`Ray Tune's new execution engine `.
* **TUNE_DISABLE_STRICT_METRIC_CHECKING**: When you report metrics to Tune via
``session.report()`` and passed a ``metric`` parameter to ``Tuner()``, a scheduler,
or a search algorithm, Tune will error
diff --git a/python/ray/air/config.py b/python/ray/air/config.py
index f8c262021f43..964f5f9afcb8 100644
--- a/python/ray/air/config.py
+++ b/python/ray/air/config.py
@@ -31,6 +31,7 @@
from ray.tune.search.sample import Domain
from ray.tune.stopper import Stopper
from ray.tune.syncer import SyncConfig
+ from ray.tune.experimental.output import AirVerbosity
from ray.tune.utils.log import Verbosity
from ray.tune.execution.placement_groups import PlacementGroupFactory
@@ -726,9 +727,12 @@ class RunConfig:
intermediate experiment progress. Defaults to CLIReporter if
running in command-line, or JupyterNotebookReporter if running in
a Jupyter notebook.
- verbose: 0, 1, 2, or 3. Verbosity mode.
+ verbose: 0, 1, or 2. Verbosity mode.
+ 0 = silent, 1 = default, 2 = verbose. Defaults to 1.
+ If the ``RAY_AIR_NEW_OUTPUT=0`` environment variable is set,
+ uses the old verbosity settings:
0 = silent, 1 = only status updates, 2 = status and brief
- results, 3 = status and detailed results. Defaults to 2.
+ results, 3 = status and detailed results.
log_to_file: Log stdout and stderr to files in
trial directories. If this is `False` (default), no files
are written. If `true`, outputs are written to `trialdir/stdout`
@@ -748,7 +752,7 @@ class RunConfig:
sync_config: Optional["SyncConfig"] = None
checkpoint_config: Optional[CheckpointConfig] = None
progress_reporter: Optional["ProgressReporter"] = None
- verbose: Union[int, "Verbosity"] = 3
+ verbose: Optional[Union[int, "AirVerbosity", "Verbosity"]] = None
log_to_file: Union[bool, str, Tuple[str, str]] = False
# Deprecated
@@ -757,6 +761,7 @@ class RunConfig:
def __post_init__(self):
from ray.tune.syncer import SyncConfig, Syncer
from ray.tune.utils.util import _resolve_storage_path
+ from ray.tune.experimental.output import AirVerbosity, get_air_verbosity
if not self.failure_config:
self.failure_config = FailureConfig()
@@ -822,6 +827,13 @@ def __post_init__(self):
"Must specify a remote `storage_path` to use a custom `syncer`."
)
+ if self.verbose is None:
+ # Default `verbose` value. For new output engine,
+ # this is AirVerbosity.DEFAULT.
+ # For old output engine, this is Verbosity.V3_TRIAL_DETAILS
+ # Todo (krfricke): Currently uses number to pass test_configs::test_repr
+ self.verbose = get_air_verbosity(AirVerbosity.DEFAULT) or 3
+
def __repr__(self):
from ray.tune.syncer import SyncConfig
diff --git a/python/ray/air/constants.py b/python/ray/air/constants.py
index ddde5372d4bf..f4a85ec59d66 100644
--- a/python/ray/air/constants.py
+++ b/python/ray/air/constants.py
@@ -66,4 +66,7 @@
AIR_ENV_VARS = {
COPY_DIRECTORY_CHECKPOINTS_INSTEAD_OF_MOVING_ENV,
DISABLE_LAZY_CHECKPOINTING_ENV,
+ "RAY_AIR_FULL_TRACEBACKS",
+ "RAY_AIR_NEW_OUTPUT",
+ "RAY_AIR_RICH_LAYOUT",
}
diff --git a/python/ray/tune/constants.py b/python/ray/tune/constants.py
index 6d3c84dc4c7f..0b6698aecb26 100644
--- a/python/ray/tune/constants.py
+++ b/python/ray/tune/constants.py
@@ -5,11 +5,11 @@
# NOTE: When adding a new environment variable, please track it in this list.
TUNE_ENV_VARS = {
"RAY_AIR_LOCAL_CACHE_DIR",
- "RAY_AIR_FULL_TRACEBACKS",
"TUNE_DISABLE_AUTO_CALLBACK_LOGGERS",
"TUNE_DISABLE_AUTO_CALLBACK_SYNCER",
"TUNE_DISABLE_AUTO_INIT",
"TUNE_DISABLE_DATED_SUBDIR",
+ "TUNE_NEW_EXECUTION",
"TUNE_DISABLE_STRICT_METRIC_CHECKING",
"TUNE_DISABLE_SIGINT_HANDLER",
"TUNE_FALLBACK_TO_LATEST_CHECKPOINT",
diff --git a/python/ray/tune/experimental/output.py b/python/ray/tune/experimental/output.py
index 3820c9549e41..bd7dcf851e4a 100644
--- a/python/ray/tune/experimental/output.py
+++ b/python/ray/tune/experimental/output.py
@@ -1,5 +1,15 @@
import sys
-from typing import Any, Collection, Dict, Iterable, List, Optional, Tuple, TYPE_CHECKING
+from typing import (
+ Any,
+ Collection,
+ Dict,
+ Iterable,
+ List,
+ Optional,
+ Tuple,
+ Union,
+ TYPE_CHECKING,
+)
import contextlib
import collections
@@ -15,6 +25,8 @@
import textwrap
import time
+from ray.tune.utils.log import Verbosity
+
try:
import rich
import rich.layout
@@ -90,10 +102,21 @@ class AirVerbosity(IntEnum):
IS_NOTEBOOK = ray.widgets.util.in_notebook()
-def get_air_verbosity() -> Optional[AirVerbosity]:
- verbosity = os.environ.get("AIR_VERBOSITY", None)
- if verbosity:
- return AirVerbosity(int(verbosity)) if verbosity else None
+def get_air_verbosity(
+ verbose: Union[int, AirVerbosity, Verbosity]
+) -> Optional[AirVerbosity]:
+ if os.environ.get("RAY_AIR_NEW_OUTPUT", "0") == "0":
+ return None
+
+ if isinstance(verbose, AirVerbosity):
+ return verbose
+
+ verbose_int = verbose if isinstance(verbose, int) else verbose.value
+
+ # Verbosity 2 and 3 both map to AirVerbosity 2
+ verbose_int = min(2, verbose_int)
+
+ return AirVerbosity(verbose_int)
def _get_time_str(start_time: float, current_time: float) -> Tuple[str, str]:
@@ -520,7 +543,7 @@ def _detect_reporter(
mode: Optional[str] = None,
):
# TODO: Add JupyterNotebook and Ray Client case later.
- rich_enabled = "ENABLE_RICH" in os.environ
+ rich_enabled = bool(int(os.environ.get("RAY_AIR_RICH_LAYOUT", "0")))
if num_samples and num_samples > 1:
if rich_enabled:
if not rich:
@@ -530,7 +553,7 @@ def _detect_reporter(
reporter = TuneTerminalReporter(verbosity, num_samples, metric, mode)
else:
if rich_enabled:
- logger.warning("`ENABLE_RICH` is only effective with Tune usecase.")
+ logger.warning("`RAY_AIR_RICH_LAYOUT` is only effective with Tune usecase.")
reporter = TrainReporter(verbosity)
return reporter
diff --git a/python/ray/tune/tests/test_client.py b/python/ray/tune/tests/test_client.py
index 6252574fead1..b8eaf0e26136 100644
--- a/python/ray/tune/tests/test_client.py
+++ b/python/ray/tune/tests/test_client.py
@@ -29,6 +29,17 @@ def start_client_server_2_cpus():
ray.shutdown()
+@pytest.fixture
+def legacy_progress_reporter():
+ old_val = os.environ.get("RAY_AIR_NEW_OUTPUT")
+ os.environ["RAY_AIR_NEW_OUTPUT"] = "0"
+ yield
+ if old_val is None:
+ os.environ.pop("RAY_AIR_NEW_OUTPUT")
+ else:
+ os.environ["RAY_AIR_NEW_OUTPUT"] = old_val
+
+
@pytest.fixture
def start_client_server_4_cpus():
ray.init(num_cpus=4)
@@ -37,49 +48,51 @@ def start_client_server_4_cpus():
ray.shutdown()
-def test_pbt_function(start_client_server_2_cpus):
+def test_pbt_function(legacy_progress_reporter, start_client_server_2_cpus):
assert ray.util.client.ray.is_connected()
from ray.tune.examples.pbt_function import run_tune_pbt
run_tune_pbt()
-def test_optuna_example(start_client_server):
+def test_optuna_example(legacy_progress_reporter, start_client_server):
assert ray.util.client.ray.is_connected()
from ray.tune.examples.optuna_example import run_optuna_tune
run_optuna_tune(smoke_test=True)
-def test_cifar10_pytorch(start_client_server_2_cpus):
+def test_cifar10_pytorch(legacy_progress_reporter, start_client_server_2_cpus):
assert ray.util.client.ray.is_connected()
from ray.tune.examples.cifar10_pytorch import main
main(num_samples=1, max_num_epochs=1, gpus_per_trial=0)
-def test_tune_mnist_keras(start_client_server_4_cpus):
+def test_tune_mnist_keras(legacy_progress_reporter, start_client_server_4_cpus):
assert ray.util.client.ray.is_connected()
from ray.tune.examples.tune_mnist_keras import tune_mnist
tune_mnist(num_training_iterations=5)
-def test_mnist_ptl_mini(start_client_server):
+def test_mnist_ptl_mini(legacy_progress_reporter, start_client_server):
assert ray.util.client.ray.is_connected()
from ray.tune.examples.mnist_ptl_mini import tune_mnist
tune_mnist(num_samples=1, num_epochs=1, gpus_per_trial=0)
-def test_xgboost_example(start_client_server):
+def test_xgboost_example(legacy_progress_reporter, start_client_server):
assert ray.util.client.ray.is_connected()
from ray.tune.examples.xgboost_example import tune_xgboost
tune_xgboost()
-def test_xgboost_dynamic_resources_example(start_client_server):
+def test_xgboost_dynamic_resources_example(
+ legacy_progress_reporter, start_client_server
+):
assert ray.util.client.ray.is_connected()
from ray.tune.examples.xgboost_dynamic_resources_example import tune_xgboost
@@ -87,7 +100,7 @@ def test_xgboost_dynamic_resources_example(start_client_server):
tune_xgboost(use_class_trainable=False)
-def test_mlflow_example(start_client_server):
+def test_mlflow_example(legacy_progress_reporter, start_client_server):
assert ray.util.client.ray.is_connected()
from ray.tune.examples.mlflow_example import tune_with_callback, tune_with_setup
@@ -96,14 +109,14 @@ def test_mlflow_example(start_client_server):
tune_with_setup(mlflow_tracking_uri, finish_fast=True)
-def test_pbt_transformers(start_client_server):
+def test_pbt_transformers(legacy_progress_reporter, start_client_server):
assert ray.util.client.ray.is_connected()
from ray.tune.examples.pbt_transformers.pbt_transformers import tune_transformer
tune_transformer(num_samples=1, gpus_per_trial=0, smoke_test=True)
-def test_jupyter_rich_output(start_client_server_4_cpus):
+def test_jupyter_rich_output(legacy_progress_reporter, start_client_server_4_cpus):
assert ray.util.client.ray.is_connected()
def dummy_objective(config):
diff --git a/python/ray/tune/tests/test_progress_reporter.py b/python/ray/tune/tests/test_progress_reporter.py
index 2691a227e9ca..d1e8a0c93119 100644
--- a/python/ray/tune/tests/test_progress_reporter.py
+++ b/python/ray/tune/tests/test_progress_reporter.py
@@ -325,12 +325,10 @@ def train(config):
# Add "verbose=3)" etc
-@pytest.mark.skipif(
- "AIR_VERBOSITY" in os.environ, reason="console v2 doesn't work with this v1 test."
-)
class ProgressReporterTest(unittest.TestCase):
def setUp(self) -> None:
os.environ["TUNE_MAX_PENDING_TRIALS_PG"] = "auto"
+ os.environ["RAY_AIR_NEW_OUTPUT"] = "0"
def mock_trial(self, status, i):
mock = MagicMock()
@@ -402,7 +400,7 @@ def test(config):
for i in range(3):
tune.report(**test_result)
- analysis = tune.run(test, num_samples=3)
+ analysis = tune.run(test, num_samples=3, verbose=3)
all_trials = analysis.trials
inferred_results = reporter._infer_user_metrics(all_trials)
for metric in inferred_results:
@@ -421,7 +419,7 @@ def report(self, *args, **kwargs):
self._output.append(progress_str)
reporter = TestReporter()
- analysis = tune.run(test, num_samples=3, progress_reporter=reporter)
+ analysis = tune.run(test, num_samples=3, progress_reporter=reporter, verbose=3)
found = {k: False for k in test_result}
for output in reporter._output:
for key in test_result:
@@ -799,7 +797,12 @@ def should_report(self, trials, done=False):
def report(self, trials, done, *sys_info):
pass
- tune.run(lambda config: 2, num_samples=1, progress_reporter=CustomReporter())
+ tune.run(
+ lambda config: 2,
+ num_samples=1,
+ progress_reporter=CustomReporter(),
+ verbose=3,
+ )
def testMaxLen(self):
trials = []
diff --git a/python/ray/tune/tune.py b/python/ray/tune/tune.py
index 4c5c7dd2983a..04398faa9ac8 100644
--- a/python/ray/tune/tune.py
+++ b/python/ray/tune/tune.py
@@ -35,6 +35,7 @@
get_air_verbosity,
_detect_reporter as _detect_air_reporter,
IS_NOTEBOOK,
+ AirVerbosity,
)
from ray.tune.impl.placeholder import create_resolvers_map, inject_placeholders
@@ -256,7 +257,7 @@ def run(
checkpoint_at_end: bool = False,
checkpoint_keep_all_ranks: bool = False,
checkpoint_upload_from_workers: bool = False,
- verbose: Union[int, Verbosity] = Verbosity.V3_TRIAL_DETAILS,
+ verbose: Optional[Union[int, AirVerbosity, Verbosity]] = None,
progress_reporter: Optional[ProgressReporter] = None,
log_to_file: bool = False,
trial_name_creator: Optional[Callable[[Trial], str]] = None,
@@ -394,9 +395,11 @@ def run(
training workers.
checkpoint_upload_from_workers: Whether to upload checkpoint files
directly from distributed training workers.
- verbose: 0, 1, 2, or 3. Verbosity mode.
- 0 = silent, 1 = only status updates, 2 = status and brief trial
- results, 3 = status and detailed trial results. Defaults to 3.
+ verbose: 0, 1, or 2. Verbosity mode.
+ 0 = silent, 1 = default, 2 = verbose. Defaults to 1.
+ If ``RAY_AIR_NEW_OUTPUT=0``, uses the old verbosity settings:
+ 0 = silent, 1 = only status updates, 2 = status and brief
+ results, 3 = status and detailed results.
progress_reporter: Progress reporter for reporting
intermediate experiment progress. Defaults to CLIReporter if
running in command-line, or JupyterNotebookReporter if running in
@@ -529,11 +532,18 @@ class and registered trainables.
DeprecationWarning,
)
+ if verbose is None:
+ # Default `verbose` value. For new output engine, this is AirVerbosity.VERBOSE.
+ # For old output engine, this is Verbosity.V3_TRIAL_DETAILS
+ verbose = get_air_verbosity(AirVerbosity.VERBOSE) or Verbosity.V3_TRIAL_DETAILS
+
if _remote:
- if get_air_verbosity() is not None:
- logger.warning(
- "Ignoring AIR_VERBOSITY setting, "
- "as it doesn't support ray client mode yet."
+ if get_air_verbosity(verbose) is not None:
+ logger.info(
+ "[output] This uses the legacy output and progress reporter, "
+ "as Ray client is not supported by the new engine. "
+ "For more information, see "
+ "https://docs.ray.io/en/master/ray-air/experimental-features.html"
)
remote_run = ray.remote(num_cpus=0)(run)
@@ -586,21 +596,28 @@ class and registered trainables.
"must be one of ['min', 'max']"
)
- air_verbosity = get_air_verbosity()
+ air_verbosity = get_air_verbosity(verbose)
if air_verbosity is not None and IS_NOTEBOOK:
- logger.warning(
- "Ignoring AIR_VERBOSITY setting, "
- "as it doesn't support JupyterNotebook mode yet."
+ logger.info(
+ "[output] This uses the legacy output and progress reporter, "
+ "as Jupyter notebooks are not supported by the new engine, yet. "
+ "For more information, please see "
+ "https://docs.ray.io/en/master/ray-air/experimental-features.html"
)
air_verbosity = None
if air_verbosity is not None:
- logger.warning(
- f"Testing new AIR console output flow with verbosity={air_verbosity}. "
- f"This will also disable the old flow - setting it to 0 now."
+ logger.info(
+ f"[output] This will use the new output engine with verbosity "
+ f"{air_verbosity}. To disable the new output and use the legacy "
+ f"output engine, set the environment variable RAY_AIR_NEW_OUTPUT=0. "
+ f"For more information, please see "
+ f"https://docs.ray.io/en/master/ray-air/experimental-features.html"
)
+ # Disable old output engine
set_verbosity(0)
else:
+ # Use old output engine
set_verbosity(verbose)
config = config or {}
@@ -979,7 +996,7 @@ class and registered trainables.
air_verbosity, search_alg.total_samples, metric=metric, mode=mode
)
- # rich live context manager has to be called encapsulting
+ # rich live context manager has to be called encapsulating
# the while loop. For other kind of reporters, no op.
# `ExitStack` allows us to *conditionally* apply context manager.
with contextlib.ExitStack() as stack:
@@ -1085,7 +1102,7 @@ def run_experiments(
experiments: Union[Experiment, Mapping, Sequence[Union[Experiment, Mapping]]],
scheduler: Optional[TrialScheduler] = None,
server_port: Optional[int] = None,
- verbose: Union[int, Verbosity] = Verbosity.V3_TRIAL_DETAILS,
+ verbose: Optional[Union[int, AirVerbosity, Verbosity]] = None,
progress_reporter: Optional[ProgressReporter] = None,
resume: Union[bool, str] = False,
reuse_actors: Optional[bool] = None,
@@ -1119,11 +1136,18 @@ def run_experiments(
if not trial_executor or isinstance(trial_executor, RayTrialExecutor):
_ray_auto_init(entrypoint="tune.run_experiments(...)")
+ if verbose is None:
+ # Default `verbose` value. For new output engine, this is AirVerbosity.VERBOSE.
+ # For old output engine, this is Verbosity.V3_TRIAL_DETAILS
+ verbose = get_air_verbosity(AirVerbosity.VERBOSE) or Verbosity.V3_TRIAL_DETAILS
+
if _remote:
- if get_air_verbosity() is not None:
- logger.warning(
- "Ignoring AIR_VERBOSITY setting, "
- "as it doesn't support ray client mode yet."
+ if get_air_verbosity(verbose) is not None:
+ logger.info(
+ "[output] This uses the legacy output and progress reporter, "
+ "as Ray client is not supported by the new engine. "
+ "For more information, see "
+ "https://docs.ray.io/en/master/ray-air/experimental-features.html"
)
remote_run = ray.remote(num_cpus=0)(run_experiments)
diff --git a/python/ray/tune/tuner.py b/python/ray/tune/tuner.py
index 1658188b9530..08a1cd2de0a9 100644
--- a/python/ray/tune/tuner.py
+++ b/python/ray/tune/tuner.py
@@ -150,11 +150,15 @@ def __init__(
"""Configure and construct a tune run."""
kwargs = locals().copy()
self._is_ray_client = ray.util.client.ray.is_connected()
- if self._is_ray_client and get_air_verbosity() is not None:
- logger.warning(
- "Ignoring AIR_VERBOSITY setting, "
- "as it doesn't support ray client mode yet."
- )
+ if self._is_ray_client:
+ _run_config = run_config or RunConfig()
+ if get_air_verbosity(_run_config.verbose) is not None:
+ logger.info(
+ "[output] This uses the legacy output and progress reporter, "
+ "as Ray client is not supported by the new engine. "
+ "For more information, see "
+ "https://docs.ray.io/en/master/ray-air/experimental-features.html"
+ )
if _tuner_internal:
if not self._is_ray_client: