diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index 39c494d142cf..c3555bf6228e 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -53,6 +53,7 @@ parts: - file: ray-air/computer-vision - file: ray-air/examples/serving_guide - file: ray-air/deployment + - file: ray-air/experimental-features - file: ray-air/examples/index sections: - file: ray-air/examples/opt_deepspeed_batch_inference diff --git a/doc/source/ray-air/experimental-features.rst b/doc/source/ray-air/experimental-features.rst new file mode 100644 index 000000000000..b8c338fee290 --- /dev/null +++ b/doc/source/ray-air/experimental-features.rst @@ -0,0 +1,133 @@ +.. _air-experimental-features: + +================================ +Experimental features in Ray AIR +================================ + +The Ray Team is testing a number of experimental features in Ray AIR. + +During development, the features +are disabled per default. You can opt-in by setting a +feature-specific environment variable. + +After some time, the Ray Team enables the feature by default to gather +more feedback from the community. In that case, you can still +disable the feature using the same environment variable to +fully revert to the old behavior. + +If you run into issues with experimental features, +`open an issue `_ +on GitHub. The Ray Team considers feedback before removing +the old implementation and making the new implementation the +default. + +.. note:: + + Experimental features can undergo frequent changes, + especially on the master branch and the nightly wheels. + +.. _air-experimental-new-output: + +Context-aware progress reporting +-------------------------------- + +.. note:: + + This feature is *disabled by default* in Ray 2.5. + + To enable, set the environment variable ``RAY_AIR_NEW_OUTPUT=1``. + +A context-aware output engine is available for Ray Train and Ray Tune runs. + +This output engine affects how the training progress +is printed in the console. The output changes depending on the execution +context: Ray Tune runs will be displayed differently to Ray Train runs. + +The features include: + +- Ray Train runs report status relevant to the single training run. + It does not use the default Ray Tune table layout from previous versions. +- The table format has been updated. +- The format of reporting configurations and observed metrics is different from pervious versions. +- Significant reduction in the default metrics displayed in the console output for runs (e.g., RLlib runs). +- Decluttered the output to improve readability. + + +This output feature only works for the regular console. +It is automatically disabled when you use Jupyter Notebooks +or Ray client. + + +.. _air-experimental-rich: + +Rich layout (sticky status) +--------------------------- + +.. note:: + + This feature is *disabled by default*. + + To enable, set the environment variable ``RAY_AIR_RICH_LAYOUT=1``. + +The :ref:`context-aware output engine ` +exposes an advanced layout using the +`rich `_ library. + +The *rich* layout provides a sticky +status table: The regular console logs are still printed +as before, but the trial overview table (in Ray Tune) is stuck to the bottom of the +screen and periodically updated. + +This feature is still in development. You can opt-in to try +it out. + +To opt-in, set the ``RAY_AIR_RICH_LAYOUT=1`` environment variable +and install rich (``pip install rich``). + +.. figure:: images/rich-sticky-status.png + + +.. _air-experimental-execution: + +Event-based trial execution engine +---------------------------------- + +.. note:: + + This feature is *enabled by default* starting Ray 2.5. + + To disable, set the environment variable ``TUNE_NEW_EXECUTION=0``. + + +Ray Tune has an updated trial execution engine. +Since Ray Tune is also the execution backend for +Ray Train, the updated engine affects both tuning and training runs. + +The update is a refactor of the :ref:`TrialRunner ` +which uses a generic Ray actor and future manager instead of +the previous ``RayTrialExecutor``. This manager exposes an +interface to react to scheduling and task execution events, which makes +it easier to maintain and develop. + +This is a drop-in replacement of an internal class, and you shouldn't see +any change to the previous behavior. + +However, if you notice any odd behavior, you can opt out of +the event-based execution engine and see if it resolves your problem. + +In that case, please `open an issue `_ +on GitHub, ideally with a reproducible script. + +Things to look out for: + +- Less trials are running in parallel than before +- It takes longer to start new trials (or goes much faster) +- The tuning run finishes, but the script does not exit +- The end-to-end runtime is much slower than before +- The CPU load on the head node is high, + even though the training jobs don't + require many resources or don't run on the head node +- Any exceptions are raised that indicate an error in starting or + stopping trials or the experiment + +Note that some edge cases may not be captured in the regression tests. Your feedback is welcome. diff --git a/doc/source/ray-air/images/rich-sticky-status.png b/doc/source/ray-air/images/rich-sticky-status.png new file mode 100644 index 000000000000..e054d2ceeb23 Binary files /dev/null and b/doc/source/ray-air/images/rich-sticky-status.png differ diff --git a/doc/source/ray-air/user-guides.rst b/doc/source/ray-air/user-guides.rst index 2c5056689160..ee92e2068367 100644 --- a/doc/source/ray-air/user-guides.rst +++ b/doc/source/ray-air/user-guides.rst @@ -110,6 +110,11 @@ Please also see the :ref:`Ray Tune environment variables `. - **RAY_AIR_FULL_TRACEBACKS**: If set to 1, will print full tracebacks for training functions, including internal code paths. Otherwise, abbreviated tracebacks that only show user code are printed. Defaults to 0 (disabled). +- **RAY_AIR_NEW_OUTPUT**: If set to 0, this disables + the :ref:`experimental new console output `. +- **RAY_AIR_RICH_LAYOUT**: If set to 1, this enables + the :ref:`stick table layout ` + (only available for Ray Tune). .. _air-multi-tenancy: @@ -125,3 +130,20 @@ If you still want to do this, refer to the :ref:`Ray Tune multi-tenancy docs ` for potential pitfalls. + +.. _air-experimental-overview: + +Experimental features in Ray 2.5+ +--------------------------------- +Starting in Ray 2.5, some experimental +features are enabled by default. + +Experimental features are enabled to allow for feedback +from users. Every experimental feature can be disabled +by setting an environment variable. Some features are +not ready for general testing and can only be *enabled* using an +environment variable. + +Please see the :ref:`experimental features ` +page for more details on the current features and how to enable +or disable them. diff --git a/doc/source/tune/api/env.rst b/doc/source/tune/api/env.rst index df8b9580f5ae..c6846107484e 100644 --- a/doc/source/tune/api/env.rst +++ b/doc/source/tune/api/env.rst @@ -21,6 +21,7 @@ These are the environment variables Ray Tune currently considers: * **TUNE_DISABLE_DATED_SUBDIR**: Ray Tune automatically adds a date string to experiment directories when the name is not specified explicitly or the trainable isn't passed as a string. Setting this environment variable to ``1`` disables adding these date strings. +* **TUNE_NEW_EXECUTION**: Disable :ref:`Ray Tune's new execution engine `. * **TUNE_DISABLE_STRICT_METRIC_CHECKING**: When you report metrics to Tune via ``session.report()`` and passed a ``metric`` parameter to ``Tuner()``, a scheduler, or a search algorithm, Tune will error diff --git a/python/ray/air/config.py b/python/ray/air/config.py index f8c262021f43..964f5f9afcb8 100644 --- a/python/ray/air/config.py +++ b/python/ray/air/config.py @@ -31,6 +31,7 @@ from ray.tune.search.sample import Domain from ray.tune.stopper import Stopper from ray.tune.syncer import SyncConfig + from ray.tune.experimental.output import AirVerbosity from ray.tune.utils.log import Verbosity from ray.tune.execution.placement_groups import PlacementGroupFactory @@ -726,9 +727,12 @@ class RunConfig: intermediate experiment progress. Defaults to CLIReporter if running in command-line, or JupyterNotebookReporter if running in a Jupyter notebook. - verbose: 0, 1, 2, or 3. Verbosity mode. + verbose: 0, 1, or 2. Verbosity mode. + 0 = silent, 1 = default, 2 = verbose. Defaults to 1. + If the ``RAY_AIR_NEW_OUTPUT=0`` environment variable is set, + uses the old verbosity settings: 0 = silent, 1 = only status updates, 2 = status and brief - results, 3 = status and detailed results. Defaults to 2. + results, 3 = status and detailed results. log_to_file: Log stdout and stderr to files in trial directories. If this is `False` (default), no files are written. If `true`, outputs are written to `trialdir/stdout` @@ -748,7 +752,7 @@ class RunConfig: sync_config: Optional["SyncConfig"] = None checkpoint_config: Optional[CheckpointConfig] = None progress_reporter: Optional["ProgressReporter"] = None - verbose: Union[int, "Verbosity"] = 3 + verbose: Optional[Union[int, "AirVerbosity", "Verbosity"]] = None log_to_file: Union[bool, str, Tuple[str, str]] = False # Deprecated @@ -757,6 +761,7 @@ class RunConfig: def __post_init__(self): from ray.tune.syncer import SyncConfig, Syncer from ray.tune.utils.util import _resolve_storage_path + from ray.tune.experimental.output import AirVerbosity, get_air_verbosity if not self.failure_config: self.failure_config = FailureConfig() @@ -822,6 +827,13 @@ def __post_init__(self): "Must specify a remote `storage_path` to use a custom `syncer`." ) + if self.verbose is None: + # Default `verbose` value. For new output engine, + # this is AirVerbosity.DEFAULT. + # For old output engine, this is Verbosity.V3_TRIAL_DETAILS + # Todo (krfricke): Currently uses number to pass test_configs::test_repr + self.verbose = get_air_verbosity(AirVerbosity.DEFAULT) or 3 + def __repr__(self): from ray.tune.syncer import SyncConfig diff --git a/python/ray/air/constants.py b/python/ray/air/constants.py index ddde5372d4bf..f4a85ec59d66 100644 --- a/python/ray/air/constants.py +++ b/python/ray/air/constants.py @@ -66,4 +66,7 @@ AIR_ENV_VARS = { COPY_DIRECTORY_CHECKPOINTS_INSTEAD_OF_MOVING_ENV, DISABLE_LAZY_CHECKPOINTING_ENV, + "RAY_AIR_FULL_TRACEBACKS", + "RAY_AIR_NEW_OUTPUT", + "RAY_AIR_RICH_LAYOUT", } diff --git a/python/ray/tune/constants.py b/python/ray/tune/constants.py index 6d3c84dc4c7f..0b6698aecb26 100644 --- a/python/ray/tune/constants.py +++ b/python/ray/tune/constants.py @@ -5,11 +5,11 @@ # NOTE: When adding a new environment variable, please track it in this list. TUNE_ENV_VARS = { "RAY_AIR_LOCAL_CACHE_DIR", - "RAY_AIR_FULL_TRACEBACKS", "TUNE_DISABLE_AUTO_CALLBACK_LOGGERS", "TUNE_DISABLE_AUTO_CALLBACK_SYNCER", "TUNE_DISABLE_AUTO_INIT", "TUNE_DISABLE_DATED_SUBDIR", + "TUNE_NEW_EXECUTION", "TUNE_DISABLE_STRICT_METRIC_CHECKING", "TUNE_DISABLE_SIGINT_HANDLER", "TUNE_FALLBACK_TO_LATEST_CHECKPOINT", diff --git a/python/ray/tune/experimental/output.py b/python/ray/tune/experimental/output.py index 3820c9549e41..bd7dcf851e4a 100644 --- a/python/ray/tune/experimental/output.py +++ b/python/ray/tune/experimental/output.py @@ -1,5 +1,15 @@ import sys -from typing import Any, Collection, Dict, Iterable, List, Optional, Tuple, TYPE_CHECKING +from typing import ( + Any, + Collection, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + TYPE_CHECKING, +) import contextlib import collections @@ -15,6 +25,8 @@ import textwrap import time +from ray.tune.utils.log import Verbosity + try: import rich import rich.layout @@ -90,10 +102,21 @@ class AirVerbosity(IntEnum): IS_NOTEBOOK = ray.widgets.util.in_notebook() -def get_air_verbosity() -> Optional[AirVerbosity]: - verbosity = os.environ.get("AIR_VERBOSITY", None) - if verbosity: - return AirVerbosity(int(verbosity)) if verbosity else None +def get_air_verbosity( + verbose: Union[int, AirVerbosity, Verbosity] +) -> Optional[AirVerbosity]: + if os.environ.get("RAY_AIR_NEW_OUTPUT", "0") == "0": + return None + + if isinstance(verbose, AirVerbosity): + return verbose + + verbose_int = verbose if isinstance(verbose, int) else verbose.value + + # Verbosity 2 and 3 both map to AirVerbosity 2 + verbose_int = min(2, verbose_int) + + return AirVerbosity(verbose_int) def _get_time_str(start_time: float, current_time: float) -> Tuple[str, str]: @@ -520,7 +543,7 @@ def _detect_reporter( mode: Optional[str] = None, ): # TODO: Add JupyterNotebook and Ray Client case later. - rich_enabled = "ENABLE_RICH" in os.environ + rich_enabled = bool(int(os.environ.get("RAY_AIR_RICH_LAYOUT", "0"))) if num_samples and num_samples > 1: if rich_enabled: if not rich: @@ -530,7 +553,7 @@ def _detect_reporter( reporter = TuneTerminalReporter(verbosity, num_samples, metric, mode) else: if rich_enabled: - logger.warning("`ENABLE_RICH` is only effective with Tune usecase.") + logger.warning("`RAY_AIR_RICH_LAYOUT` is only effective with Tune usecase.") reporter = TrainReporter(verbosity) return reporter diff --git a/python/ray/tune/tests/test_client.py b/python/ray/tune/tests/test_client.py index 6252574fead1..b8eaf0e26136 100644 --- a/python/ray/tune/tests/test_client.py +++ b/python/ray/tune/tests/test_client.py @@ -29,6 +29,17 @@ def start_client_server_2_cpus(): ray.shutdown() +@pytest.fixture +def legacy_progress_reporter(): + old_val = os.environ.get("RAY_AIR_NEW_OUTPUT") + os.environ["RAY_AIR_NEW_OUTPUT"] = "0" + yield + if old_val is None: + os.environ.pop("RAY_AIR_NEW_OUTPUT") + else: + os.environ["RAY_AIR_NEW_OUTPUT"] = old_val + + @pytest.fixture def start_client_server_4_cpus(): ray.init(num_cpus=4) @@ -37,49 +48,51 @@ def start_client_server_4_cpus(): ray.shutdown() -def test_pbt_function(start_client_server_2_cpus): +def test_pbt_function(legacy_progress_reporter, start_client_server_2_cpus): assert ray.util.client.ray.is_connected() from ray.tune.examples.pbt_function import run_tune_pbt run_tune_pbt() -def test_optuna_example(start_client_server): +def test_optuna_example(legacy_progress_reporter, start_client_server): assert ray.util.client.ray.is_connected() from ray.tune.examples.optuna_example import run_optuna_tune run_optuna_tune(smoke_test=True) -def test_cifar10_pytorch(start_client_server_2_cpus): +def test_cifar10_pytorch(legacy_progress_reporter, start_client_server_2_cpus): assert ray.util.client.ray.is_connected() from ray.tune.examples.cifar10_pytorch import main main(num_samples=1, max_num_epochs=1, gpus_per_trial=0) -def test_tune_mnist_keras(start_client_server_4_cpus): +def test_tune_mnist_keras(legacy_progress_reporter, start_client_server_4_cpus): assert ray.util.client.ray.is_connected() from ray.tune.examples.tune_mnist_keras import tune_mnist tune_mnist(num_training_iterations=5) -def test_mnist_ptl_mini(start_client_server): +def test_mnist_ptl_mini(legacy_progress_reporter, start_client_server): assert ray.util.client.ray.is_connected() from ray.tune.examples.mnist_ptl_mini import tune_mnist tune_mnist(num_samples=1, num_epochs=1, gpus_per_trial=0) -def test_xgboost_example(start_client_server): +def test_xgboost_example(legacy_progress_reporter, start_client_server): assert ray.util.client.ray.is_connected() from ray.tune.examples.xgboost_example import tune_xgboost tune_xgboost() -def test_xgboost_dynamic_resources_example(start_client_server): +def test_xgboost_dynamic_resources_example( + legacy_progress_reporter, start_client_server +): assert ray.util.client.ray.is_connected() from ray.tune.examples.xgboost_dynamic_resources_example import tune_xgboost @@ -87,7 +100,7 @@ def test_xgboost_dynamic_resources_example(start_client_server): tune_xgboost(use_class_trainable=False) -def test_mlflow_example(start_client_server): +def test_mlflow_example(legacy_progress_reporter, start_client_server): assert ray.util.client.ray.is_connected() from ray.tune.examples.mlflow_example import tune_with_callback, tune_with_setup @@ -96,14 +109,14 @@ def test_mlflow_example(start_client_server): tune_with_setup(mlflow_tracking_uri, finish_fast=True) -def test_pbt_transformers(start_client_server): +def test_pbt_transformers(legacy_progress_reporter, start_client_server): assert ray.util.client.ray.is_connected() from ray.tune.examples.pbt_transformers.pbt_transformers import tune_transformer tune_transformer(num_samples=1, gpus_per_trial=0, smoke_test=True) -def test_jupyter_rich_output(start_client_server_4_cpus): +def test_jupyter_rich_output(legacy_progress_reporter, start_client_server_4_cpus): assert ray.util.client.ray.is_connected() def dummy_objective(config): diff --git a/python/ray/tune/tests/test_progress_reporter.py b/python/ray/tune/tests/test_progress_reporter.py index 2691a227e9ca..d1e8a0c93119 100644 --- a/python/ray/tune/tests/test_progress_reporter.py +++ b/python/ray/tune/tests/test_progress_reporter.py @@ -325,12 +325,10 @@ def train(config): # Add "verbose=3)" etc -@pytest.mark.skipif( - "AIR_VERBOSITY" in os.environ, reason="console v2 doesn't work with this v1 test." -) class ProgressReporterTest(unittest.TestCase): def setUp(self) -> None: os.environ["TUNE_MAX_PENDING_TRIALS_PG"] = "auto" + os.environ["RAY_AIR_NEW_OUTPUT"] = "0" def mock_trial(self, status, i): mock = MagicMock() @@ -402,7 +400,7 @@ def test(config): for i in range(3): tune.report(**test_result) - analysis = tune.run(test, num_samples=3) + analysis = tune.run(test, num_samples=3, verbose=3) all_trials = analysis.trials inferred_results = reporter._infer_user_metrics(all_trials) for metric in inferred_results: @@ -421,7 +419,7 @@ def report(self, *args, **kwargs): self._output.append(progress_str) reporter = TestReporter() - analysis = tune.run(test, num_samples=3, progress_reporter=reporter) + analysis = tune.run(test, num_samples=3, progress_reporter=reporter, verbose=3) found = {k: False for k in test_result} for output in reporter._output: for key in test_result: @@ -799,7 +797,12 @@ def should_report(self, trials, done=False): def report(self, trials, done, *sys_info): pass - tune.run(lambda config: 2, num_samples=1, progress_reporter=CustomReporter()) + tune.run( + lambda config: 2, + num_samples=1, + progress_reporter=CustomReporter(), + verbose=3, + ) def testMaxLen(self): trials = [] diff --git a/python/ray/tune/tune.py b/python/ray/tune/tune.py index 4c5c7dd2983a..04398faa9ac8 100644 --- a/python/ray/tune/tune.py +++ b/python/ray/tune/tune.py @@ -35,6 +35,7 @@ get_air_verbosity, _detect_reporter as _detect_air_reporter, IS_NOTEBOOK, + AirVerbosity, ) from ray.tune.impl.placeholder import create_resolvers_map, inject_placeholders @@ -256,7 +257,7 @@ def run( checkpoint_at_end: bool = False, checkpoint_keep_all_ranks: bool = False, checkpoint_upload_from_workers: bool = False, - verbose: Union[int, Verbosity] = Verbosity.V3_TRIAL_DETAILS, + verbose: Optional[Union[int, AirVerbosity, Verbosity]] = None, progress_reporter: Optional[ProgressReporter] = None, log_to_file: bool = False, trial_name_creator: Optional[Callable[[Trial], str]] = None, @@ -394,9 +395,11 @@ def run( training workers. checkpoint_upload_from_workers: Whether to upload checkpoint files directly from distributed training workers. - verbose: 0, 1, 2, or 3. Verbosity mode. - 0 = silent, 1 = only status updates, 2 = status and brief trial - results, 3 = status and detailed trial results. Defaults to 3. + verbose: 0, 1, or 2. Verbosity mode. + 0 = silent, 1 = default, 2 = verbose. Defaults to 1. + If ``RAY_AIR_NEW_OUTPUT=0``, uses the old verbosity settings: + 0 = silent, 1 = only status updates, 2 = status and brief + results, 3 = status and detailed results. progress_reporter: Progress reporter for reporting intermediate experiment progress. Defaults to CLIReporter if running in command-line, or JupyterNotebookReporter if running in @@ -529,11 +532,18 @@ class and registered trainables. DeprecationWarning, ) + if verbose is None: + # Default `verbose` value. For new output engine, this is AirVerbosity.VERBOSE. + # For old output engine, this is Verbosity.V3_TRIAL_DETAILS + verbose = get_air_verbosity(AirVerbosity.VERBOSE) or Verbosity.V3_TRIAL_DETAILS + if _remote: - if get_air_verbosity() is not None: - logger.warning( - "Ignoring AIR_VERBOSITY setting, " - "as it doesn't support ray client mode yet." + if get_air_verbosity(verbose) is not None: + logger.info( + "[output] This uses the legacy output and progress reporter, " + "as Ray client is not supported by the new engine. " + "For more information, see " + "https://docs.ray.io/en/master/ray-air/experimental-features.html" ) remote_run = ray.remote(num_cpus=0)(run) @@ -586,21 +596,28 @@ class and registered trainables. "must be one of ['min', 'max']" ) - air_verbosity = get_air_verbosity() + air_verbosity = get_air_verbosity(verbose) if air_verbosity is not None and IS_NOTEBOOK: - logger.warning( - "Ignoring AIR_VERBOSITY setting, " - "as it doesn't support JupyterNotebook mode yet." + logger.info( + "[output] This uses the legacy output and progress reporter, " + "as Jupyter notebooks are not supported by the new engine, yet. " + "For more information, please see " + "https://docs.ray.io/en/master/ray-air/experimental-features.html" ) air_verbosity = None if air_verbosity is not None: - logger.warning( - f"Testing new AIR console output flow with verbosity={air_verbosity}. " - f"This will also disable the old flow - setting it to 0 now." + logger.info( + f"[output] This will use the new output engine with verbosity " + f"{air_verbosity}. To disable the new output and use the legacy " + f"output engine, set the environment variable RAY_AIR_NEW_OUTPUT=0. " + f"For more information, please see " + f"https://docs.ray.io/en/master/ray-air/experimental-features.html" ) + # Disable old output engine set_verbosity(0) else: + # Use old output engine set_verbosity(verbose) config = config or {} @@ -979,7 +996,7 @@ class and registered trainables. air_verbosity, search_alg.total_samples, metric=metric, mode=mode ) - # rich live context manager has to be called encapsulting + # rich live context manager has to be called encapsulating # the while loop. For other kind of reporters, no op. # `ExitStack` allows us to *conditionally* apply context manager. with contextlib.ExitStack() as stack: @@ -1085,7 +1102,7 @@ def run_experiments( experiments: Union[Experiment, Mapping, Sequence[Union[Experiment, Mapping]]], scheduler: Optional[TrialScheduler] = None, server_port: Optional[int] = None, - verbose: Union[int, Verbosity] = Verbosity.V3_TRIAL_DETAILS, + verbose: Optional[Union[int, AirVerbosity, Verbosity]] = None, progress_reporter: Optional[ProgressReporter] = None, resume: Union[bool, str] = False, reuse_actors: Optional[bool] = None, @@ -1119,11 +1136,18 @@ def run_experiments( if not trial_executor or isinstance(trial_executor, RayTrialExecutor): _ray_auto_init(entrypoint="tune.run_experiments(...)") + if verbose is None: + # Default `verbose` value. For new output engine, this is AirVerbosity.VERBOSE. + # For old output engine, this is Verbosity.V3_TRIAL_DETAILS + verbose = get_air_verbosity(AirVerbosity.VERBOSE) or Verbosity.V3_TRIAL_DETAILS + if _remote: - if get_air_verbosity() is not None: - logger.warning( - "Ignoring AIR_VERBOSITY setting, " - "as it doesn't support ray client mode yet." + if get_air_verbosity(verbose) is not None: + logger.info( + "[output] This uses the legacy output and progress reporter, " + "as Ray client is not supported by the new engine. " + "For more information, see " + "https://docs.ray.io/en/master/ray-air/experimental-features.html" ) remote_run = ray.remote(num_cpus=0)(run_experiments) diff --git a/python/ray/tune/tuner.py b/python/ray/tune/tuner.py index 1658188b9530..08a1cd2de0a9 100644 --- a/python/ray/tune/tuner.py +++ b/python/ray/tune/tuner.py @@ -150,11 +150,15 @@ def __init__( """Configure and construct a tune run.""" kwargs = locals().copy() self._is_ray_client = ray.util.client.ray.is_connected() - if self._is_ray_client and get_air_verbosity() is not None: - logger.warning( - "Ignoring AIR_VERBOSITY setting, " - "as it doesn't support ray client mode yet." - ) + if self._is_ray_client: + _run_config = run_config or RunConfig() + if get_air_verbosity(_run_config.verbose) is not None: + logger.info( + "[output] This uses the legacy output and progress reporter, " + "as Ray client is not supported by the new engine. " + "For more information, see " + "https://docs.ray.io/en/master/ray-air/experimental-features.html" + ) if _tuner_internal: if not self._is_ray_client: