From ef3719a07ad2cc0c40d08b124b0bf0bb26e29c76 Mon Sep 17 00:00:00 2001 From: Callum Wells <68609181+swells2020@users.noreply.github.com> Date: Thu, 24 Aug 2023 10:07:44 +0100 Subject: [PATCH] [RPD-299] Create CLI stack remove command (#204) * [RPD-239] Improve the way the UI handles latency when contacting Azure services (#198) * [RPD-248] Add a basic example demonstrating how to use the API (#199) * updates docs * removes scratch.py * fixes typos * updates for comments * adds cli command to cli module * fix circular import (#201) * bumping to version v0.2.9 for release * adds tests --------- Co-authored-by: KirsoppJ <40233184+KirsoppJ@users.noreply.github.com> Co-authored-by: Jonathan Carlton --- RELEASE_NOTES.md | 18 +++++++++ docs/getting-started.md | 40 +++++++++++++++++- pyproject.toml | 2 +- src/matcha_ml/VERSION | 2 +- src/matcha_ml/cli/_validation.py | 4 ++ src/matcha_ml/cli/cli.py | 29 +++++++++++++ src/matcha_ml/core/__init__.py | 2 + src/matcha_ml/core/core.py | 4 ++ src/matcha_ml/runners/base_runner.py | 14 ++++--- tests/test_cli/test_stack.py | 56 +++++++++++++++++++++++++- tests/test_runners/test_base_runner.py | 14 ------- 11 files changed, 161 insertions(+), 24 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4e473491..d0b7e4fc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,21 @@ +# v0.2.9 + +This is a minor release to address a bug and improve documentation based on the changes introduced in v0.2.8. + +Date: 23rd August 2023 + +## Bug Fixes + +* Fixed a circular import bug ([#201](https://github.com/fuzzylabs/matcha/pull/201)) + +## Documentation + +* Adds API-based examples to the getting started guide ([#119](https://github.com/fuzzylabs/matcha/pull/199)) + +See all changes here: https://github.com/fuzzylabs/matcha/compare/v0.2.8...v0.2.9 + +--- + # Stacks 📚 LLMs are all the rage at the moment, with new and improved models being released almost daily. These models are quite large (as implied by the name) and cannot be hosted on standard personal computers, therefore we need to use cloud infrastructure to manage and deploy these models. However, standing up and managing these cloud resources isn't typically the forte of a lot of those interested in LLMs. diff --git a/docs/getting-started.md b/docs/getting-started.md index bb9f3909..fe5174af 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -2,6 +2,8 @@ In this guide, we'll walk you through how to provision your first machine learning infrastructure to Azure, and then use that infrastructure to train and deploy a model. The model we're using is a movie recommender, and we picked this because it's one that beginners can get up and running with quickly. +There are two ways to interact with Matcha; via the CLI tool, or through the API. Throughout this guide we'll demonstrate how to get started using either method. + There are five things we'll cover: * [Pre-requisites](#pre-requisites): everything you need to set up before starting. @@ -78,16 +80,27 @@ When you run this command, you'll be taken to the Azure login screen in a web br Next, let's provision: +CLI: ```bash matcha provision ``` -Initially, Matcha will ask you a few questions about how you'd like your infrastructure to be set up. Specifically, it will ask for a _name_ for your infrastructure, a _region_ to deploy it to. Once these details are provided, Matcha will proceed to initialize a remote state manager and ask for a password. After that, it will go ahead of provision infrastructure. +> Note: users have the choice of passing optional arguments representing the location, prefix, and password parameters by using '--location', '--prefix', or '--password'. For example; `--location uksouth --prefix test123 --password strong_password`. + +API: +```python +import matcha_ml.core as matcha + +matcha_state_object: MatchaState = matcha.provision(location="uksouth", prefix="test123", password="strong_password") +``` + +Initially, Matcha will ask you a few questions about how you'd like your infrastructure to be set up. Specifically, it will ask for a _location_ for your infrastructure, a _prefix_ to deploy it to. Once these details are provided, Matcha will proceed to initialize a remote state manager and ask for a password. After that, it will go ahead of provision infrastructure. > Note: provisioning can take up to 20 minutes. Once provisioning is completed, you can query Matcha, using the `get` command: +CLI: ```bash matcha get ``` @@ -152,6 +165,23 @@ Experiment tracker By default, Matcha will hide sensitive resource properties. If you need one of these properties, then you can add the `--show-sensitive` flag to your `get` command. +API: +```python +import matcha_ml.core as matcha + +matcha_state_object: MatchaState = matcha.get() +``` + +As with the CLI tool, users have the ability to 'get' specific resources by passing optional `resource_name` and `property_name` arguments to the get function, as demonstrated below: + +```python +import matcha_ml.core as matcha + +matcha_state_object: MatchaState = matcha.get(resource_name="experiment_tracker", property_name="flavor") +``` + +> Note: the `get()` method will return a `MatchaState` object which represents the provisioned state. The `MatchaState` object contains the `get_component()` method, which will return (where applicable) a `MatchaStateComponent` object representing the specified Matcha state component. In turn, each `MatchaStateComponent` object has a `find_property()` method that will allow the user to be able to access individual component properties. + # 🤝 Sharing resources You'll notice that a configuration file is create as part of the provisioning process - it's called `matcha.config.json`. This file stores the information necessary for Matcha to identify the resource group and storage container that holds the details of the provisioned resources. @@ -236,10 +266,18 @@ This will result in a score, which represents how strongly we recommend movie ID The final thing you'll want to do is decommission the infrastructure that Matcha has set up during this guide. Matcha includes a `destroy` command which will remove everything that has been provisioned, which avoids running up an Azure bill! +CLI: ```bash matcha destroy ``` +API: +```python +import matcha_ml.core as matcha + +matcha.destroy() +``` + > Note: that this command is irreversible will remove all the resources deployed by `matcha provision` including the resource group, so make sure you save any data you wish to keep before running this command. > > You may also notice that an additional resource has appeared in Azure called 'NetworkWatcherRG' (if it wasn't already there). This is a resource that is automatically provisioned by Azure in each region when there is in-coming traffic to a provisioned resource and isn't controlled by Matcha. More information can be found [here](https://learn.microsoft.com/en-us/azure/network-watcher/network-watcher-monitoring-overview) on how to manage or remove this resource. diff --git a/pyproject.toml b/pyproject.toml index b372dccf..a7b0118f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "matcha-ml" -version = "0.2.8" +version = "0.2.9" description = "Matcha: An open source tool for provisioning MLOps environments to the cloud." authors = ["FuzzyLabs "] license = "Apache-2.0" diff --git a/src/matcha_ml/VERSION b/src/matcha_ml/VERSION index a45be462..1866a362 100644 --- a/src/matcha_ml/VERSION +++ b/src/matcha_ml/VERSION @@ -1 +1 @@ -0.2.8 +0.2.9 diff --git a/src/matcha_ml/cli/_validation.py b/src/matcha_ml/cli/_validation.py index d9e4dadd..5afef1b7 100644 --- a/src/matcha_ml/cli/_validation.py +++ b/src/matcha_ml/cli/_validation.py @@ -4,6 +4,8 @@ from typer import BadParameter +from matcha_ml.cli.ui.print_messages import print_status +from matcha_ml.cli.ui.status_message_builders import build_status from matcha_ml.core._validation import is_valid_prefix, is_valid_region from matcha_ml.errors import MatchaInputError from matcha_ml.services import AzureClient @@ -66,6 +68,7 @@ def region_typer_callback(region: str) -> str: Returns: str: the region after checks are passed. """ + print_status(build_status("Validating region selection with Azure...")) if not region: return region @@ -90,6 +93,7 @@ def prefix_typer_callback(prefix: str) -> str: Returns: str: if valid, the prefix is returned. """ + print_status(build_status("Validating prefix selection with Azure...")) if not prefix: return prefix diff --git a/src/matcha_ml/cli/cli.py b/src/matcha_ml/cli/cli.py index 2bfaa837..8cc1ff2f 100644 --- a/src/matcha_ml/cli/cli.py +++ b/src/matcha_ml/cli/cli.py @@ -23,6 +23,7 @@ build_step_success_status, ) from matcha_ml.cli.ui.user_approval_functions import is_user_approved +from matcha_ml.core.core import stack_remove from matcha_ml.errors import MatchaError, MatchaInputError app = typer.Typer(no_args_is_help=True, pretty_exceptions_show_locals=False) @@ -266,5 +267,33 @@ def set(stack: str = typer.Argument("default")) -> None: raise typer.Exit() +@stack_app.command(help="Remove a module from the current Matcha stack.") +def remove(module: str = typer.Argument(None)) -> None: + """Remove a module from the current Matcha stack. + + Args: + module (str): the name of the module to be removed. + """ + if module: + try: + stack_remove(module) + print_status( + build_status( + f"Matcha '{module}' module has been removed from the current stack." + ) + ) + except MatchaInputError as e: + print_error(str(e)) + raise typer.Exit() + except MatchaError as e: + print_error(str(e)) + raise typer.Exit() + else: + print_error( + "No module specified. Please run `matcha stack remove` again and provide the name of the module you wish to remove." + ) + raise typer.Exit() + + if __name__ == "__main__": app() diff --git a/src/matcha_ml/core/__init__.py b/src/matcha_ml/core/__init__.py index 97bac49d..86077f68 100644 --- a/src/matcha_ml/core/__init__.py +++ b/src/matcha_ml/core/__init__.py @@ -7,6 +7,7 @@ provision, remove_state_lock, stack_set, + stack_remove, ) __all__ = [ @@ -17,4 +18,5 @@ "destroy", "provision", "stack_set", + "stack_remove", ] diff --git a/src/matcha_ml/core/core.py b/src/matcha_ml/core/core.py index 5e3ea522..acdaab77 100644 --- a/src/matcha_ml/core/core.py +++ b/src/matcha_ml/core/core.py @@ -366,3 +366,7 @@ def stack_set(stack_name: str) -> None: ) MatchaConfigService.update(stack) + +def stack_remove(module_name: str) -> str: + """A placeholder for the stack remove logic in core.""" + return module_name \ No newline at end of file diff --git a/src/matcha_ml/runners/base_runner.py b/src/matcha_ml/runners/base_runner.py index 5875c94b..97ff75f8 100644 --- a/src/matcha_ml/runners/base_runner.py +++ b/src/matcha_ml/runners/base_runner.py @@ -1,7 +1,8 @@ """Run terraform templates to provision and deprovision resources.""" import os +from abc import abstractmethod from multiprocessing.pool import ThreadPool -from typing import Optional, Tuple +from typing import Any, Optional import typer @@ -18,7 +19,6 @@ TerraformConfig, TerraformService, ) -from matcha_ml.state.matcha_state import MatchaStateService SPINNER = "dots" @@ -183,10 +183,12 @@ def _destroy_terraform(self, msg: str = "") -> None: if tf_result.return_code != 0: raise MatchaTerraformError(tf_error=tf_result.std_err) - def provision(self) -> MatchaStateService: + @abstractmethod + def provision(self) -> Any: """Provision resources required for the deployment.""" - raise NotImplementedError + pass - def deprovision(self) -> None: + @abstractmethod + def deprovision(self) -> Any: """Destroy the provisioned resources.""" - raise NotImplementedError + pass diff --git a/tests/test_cli/test_stack.py b/tests/test_cli/test_stack.py index 29f763cb..846c6c7f 100644 --- a/tests/test_cli/test_stack.py +++ b/tests/test_cli/test_stack.py @@ -1,6 +1,7 @@ """Test suit to test the stack command and all its subcommands.""" import os +from unittest.mock import MagicMock, patch from typer.testing import CliRunner @@ -8,7 +9,7 @@ from matcha_ml.config import MatchaConfig, MatchaConfigService from matcha_ml.state.remote_state_manager import RemoteStateManager -INTERNAL_FUNCTION_STUB = "matcha_ml.core" +INTERNAL_FUNCTION_STUB = "matcha_ml.core.core" def test_cli_stack_command_help_option(runner: CliRunner) -> None: @@ -158,3 +159,56 @@ def test_stack_set_file_modified( assert "stack" in new_config_dict assert new_config_dict["stack"]["name"] == "llm" assert config_dict.items() <= new_config_dict.items() + + +def test_cli_stack_set_remove_help_option(runner: CliRunner) -> None: + """Tests the --help option for the cli stack remove sub-command. + + Args: + runner (CliRunner): typer CLI runner. + """ + result = runner.invoke(app, ["stack", "remove", "--help"]) + + assert result.exit_code == 0 + + assert "Remove a module from the current Matcha stack." in result.stdout + + +def test_cli_stack_remove_command_without_args(runner: CliRunner) -> None: + """Tests the --help option for the cli stack remove sub-command. + + Args: + runner (CliRunner): typer CLI runner. + """ + result = runner.invoke(app, ["stack", "remove"]) + + assert result.exit_code == 0 + + assert ( + "No module specified. Please run `matcha stack remove` again and provide the name\nof the module you wish to remove.\n" + in result.stdout + ) + + +@patch(f"{INTERNAL_FUNCTION_STUB}.stack_remove") +def test_cli_stack_remove_command_with_args( + mocked_stack_remove: MagicMock, + matcha_testing_directory: str, + runner: CliRunner, +) -> None: + """Tests the cli stack set sub-command with args. + + Args: + mocked_stack_remove (MagicMock): a mocked stack_remove function. + matcha_testing_directory (str): a temporary working directory. + runner (CliRunner): typer CLI runner. + """ + os.chdir(matcha_testing_directory) + result = runner.invoke(app, ["stack", "remove", "experiment_tracker"]) + + assert result.exit_code == 0 + assert mocked_stack_remove.assert_called_once + assert ( + "Matcha 'experiment_tracker' module has been removed from the current stack." + in result.stdout + ) diff --git a/tests/test_runners/test_base_runner.py b/tests/test_runners/test_base_runner.py index 77d30656..5cc43986 100644 --- a/tests/test_runners/test_base_runner.py +++ b/tests/test_runners/test_base_runner.py @@ -168,17 +168,3 @@ def test_destroy_terraform(capsys: SysCapture): str(exc_info.value) == "Terraform failed because of the following error: 'Destroy failed'." ) - - -def test_provision(): - """Test provision function in BaseRunner class raises NotImplemented exception.""" - template_runner = BaseRunner() - with pytest.raises(NotImplementedError): - template_runner.provision() - - -def test_deprovision(): - """Test deprovision function in BaseRunner class raises NotImplemented exception.""" - template_runner = BaseRunner() - with pytest.raises(NotImplementedError): - template_runner.deprovision()