From 22ebc0ebac86fede00aee43e75ef7f6a0e56b301 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 <60785969+matusdrobuliak66@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:45:52 +0100 Subject: [PATCH 01/24] =?UTF-8?q?=E2=9C=A8=20listen=20to=20wallet=20events?= =?UTF-8?q?=20in=20payment=20system=20(#5012)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/servicelib/rabbitmq/_client.py | 14 +++- .../tests/rabbitmq/test_rabbitmq.py | 17 ++++ .../core/application.py | 4 + .../services/auto_recharge_listener.py | 47 +++++++++++ .../services/auto_recharge_process_message.py | 20 +++++ .../services/rabbitmq.py | 2 + services/payments/tests/unit/conftest.py | 3 + .../test_services_auto_recharge_listener.py | 80 +++++++++++++++++++ .../wallets/payments/test_payments_methods.py | 1 - 9 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 services/payments/src/simcore_service_payments/services/auto_recharge_listener.py create mode 100644 services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py create mode 100644 services/payments/tests/unit/test_services_auto_recharge_listener.py diff --git a/packages/service-library/src/servicelib/rabbitmq/_client.py b/packages/service-library/src/servicelib/rabbitmq/_client.py index ae668f26737..542bd2ba2ee 100644 --- a/packages/service-library/src/servicelib/rabbitmq/_client.py +++ b/packages/service-library/src/servicelib/rabbitmq/_client.py @@ -63,6 +63,9 @@ async def _get_channel(self) -> aio_pika.abc.AbstractChannel: channel.close_callbacks.add(self._channel_close_callback) return channel + async def _get_consumer_tag(self, exchange_name) -> str: + return f"{get_rabbitmq_client_unique_name(self.client_name)}_{exchange_name}" + async def subscribe( self, exchange_name: str, @@ -139,10 +142,11 @@ async def _on_message( ) await message.nack() + _consumer_tag = await self._get_consumer_tag(exchange_name) await queue.consume( _on_message, exclusive=exclusive_queue, - consumer_tag=f"{get_rabbitmq_client_unique_name(self.client_name)}_{exchange_name}", + consumer_tag=_consumer_tag, ) output: str = queue.name return output @@ -214,3 +218,11 @@ async def publish(self, exchange_name: str, message: RabbitMessage) -> None: aio_pika.Message(message.body()), routing_key=message.routing_key() or "", ) + + async def unsubscribe_consumer(self, exchange_name: str): + assert self._channel_pool # nosec + async with self._channel_pool.acquire() as channel: + queue_name = exchange_name + queue = await channel.get_queue(queue_name) + _consumer_tag = await self._get_consumer_tag(exchange_name) + await queue.cancel(_consumer_tag) diff --git a/packages/service-library/tests/rabbitmq/test_rabbitmq.py b/packages/service-library/tests/rabbitmq/test_rabbitmq.py index bf95e267d7c..af93023ea5f 100644 --- a/packages/service-library/tests/rabbitmq/test_rabbitmq.py +++ b/packages/service-library/tests/rabbitmq/test_rabbitmq.py @@ -435,3 +435,20 @@ async def test_rabbit_not_using_the_same_exchange_type_raises( # now do a second subscribtion wiht topics, will create a TOPICS exchange with pytest.raises(aio_pika.exceptions.ChannelPreconditionFailed): await client.subscribe(exchange_name, mocked_message_parser, topics=[]) + + +@pytest.mark.no_cleanup_check_rabbitmq_server_has_no_errors() +async def test_unsubscribe_consumer( + rabbitmq_client: Callable[[str], RabbitMQClient], + random_exchange_name: Callable[[], str], + mocked_message_parser: mock.AsyncMock, +): + exchange_name = f"{random_exchange_name()}" + client = rabbitmq_client("consumer") + await client.subscribe(exchange_name, mocked_message_parser, exclusive_queue=False) + # Unsubsribe just a consumer, the queue will be still there + await client.unsubscribe_consumer(exchange_name) + # Unsubsribe the queue + await client.unsubscribe(exchange_name) + with pytest.raises(aio_pika.exceptions.ChannelNotFoundEntity): + await client.unsubscribe(exchange_name) diff --git a/services/payments/src/simcore_service_payments/core/application.py b/services/payments/src/simcore_service_payments/core/application.py index 899f2735d0f..815355b1cf9 100644 --- a/services/payments/src/simcore_service_payments/core/application.py +++ b/services/payments/src/simcore_service_payments/core/application.py @@ -11,6 +11,7 @@ ) from ..api.rest.routes import setup_rest_api from ..api.rpc.routes import setup_rpc_api_routes +from ..services.auto_recharge_listener import setup_auto_recharge_listener from ..services.payments_gateway import setup_payments_gateway from ..services.postgres import setup_postgres from ..services.rabbitmq import setup_rabbitmq @@ -51,6 +52,9 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: # APIs w/ RUT setup_resource_usage_tracker(app) + # Listening to Rabbitmq + setup_auto_recharge_listener(app) + # ERROR HANDLERS # ... add here ... diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_listener.py b/services/payments/src/simcore_service_payments/services/auto_recharge_listener.py new file mode 100644 index 00000000000..d60c22d84f3 --- /dev/null +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_listener.py @@ -0,0 +1,47 @@ +import functools +import logging + +from fastapi import FastAPI +from models_library.rabbitmq_messages import WalletCreditsMessage +from servicelib.logging_utils import log_context +from servicelib.rabbitmq import RabbitMQClient + +from .auto_recharge_process_message import process_message +from .rabbitmq import get_rabbitmq_client + +_logger = logging.getLogger(__name__) + + +async def _subscribe_to_rabbitmq(app) -> str: + with log_context(_logger, logging.INFO, msg="Subscribing to rabbitmq channel"): + rabbit_client: RabbitMQClient = get_rabbitmq_client(app) + subscribed_queue: str = await rabbit_client.subscribe( + WalletCreditsMessage.get_channel_name(), + message_handler=functools.partial(process_message, app), + exclusive_queue=False, + topics=["#"], + ) + return subscribed_queue + + +async def _unsubscribe_consumer(app) -> None: + with log_context(_logger, logging.INFO, msg="Unsubscribing from rabbitmq queue"): + rabbit_client: RabbitMQClient = get_rabbitmq_client(app) + await rabbit_client.unsubscribe_consumer( + WalletCreditsMessage.get_channel_name(), + ) + + +def setup_auto_recharge_listener(app: FastAPI): + async def _on_startup(): + app.state.auto_recharge_rabbitmq_consumer = await _subscribe_to_rabbitmq(app) + + async def _on_shutdown(): + assert app.state.auto_recharge_rabbitmq_consumer # nosec + if app.state.rabbitmq_client: + # NOTE: We want to have persistent queue, therefore we will unsubscribe only consumer + await _unsubscribe_consumer(app) + app.state.auto_recharge_rabbitmq_constumer = None + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py new file mode 100644 index 00000000000..8480063b19b --- /dev/null +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -0,0 +1,20 @@ +import logging + +from fastapi import FastAPI +from models_library.rabbitmq_messages import WalletCreditsMessage +from pydantic import parse_raw_as + +_logger = logging.getLogger(__name__) + + +async def process_message(app: FastAPI, data: bytes) -> bool: + assert app # nosec + rabbit_message = parse_raw_as(WalletCreditsMessage, data) + _logger.debug("Process msg: %s", rabbit_message) + + # 1. Check if auto-recharge functionality is ON for wallet_id + # 2. Check if wallet credits are bellow the threshold + # 3. Get Payment method + # 4. Pay with payment method + + return True diff --git a/services/payments/src/simcore_service_payments/services/rabbitmq.py b/services/payments/src/simcore_service_payments/services/rabbitmq.py index 5083c4f6b8a..58facd39868 100644 --- a/services/payments/src/simcore_service_payments/services/rabbitmq.py +++ b/services/payments/src/simcore_service_payments/services/rabbitmq.py @@ -31,8 +31,10 @@ async def _on_startup() -> None: async def _on_shutdown() -> None: if app.state.rabbitmq_client: await app.state.rabbitmq_client.close() + app.state.rabbitmq_client = None if app.state.rabbitmq_rpc_server: await app.state.rabbitmq_rpc_server.close() + app.state.rabbitmq_rpc_server = None app.add_event_handler("startup", _on_startup) app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index f177e747db8..b46bc3a4673 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -57,6 +57,9 @@ def _do(): # The following services are affected if rabbitmq is not in place mocker.patch("simcore_service_payments.core.application.setup_rabbitmq") mocker.patch("simcore_service_payments.core.application.setup_rpc_api_routes") + mocker.patch( + "simcore_service_payments.core.application.setup_auto_recharge_listener" + ) return _do diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py new file mode 100644 index 00000000000..9557dba382d --- /dev/null +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -0,0 +1,80 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +from collections.abc import Callable +from decimal import Decimal +from unittest import mock + +import pytest +from fastapi import FastAPI +from models_library.rabbitmq_messages import WalletCreditsMessage +from pytest_mock.plugin import MockerFixture +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from servicelib.rabbitmq import RabbitMQClient +from tenacity._asyncio import AsyncRetrying +from tenacity.retry import retry_if_exception_type +from tenacity.stop import stop_after_delay +from tenacity.wait import wait_fixed + +pytest_simcore_core_services_selection = [ + "postgres", + "rabbit", +] +pytest_simcore_ops_services_selection = [ + "adminer", +] + + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, + rabbit_env_vars_dict: EnvVarsDict, # rabbitMQ settings from 'rabbit' service + postgres_env_vars_dict: EnvVarsDict, + wait_for_postgres_ready_and_db_migrated: None, + external_environment: EnvVarsDict, +): + # set environs + monkeypatch.delenv("PAYMENTS_RABBITMQ", raising=False) + monkeypatch.delenv("PAYMENTS_POSTGRES", raising=False) + + return setenvs_from_dict( + monkeypatch, + { + **app_environment, + **rabbit_env_vars_dict, + **postgres_env_vars_dict, + **external_environment, + "POSTGRES_CLIENT_NAME": "payments-service-pg-client", + }, + ) + + +@pytest.fixture +async def mocked_message_parser(mocker: MockerFixture) -> mock.AsyncMock: + return mocker.patch( + "simcore_service_payments.services.auto_recharge_listener.process_message" + ) + + +async def test_process_event_functions( + mocked_message_parser: mock.AsyncMock, + app: FastAPI, + rabbitmq_client: Callable[[str], RabbitMQClient], +): + publisher = rabbitmq_client("publisher") + msg = WalletCreditsMessage(wallet_id=1, credits=Decimal(120.5)) + await publisher.publish(WalletCreditsMessage.get_channel_name(), msg) + + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), + stop=stop_after_delay(5), + retry=retry_if_exception_type(AssertionError), + reraise=True, + ): + with attempt: + mocked_message_parser.assert_called_once() diff --git a/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py b/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py index 500ede13316..9b1fe418560 100644 --- a/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py +++ b/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py @@ -334,7 +334,6 @@ async def wallet_payment_method_id( return await _add_payment_method(client, wallet_id=logged_user_wallet.wallet_id) -@pytest.mark.testit async def test_one_time_payment_with_payment_method( latest_osparc_price: Decimal, client: TestClient, From 0284b248a212861d0143a9a5cba4c26d136da232 Mon Sep 17 00:00:00 2001 From: Ignacio Pascual <4764217+ignapas@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:52:15 +0100 Subject: [PATCH 02/24] Share wallet bug (#5021) Fixes #5006 --- .../client/source/class/osparc/desktop/wallets/MembersList.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/static-webserver/client/source/class/osparc/desktop/wallets/MembersList.js b/services/static-webserver/client/source/class/osparc/desktop/wallets/MembersList.js index 9104752a285..2a827c0fc99 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/wallets/MembersList.js +++ b/services/static-webserver/client/source/class/osparc/desktop/wallets/MembersList.js @@ -132,7 +132,7 @@ qx.Class.define("osparc.desktop.wallets.MembersList", { serializedData["resourceType"] = "wallet"; const showOrganizations = false; const collaboratorsManager = new osparc.share.NewCollaboratorsManager(serializedData, showOrganizations); - collaboratorsManager.addListener("addCollaborators", e => { + collaboratorsManager.addListener("addEditors", e => { const cb = () => collaboratorsManager.close(); this.__addMembers(e.getData(), cb); }, this); From 6f515976f7bd49651f4064762d5e0c6920f77c09 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Mon, 13 Nov 2023 18:08:58 +0100 Subject: [PATCH 03/24] =?UTF-8?q?=E2=99=BB=EF=B8=8FAutoscaling:=20Debug=20?= =?UTF-8?q?logs=20for=20issue=20with=20scaling=20up=20(#5025)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../utils/utils_docker.py | 33 +++++++++++------- services/autoscaling/tests/unit/conftest.py | 1 + .../unit/test_modules_auto_scaling_dynamic.py | 34 +++++++++++++++---- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py index 0e5e88585c2..65b8ce8af05 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py @@ -107,21 +107,24 @@ def _check_if_node_is_removable(node: Node) -> bool: def _is_task_waiting_for_resources(task: Task) -> bool: # NOTE: https://docs.docker.com/engine/swarm/how-swarm-mode-works/swarm-task-states/ - if ( - not task.Status - or not task.Status.State - or not task.Status.Message - or not task.Status.Err + with log_context( + logger, level=logging.DEBUG, msg=f"_is_task_waiting_for_resources: {task}" ): - return False - return ( - task.Status.State == TaskState.pending - and task.Status.Message == _PENDING_DOCKER_TASK_MESSAGE - and ( - _INSUFFICIENT_RESOURCES_DOCKER_TASK_ERR in task.Status.Err - or _NOT_SATISFIED_SCHEDULING_CONSTRAINTS_TASK_ERR in task.Status.Err + if ( + not task.Status + or not task.Status.State + or not task.Status.Message + or not task.Status.Err + ): + return False + return ( + task.Status.State == TaskState.pending + and task.Status.Message == _PENDING_DOCKER_TASK_MESSAGE + and ( + _INSUFFICIENT_RESOURCES_DOCKER_TASK_ERR in task.Status.Err + or _NOT_SATISFIED_SCHEDULING_CONSTRAINTS_TASK_ERR in task.Status.Err + ) ) - ) async def _associated_service_has_no_node_placement_contraints( @@ -187,6 +190,10 @@ async def pending_service_tasks_with_insufficient_resources( ) sorted_tasks = sorted(tasks, key=_by_created_dt) + logger.debug( + "found following tasks that might trigger autoscaling: %s", + [task.ID for task in tasks], + ) return [ task diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index 619ec908545..e388acba225 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -514,6 +514,7 @@ def aws_allowed_ec2_instance_type_names() -> list[InstanceTypeType]: "t2.2xlarge", "g3.4xlarge", "g4dn.2xlarge", + "g4dn.8xlarge", "r5n.4xlarge", "r5n.8xlarge", ] diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py index 0044c157b0d..cb41d2d3278 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py @@ -379,7 +379,9 @@ async def _assert_ec2_instances( assert len(all_instances["Reservations"]) == num_reservations for reservation in all_instances["Reservations"]: assert "Instances" in reservation - assert len(reservation["Instances"]) == num_instances + assert ( + len(reservation["Instances"]) == num_instances + ), f"created {num_instances} instances of {reservation['Instances'][0]['InstanceType'] if num_instances > 0 else 'n/a'}" for instance in reservation["Instances"]: assert "InstanceType" in instance assert instance["InstanceType"] == instance_type @@ -440,7 +442,7 @@ async def _assert_ec2_instances( ), ], ) -async def test_cluster_scaling_up_and_down( +async def test_cluster_scaling_up_and_down( # noqa: PLR0915 minimal_configuration: None, service_monitored_labels: dict[DockerLabelKey, str], app_settings: ApplicationSettings, @@ -686,6 +688,7 @@ async def test_cluster_scaling_up_and_down( @dataclass(frozen=True) class _ScaleUpParams: + imposed_instance_type: str | None service_resources: Resources num_services: int expected_instance_type: str @@ -697,15 +700,28 @@ class _ScaleUpParams: [ pytest.param( _ScaleUpParams( + imposed_instance_type=None, service_resources=Resources( cpus=5, ram=parse_obj_as(ByteSize, "36Gib") ), num_services=10, - expected_instance_type="g3.4xlarge", + expected_instance_type="g3.4xlarge", # 1 GPU, 16 CPUs, 122GiB expected_num_instances=4, ), id="sim4life-light", - ) + ), + pytest.param( + _ScaleUpParams( + imposed_instance_type="g4dn.8xlarge", + service_resources=Resources( + cpus=5, ram=parse_obj_as(ByteSize, "20480MB") + ), + num_services=7, + expected_instance_type="g4dn.8xlarge", # 1 GPU, 32 CPUs, 128GiB + expected_num_instances=2, + ), + id="sim4life", + ), ], ) async def test_cluster_scaling_up_starts_multiple_instances( @@ -714,13 +730,12 @@ async def test_cluster_scaling_up_starts_multiple_instances( app_settings: ApplicationSettings, initialized_app: FastAPI, create_service: Callable[ - [dict[str, Any], dict[DockerLabelKey, str], str], Awaitable[Service] + [dict[str, Any], dict[DockerLabelKey, str], str, list[str]], Awaitable[Service] ], task_template: dict[str, Any], create_task_reservations: Callable[[int, int], dict[str, Any]], ec2_client: EC2Client, mock_tag_node: mock.Mock, - fake_node: Node, scale_up_params: _ScaleUpParams, mock_rabbitmq_post_message: mock.Mock, mock_find_node_with_name: mock.Mock, @@ -741,6 +756,11 @@ async def test_cluster_scaling_up_starts_multiple_instances( ), service_monitored_labels, "pending", + [ + f"node.labels.{DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY}=={scale_up_params.imposed_instance_type}" + ] + if scale_up_params.imposed_instance_type + else [], ) for _ in range(scale_up_params.num_services) ) @@ -756,7 +776,7 @@ async def test_cluster_scaling_up_starts_multiple_instances( ec2_client, num_reservations=1, num_instances=scale_up_params.expected_num_instances, - instance_type="g3.4xlarge", + instance_type=scale_up_params.expected_instance_type, instance_state="running", ) From 9cabec07f034fa1bd1e76dacbba54e07b02f39bb Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:27:34 +0100 Subject: [PATCH 04/24] =?UTF-8?q?=F0=9F=93=9D=20Maintenance:=20cleanup=20r?= =?UTF-8?q?eadme=20and=20vscode=20settings=20(#5023)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/extensions.json | 4 ++++ .vscode/settings.template.json | 26 ++++++++------------------ README.md | 12 ++++-------- 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 24bd21ba4e7..65f20197c67 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -3,9 +3,13 @@ "charliermarsh.ruff", "eamodio.gitlens", "exiasr.hadolint", + "ms-azuretools.vscode-docker", + "ms-python.black-formatter", + "ms-python.pylint", "ms-python.python", "njpwerner.autodocstring", "samuelcolvin.jinjahtml", "timonwong.shellcheck", + "vscode-icons-team.vscode-icons", ] } diff --git a/.vscode/settings.template.json b/.vscode/settings.template.json index 731b7d3b0f8..3a4405e9594 100644 --- a/.vscode/settings.template.json +++ b/.vscode/settings.template.json @@ -1,5 +1,7 @@ // This is a template. Clone and replace extension ".template.json" by ".json" { + "autoDocstring.docstringFormat": "pep257", + "editor.tabSize": 2, "editor.insertSpaces": true, "editor.detectIndentation": false, @@ -27,9 +29,8 @@ "**/.git/subtree-cache/**": true, "**/node_modules/*/**": true }, - "python.formatting.autopep8Args": [ - "--max-line-length 140" - ], + "python.analysis.autoImportCompletions": true, + "python.analysis.typeCheckingMode": "basic", "python.analysis.extraPaths": [ "./packages/models-library/src", "./packages/postgres-database/src", @@ -54,26 +55,15 @@ "[makefile]": { "editor.insertSpaces": false }, - "python.testing.pytestEnabled": true, - "autoDocstring.docstringFormat": "pep257", "hadolint.hadolintPath": "${workspaceFolder}/scripts/hadolint.bash", "hadolint.cliOptions": [], - "shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck.bash", - "shellcheck.run": "onSave", - "shellcheck.enableQuickFix": true, - "python.formatting.provider": "black", - "isort.path": [ - "${workspaceFolder}/.venv/bin/isort" - ], - "isort.args": [ - "--settings-path=${workspaceFolder}/.isort.cfg" - ], - "python.analysis.autoImportCompletions": true, - "python.analysis.typeCheckingMode": "basic", "ruff.lint.args": [ "--config=${workspaceFolder}/.ruff.toml" ], "ruff.path": [ "${workspaceFolder}/.venv/bin/ruff" - ] + ], + "shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck.bash", + "shellcheck.run": "onSave", + "shellcheck.enableQuickFix": true } diff --git a/README.md b/README.md index 02051728b81..1cb462938a5 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@

- [![black_badge]](https://github.com/psf/black) [![ci_badge]](https://github.com/ITISFoundation/osparc-simcore/actions/workflows/ci-testing-deploy.yml) @@ -28,7 +27,6 @@ [osparc_status]:https://img.shields.io/badge/dynamic/json?label=osparc.io&query=%24.status.description&url=https%3A%2F%2Fstatus.osparc.io%2Fapi%2Fv2%2Fstatus.json - The SIM-CORE, named **o2S2PARC** – **O**pen **O**nline **S**imulations for **S**timulating **P**eripheral **A**ctivity to **R**elieve **C**onditions – is one of the three integrative cores of the SPARC program’s Data Resource Center (DRC). The aim of o2S2PARC is to establish a comprehensive, freely accessible, intuitive, and interactive online platform for simulating peripheral nerve system neuromodulation/ stimulation and its impact on organ physiology in a precise and predictive manner. To achieve this, the platform will comprise both state-of-the art and highly detailed animal and human anatomical models with realistic tissue property distributions that make it possible to perform simulations ranging from the molecular scale up to the complexity of the human body. @@ -72,15 +70,14 @@ Services are deployed in two stacks:``simcore-stack`` comprises all core-service To build and run: - git -- docker +- [docker](https://docs.docker.com/engine/install/ubuntu/#installation-methods) - make >=4.2 - awk, jq (optional tools within makefiles) To develop, in addition: -- python 3.10 -- nodejs for client part (this dependency will be deprecated soon) -- swagger-cli (make sure to have a recent version of nodejs) +- *python 3.10*: we recommend using the python manager [pyenv](https://brain2life.hashnode.dev/how-to-install-pyenv-python-version-manager-on-ubuntu-2004) +- *nodejs* for client part: we recommend using the node manager [nvm](https://github.com/nvm-sh/nvm#install--update-script) - [vscode] (highly recommended) To verify current base OS, Docker and Python build versions have a look at: @@ -135,7 +132,7 @@ To upgrade a single requirement named `fastapi`run: - [Git release workflow](docs/releasing-workflow-instructions.md) - Public [releases](https://github.com/ITISFoundation/osparc-simcore/releases) -- Production in https://osparc.io +- Production in - [Staging instructions](docs/releasing-workflow-instructions.md#staging-example) - [User Manual](https://itisfoundation.github.io/osparc-manual/) @@ -191,6 +188,5 @@ This project is licensed under the terms of the [MIT license](LICENSE).

-[chocolatey]:https://chocolatey.org/ [vscode]:https://code.visualstudio.com/ [WSL2]:https://docs.microsoft.com/en-us/windows/wsl From 4d75a0c3239d6d7adf203dc4cdcceedece4c7ca5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Tue, 14 Nov 2023 08:56:04 +0000 Subject: [PATCH 05/24] =?UTF-8?q?=F0=9F=90=9B=20adding=20missing=20env=20v?= =?UTF-8?q?ar=20to=20docker-compose.yaml=20(#5011)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .env-devel | 1 + docs/env-vars.md | 10 ++++++++++ services/docker-compose.yml | 1 + 3 files changed, 12 insertions(+) create mode 100644 docs/env-vars.md diff --git a/.env-devel b/.env-devel index 5bc2e21c741..f2e27bf1c32 100644 --- a/.env-devel +++ b/.env-devel @@ -47,6 +47,7 @@ DIRECTOR_V2_DEV_FEATURES_ENABLED=0 DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest} DYNAMIC_SIDECAR_LOG_LEVEL=DEBUG DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS={} +DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=[] FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "unknown@osparc.io", "affiliation": "unknown"}}' diff --git a/docs/env-vars.md b/docs/env-vars.md new file mode 100644 index 00000000000..7a389701565 --- /dev/null +++ b/docs/env-vars.md @@ -0,0 +1,10 @@ +# Environment variables management + +As a developer you will need to extend the current env vars for a service. + +The following rules must be followed: + +1. for each service that requires it, add it to the `services/docker-compose.yml` file (such as `MY_VAR=${MY_VAR}`) +2. add a meaningful default value for development inside `.env-devel` so that developers can work, and that `osparc-simcore` is **self contained**. + - **NOTE** if the variable has a default inside the code, put the same value here +3. inside the repo where devops keep all the secrets follow the instructions to add the new env var diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 34fc67d1b02..0d37c5d002d 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -198,6 +198,7 @@ services: - SIMCORE_SERVICES_NETWORK_NAME=interactive_services_subnet - TRACING_THRIFT_COMPACT_ENDPOINT=${TRACING_THRIFT_COMPACT_ENDPOINT} - DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS=${DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS} + - DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=${DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS} env_file: - ../.env volumes: From 992d8be2f37b341bce72e0d748cecf8c416978ff Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:33:00 +0100 Subject: [PATCH 06/24] =?UTF-8?q?=E2=9C=A8=20Re-introduce=20aws-library=20?= =?UTF-8?q?(#5031)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/codeql/codeql-config.yml | 1 + .github/workflows/ci-testing-deploy.yml | 47 ++ .vscode/settings.template.json | 1 + ci/github/unit-testing/aws-library.bash | 41 ++ packages/aws-library/Makefile | 49 ++ packages/aws-library/README.md | 22 + packages/aws-library/VERSION | 1 + packages/aws-library/requirements/Makefile | 6 + packages/aws-library/requirements/_base.in | 12 + packages/aws-library/requirements/_base.txt | 200 ++++++++ packages/aws-library/requirements/_test.in | 25 + packages/aws-library/requirements/_test.txt | 326 +++++++++++++ packages/aws-library/requirements/_tools.in | 5 + packages/aws-library/requirements/_tools.txt | 92 ++++ packages/aws-library/requirements/ci.txt | 20 + packages/aws-library/requirements/dev.txt | 20 + packages/aws-library/setup.cfg | 17 + packages/aws-library/setup.py | 58 +++ .../aws-library/src/aws_library/__init__.py | 3 + .../src/aws_library/ec2/__init__.py | 0 .../aws-library/src/aws_library/ec2/client.py | 284 ++++++++++++ .../aws-library/src/aws_library/ec2/errors.py | 19 + .../aws-library/src/aws_library/ec2/models.py | 76 ++++ .../aws-library/src/aws_library/ec2/utils.py | 10 + packages/aws-library/tests/conftest.py | 23 + packages/aws-library/tests/test_ec2_client.py | 430 ++++++++++++++++++ packages/aws-library/tests/test_ec2_models.py | 124 +++++ packages/aws-library/tests/test_ec2_utils.py | 7 + .../ec2_instances.py | 2 +- packages/pytest-simcore/setup.py | 1 + .../src/pytest_simcore/aioresponses_mocker.py | 2 +- .../src/pytest_simcore/aws_ec2_service.py | 149 ++++++ .../src/pytest_simcore/aws_server.py | 86 ++++ .../src/pytest_simcore/aws_services.py | 83 ---- .../src/pytest_simcore/docker_compose.py | 19 +- .../src/pytest_simcore/docker_registry.py | 2 +- .../src/pytest_simcore/docker_swarm.py | 2 +- .../pytest_simcore/helpers/utils_docker.py | 14 - .../src/pytest_simcore/helpers/utils_host.py | 14 + .../src/pytest_simcore/httpbin_service.py | 2 +- .../src/pytest_simcore/minio_service.py | 7 +- .../src/pytest_simcore/postgres_service.py | 3 +- .../src/pytest_simcore/rabbit_service.py | 3 +- .../src/pytest_simcore/redis_service.py | 17 +- .../src/pytest_simcore/simcore_services.py | 3 +- .../pytest_simcore/simcore_storage_service.py | 7 +- .../clusters_keeper/ec2_instances.py | 6 +- .../src/settings_library/ec2.py | 26 ++ packages/simcore-sdk/tests/conftest.py | 2 +- .../test_node_ports_common_file_io_utils.py | 6 +- services/agent/requirements/_base.in | 1 + services/agent/requirements/_base.txt | 2 + services/agent/requirements/_test.txt | 8 +- services/agent/tests/conftest.py | 6 +- services/api-server/tests/unit/conftest.py | 3 +- services/autoscaling/requirements/_base.in | 3 +- services/autoscaling/requirements/_base.txt | 19 +- services/autoscaling/requirements/ci.txt | 3 +- services/autoscaling/requirements/dev.txt | 1 + services/autoscaling/requirements/prod.txt | 1 + .../core/settings.py | 10 +- .../src/simcore_service_autoscaling/models.py | 59 +-- .../modules/auto_scaling_core.py | 39 +- .../modules/auto_scaling_mode_base.py | 3 +- .../auto_scaling_mode_computational.py | 10 +- .../modules/auto_scaling_mode_dynamic.py | 3 +- .../modules/dask.py | 10 +- .../modules/ec2.py | 235 +--------- .../utils/auto_scaling_core.py | 6 +- .../utils/computational_scaling.py | 9 +- .../utils/dynamic_scaling.py | 3 +- .../utils/rabbitmq.py | 3 +- .../utils/utils_docker.py | 4 +- .../utils/utils_ec2.py | 3 +- services/autoscaling/tests/unit/conftest.py | 220 ++------- .../autoscaling/tests/unit/test_api_health.py | 2 +- .../tests/unit/test_core_settings.py | 14 +- .../autoscaling/tests/unit/test_models.py | 84 +--- ...test_modules_auto_scaling_computational.py | 30 +- .../unit/test_modules_auto_scaling_dynamic.py | 27 +- .../unit/test_modules_auto_scaling_task.py | 2 +- .../tests/unit/test_modules_dask.py | 2 +- .../tests/unit/test_modules_ec2.py | 329 +------------- .../unit/test_utils_computational_scaling.py | 3 +- .../tests/unit/test_utils_docker.py | 14 +- .../tests/unit/test_utils_dynamic_scaling.py | 3 +- .../autoscaling/tests/unit/test_utils_ec2.py | 8 +- .../clusters-keeper/requirements/_base.in | 3 +- .../clusters-keeper/requirements/_base.txt | 98 +++- .../clusters-keeper/requirements/_test.txt | 4 +- services/clusters-keeper/requirements/ci.txt | 1 + services/clusters-keeper/requirements/dev.txt | 1 + .../clusters-keeper/requirements/prod.txt | 1 + services/clusters-keeper/setup.py | 1 + .../core/settings.py | 28 +- .../simcore_service_clusters_keeper/models.py | 19 - .../modules/clusters.py | 44 +- .../modules/clusters_management_core.py | 5 +- .../modules/ec2.py | 286 +----------- .../rpc/clusters.py | 2 +- .../rpc/ec2_instances.py | 12 +- .../utils/clusters.py | 10 +- .../utils/dask.py | 3 +- .../utils/ec2.py | 2 +- .../clusters-keeper/tests/unit/conftest.py | 238 ++-------- .../tests/unit/test_api_health.py | 2 +- .../tests/unit/test_modules_clusters.py | 11 +- .../test_modules_clusters_management_core.py | 8 +- .../test_modules_clusters_management_task.py | 2 +- .../tests/unit/test_modules_ec2.py | 340 +------------- .../tests/unit/test_rpc_clusters.py | 7 +- .../tests/unit/test_rpc_ec2_instances.py | 10 +- .../tests/unit/test_utils_clusters.py | 4 +- .../tests/unit/test_utils_ec2.py | 5 +- services/dask-sidecar/tests/unit/conftest.py | 2 +- .../db/repositories/comp_tasks/_utils.py | 6 +- .../02/test_dynamic_services_routes.py | 2 +- ...t_dynamic_sidecar_nodeports_integration.py | 2 +- ...ixed_dynamic_sidecar_and_legacy_project.py | 2 +- .../director-v2/tests/integration/02/utils.py | 2 +- .../with_dbs/test_api_route_computations.py | 6 +- .../unit/test_modules_outputs_watcher.py | 1 + .../tests/integration/conftest.py | 6 +- .../tests/integration/test_clusters.py | 2 +- .../tests/integration/test_dask_sidecar.py | 2 +- .../tests/integration/test_gateway.py | 2 +- .../tests/system/test_deploy.py | 2 +- services/storage/tests/conftest.py | 2 +- 128 files changed, 2681 insertions(+), 2062 deletions(-) create mode 100755 ci/github/unit-testing/aws-library.bash create mode 100644 packages/aws-library/Makefile create mode 100644 packages/aws-library/README.md create mode 100644 packages/aws-library/VERSION create mode 100644 packages/aws-library/requirements/Makefile create mode 100644 packages/aws-library/requirements/_base.in create mode 100644 packages/aws-library/requirements/_base.txt create mode 100644 packages/aws-library/requirements/_test.in create mode 100644 packages/aws-library/requirements/_test.txt create mode 100644 packages/aws-library/requirements/_tools.in create mode 100644 packages/aws-library/requirements/_tools.txt create mode 100644 packages/aws-library/requirements/ci.txt create mode 100644 packages/aws-library/requirements/dev.txt create mode 100644 packages/aws-library/setup.cfg create mode 100644 packages/aws-library/setup.py create mode 100644 packages/aws-library/src/aws_library/__init__.py create mode 100644 packages/aws-library/src/aws_library/ec2/__init__.py create mode 100644 packages/aws-library/src/aws_library/ec2/client.py create mode 100644 packages/aws-library/src/aws_library/ec2/errors.py create mode 100644 packages/aws-library/src/aws_library/ec2/models.py create mode 100644 packages/aws-library/src/aws_library/ec2/utils.py create mode 100644 packages/aws-library/tests/conftest.py create mode 100644 packages/aws-library/tests/test_ec2_client.py create mode 100644 packages/aws-library/tests/test_ec2_models.py create mode 100644 packages/aws-library/tests/test_ec2_utils.py create mode 100644 packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py create mode 100644 packages/pytest-simcore/src/pytest_simcore/aws_server.py delete mode 100644 packages/pytest-simcore/src/pytest_simcore/aws_services.py create mode 100644 packages/pytest-simcore/src/pytest_simcore/helpers/utils_host.py create mode 100644 packages/settings-library/src/settings_library/ec2.py delete mode 100644 services/clusters-keeper/src/simcore_service_clusters_keeper/models.py diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index ace67457635..adac3b13795 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -3,6 +3,7 @@ name: "ospac-simcore CodeQL config" disable-default-queries: false paths: + - packages/aws-library/src - packages/dask-task-models-library/src - packages/models-library/src/models_library - packages/postgres-database/src/simcore_postgres_database diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 4f001b7fcb6..6e184986192 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -50,6 +50,7 @@ jobs: runs-on: ubuntu-latest # Set job outputs to values from filter step outputs: + aws-library: ${{ steps.filter.outputs.aws-library }} dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }} models-library: ${{ steps.filter.outputs.models-library }} postgres-database: ${{ steps.filter.outputs.postgres-database }} @@ -87,6 +88,12 @@ jobs: id: filter with: filters: | + aws-library: + - 'packages/aws-library/**' + - 'packages/pytest-simcore/**' + - 'services/docker-compose*' + - 'scripts/mypy/*' + - 'mypy.ini' dask-task-models-library: - 'packages/dask-task-models-library/**' - 'packages/pytest-simcore/**' @@ -804,6 +811,45 @@ jobs: with: flags: unittests #optional + unit-test-aws-library: + needs: changes + if: ${{ needs.changes.outputs.aws-library == 'true' || github.event_name == 'push' }} + timeout-minutes: 18 # if this timeout gets too small, then split the tests + name: "[unit] aws-library" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.10"] + os: [ubuntu-22.04] + docker_buildx: [v0.10.4] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + version: ${{ matrix.docker_buildx }} + driver: docker-container + - name: setup python environment + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + cache: "pip" + cache-dependency-path: "packages/aws-library/requirements/ci.txt" + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - name: install + run: ./ci/github/unit-testing/aws-library.bash install + - name: typecheck + run: ./ci/github/unit-testing/aws-library.bash typecheck + - name: test + if: always() + run: ./ci/github/unit-testing/aws-library.bash test + - uses: codecov/codecov-action@v3.1.4 + with: + flags: unittests #optional + unit-test-dask-task-models-library: needs: changes if: ${{ needs.changes.outputs.dask-task-models-library == 'true' || github.event_name == 'push' }} @@ -1447,6 +1493,7 @@ jobs: unit-test-catalog, unit-test-clusters-keeper, unit-test-dask-sidecar, + unit-test-aws-library, unit-test-dask-task-models-library, unit-test-datcore-adapter, unit-test-director-v2, diff --git a/.vscode/settings.template.json b/.vscode/settings.template.json index 3a4405e9594..8df39917def 100644 --- a/.vscode/settings.template.json +++ b/.vscode/settings.template.json @@ -32,6 +32,7 @@ "python.analysis.autoImportCompletions": true, "python.analysis.typeCheckingMode": "basic", "python.analysis.extraPaths": [ + "./packages/aws-library/src", "./packages/models-library/src", "./packages/postgres-database/src", "./packages/postgres-database/tests", diff --git a/ci/github/unit-testing/aws-library.bash b/ci/github/unit-testing/aws-library.bash new file mode 100755 index 00000000000..6250656e18f --- /dev/null +++ b/ci/github/unit-testing/aws-library.bash @@ -0,0 +1,41 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +install() { + bash ci/helpers/ensure_python_pip.bash + make devenv + # shellcheck source=/dev/null + source .venv/bin/activate + pushd packages/aws-library + make install-ci + popd + .venv/bin/pip list --verbose +} + +test() { + # shellcheck source=/dev/null + source .venv/bin/activate + pushd packages/aws-library + make tests-ci + popd +} + +typecheck() { + pushd packages/aws-library + make mypy + popd +} + +# Check if the function exists (bash specific) +if declare -f "$1" >/dev/null; then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi diff --git a/packages/aws-library/Makefile b/packages/aws-library/Makefile new file mode 100644 index 00000000000..d7bf9225383 --- /dev/null +++ b/packages/aws-library/Makefile @@ -0,0 +1,49 @@ +# +# Targets for DEVELOPMENT of aws Library +# +include ../../scripts/common.Makefile +include ../../scripts/common-package.Makefile + +.PHONY: requirements +requirements: ## compiles pip requirements (.in -> .txt) + @$(MAKE_C) requirements reqs + + +.PHONY: install-dev install-prod install-ci +install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode + # installing in $(subst install-,,$@) mode + pip-sync requirements/$(subst install-,,$@).txt + + +.PHONY: tests tests-ci +tests: ## runs unit tests + # running unit tests + @pytest \ + --asyncio-mode=auto \ + --color=yes \ + --cov-config=../../.coveragerc \ + --cov-report=term-missing \ + --cov=aws_library \ + --durations=10 \ + --exitfirst \ + --failed-first \ + --pdb \ + -vv \ + $(CURDIR)/tests + +tests-ci: ## runs unit tests + # running unit tests + @pytest \ + --asyncio-mode=auto \ + --color=yes \ + --cov-append \ + --cov-config=../../.coveragerc \ + --cov-report=term-missing \ + --cov-report=xml \ + --cov=aws_library \ + --durations=10 \ + --log-date-format="%Y-%m-%d %H:%M:%S" \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --verbose \ + -m "not heavy_load" \ + $(CURDIR)/tests diff --git a/packages/aws-library/README.md b/packages/aws-library/README.md new file mode 100644 index 00000000000..c7df3095401 --- /dev/null +++ b/packages/aws-library/README.md @@ -0,0 +1,22 @@ +# simcore AWS library + +Provides a wrapper around AWS python libraries. + +Requirements to be compatible with the library: + +- only AWS-related code + + +## Installation + +```console +make help +make install-dev +``` + +## Test + +```console +make help +make test-dev +``` diff --git a/packages/aws-library/VERSION b/packages/aws-library/VERSION new file mode 100644 index 00000000000..6e8bf73aa55 --- /dev/null +++ b/packages/aws-library/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/packages/aws-library/requirements/Makefile b/packages/aws-library/requirements/Makefile new file mode 100644 index 00000000000..3f25442b790 --- /dev/null +++ b/packages/aws-library/requirements/Makefile @@ -0,0 +1,6 @@ +# +# Targets to pip-compile requirements +# +include ../../../requirements/base.Makefile + +# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in new file mode 100644 index 00000000000..a0fd39eb41f --- /dev/null +++ b/packages/aws-library/requirements/_base.in @@ -0,0 +1,12 @@ +# +# Specifies third-party dependencies for 'aws-library' +# +--constraint ../../../requirements/constraints.txt +--requirement ../../../packages/models-library/requirements/_base.in +--requirement ../../../packages/service-library/requirements/_base.in +--requirement ../../../packages/settings-library/requirements/_base.in + +aioboto3 +aiocache +pydantic[email] +types-aiobotocore[ec2] diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt new file mode 100644 index 00000000000..2e125c14ad8 --- /dev/null +++ b/packages/aws-library/requirements/_base.txt @@ -0,0 +1,200 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/_base.txt --strip-extras requirements/_base.in +# +aio-pika==9.3.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aioboto3==12.0.0 + # via -r requirements/_base.in +aiobotocore==2.7.0 + # via + # aioboto3 + # aiobotocore +aiocache==0.12.2 + # via -r requirements/_base.in +aiodebug==2.3.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiodocker==0.21.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiofiles==23.2.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiohttp==3.8.6 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # aiobotocore + # aiodocker +aioitertools==0.11.0 + # via aiobotocore +aiormq==6.7.7 + # via aio-pika +aiosignal==1.3.1 + # via aiohttp +arrow==1.3.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in +async-timeout==4.0.3 + # via + # aiohttp + # redis +attrs==23.1.0 + # via + # aiohttp + # jsonschema + # referencing +boto3==1.28.64 + # via aiobotocore +botocore==1.31.64 + # via + # aiobotocore + # boto3 + # s3transfer +botocore-stubs==1.31.85 + # via types-aiobotocore +charset-normalizer==3.3.2 + # via aiohttp +click==8.1.7 + # via typer +dnspython==2.4.2 + # via email-validator +email-validator==2.1.0.post1 + # via pydantic +frozenlist==1.4.0 + # via + # aiohttp + # aiosignal +idna==3.4 + # via + # email-validator + # yarl +jmespath==1.0.1 + # via + # boto3 + # botocore +jsonschema==4.19.2 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +jsonschema-specifications==2023.7.1 + # via jsonschema +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +multidict==6.0.4 + # via + # aiohttp + # yarl +orjson==3.9.10 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pamqp==3.2.1 + # via aiormq +pydantic==1.10.13 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/_base.in +pygments==2.16.1 + # via rich +pyinstrument==4.6.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +python-dateutil==2.8.2 + # via + # arrow + # botocore +pyyaml==6.0.1 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in +redis==5.0.1 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in +referencing==0.29.3 + # via + # -c requirements/../../../packages/service-library/requirements/./constraints.txt + # jsonschema + # jsonschema-specifications +rich==13.6.0 + # via + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +rpds-py==0.12.0 + # via + # jsonschema + # referencing +s3transfer==0.7.0 + # via boto3 +six==1.16.0 + # via python-dateutil +tenacity==8.2.3 + # via -r requirements/../../../packages/service-library/requirements/_base.in +toolz==0.12.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +tqdm==4.66.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +typer==0.9.0 + # via + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +types-aiobotocore==2.7.0 + # via -r requirements/_base.in +types-aiobotocore-ec2==2.7.0 + # via types-aiobotocore +types-awscrt==0.19.10 + # via botocore-stubs +types-python-dateutil==2.8.19.14 + # via arrow +typing-extensions==4.8.0 + # via + # aiodebug + # aiodocker + # pydantic + # typer + # types-aiobotocore + # types-aiobotocore-ec2 +urllib3==2.0.7 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # botocore +wrapt==1.16.0 + # via aiobotocore +yarl==1.9.2 + # via + # aio-pika + # aiohttp + # aiormq diff --git a/packages/aws-library/requirements/_test.in b/packages/aws-library/requirements/_test.in new file mode 100644 index 00000000000..75b32719060 --- /dev/null +++ b/packages/aws-library/requirements/_test.in @@ -0,0 +1,25 @@ +# +# Specifies dependencies required to run 'models-library' +# +--constraint ../../../requirements/constraints.txt + +# Adds base AS CONSTRAINT specs, not requirement. +# - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt +# +--constraint _base.txt + +# testing +coverage +faker +moto[server] +pint +pytest +pytest-aiohttp # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76 +pytest-cov +pytest-icdiff +pytest-instafail +pytest-mock +pytest-runner +pytest-sugar +python-dotenv +pyyaml diff --git a/packages/aws-library/requirements/_test.txt b/packages/aws-library/requirements/_test.txt new file mode 100644 index 00000000000..a588a55a40a --- /dev/null +++ b/packages/aws-library/requirements/_test.txt @@ -0,0 +1,326 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/_test.txt --strip-extras requirements/_test.in +# +aiohttp==3.8.6 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # pytest-aiohttp +aiosignal==1.3.1 + # via + # -c requirements/_base.txt + # aiohttp +async-timeout==4.0.3 + # via + # -c requirements/_base.txt + # aiohttp +attrs==23.1.0 + # via + # -c requirements/_base.txt + # aiohttp + # jschema-to-python + # jsonschema + # referencing + # sarif-om +aws-sam-translator==1.79.0 + # via cfn-lint +aws-xray-sdk==2.12.1 + # via moto +blinker==1.7.0 + # via flask +boto3==1.28.64 + # via + # -c requirements/_base.txt + # aws-sam-translator + # moto +botocore==1.31.64 + # via + # -c requirements/_base.txt + # aws-xray-sdk + # boto3 + # moto + # s3transfer +certifi==2023.7.22 + # via + # -c requirements/../../../requirements/constraints.txt + # requests +cffi==1.16.0 + # via cryptography +cfn-lint==0.83.2 + # via moto +charset-normalizer==3.3.2 + # via + # -c requirements/_base.txt + # aiohttp + # requests +click==8.1.7 + # via + # -c requirements/_base.txt + # flask +coverage==7.3.2 + # via + # -r requirements/_test.in + # pytest-cov +cryptography==41.0.5 + # via + # -c requirements/../../../requirements/constraints.txt + # moto + # python-jose + # sshpubkeys +docker==6.1.3 + # via moto +ecdsa==0.18.0 + # via + # moto + # python-jose + # sshpubkeys +exceptiongroup==1.1.3 + # via pytest +faker==20.0.0 + # via -r requirements/_test.in +flask==3.0.0 + # via + # flask-cors + # moto +flask-cors==4.0.0 + # via moto +frozenlist==1.4.0 + # via + # -c requirements/_base.txt + # aiohttp + # aiosignal +graphql-core==3.2.3 + # via moto +icdiff==2.0.7 + # via pytest-icdiff +idna==3.4 + # via + # -c requirements/_base.txt + # requests + # yarl +iniconfig==2.0.0 + # via pytest +itsdangerous==2.1.2 + # via flask +jinja2==3.1.2 + # via + # -c requirements/../../../requirements/constraints.txt + # flask + # moto +jmespath==1.0.1 + # via + # -c requirements/_base.txt + # boto3 + # botocore +jschema-to-python==1.2.3 + # via cfn-lint +jsondiff==2.0.0 + # via moto +jsonpatch==1.33 + # via cfn-lint +jsonpickle==3.0.2 + # via jschema-to-python +jsonpointer==2.4 + # via jsonpatch +jsonschema==4.19.2 + # via + # -c requirements/_base.txt + # aws-sam-translator + # cfn-lint + # openapi-schema-validator + # openapi-spec-validator +jsonschema-path==0.3.1 + # via openapi-spec-validator +jsonschema-specifications==2023.7.1 + # via + # -c requirements/_base.txt + # jsonschema + # openapi-schema-validator +junit-xml==1.9 + # via cfn-lint +lazy-object-proxy==1.9.0 + # via openapi-spec-validator +markupsafe==2.1.3 + # via + # jinja2 + # werkzeug +moto==4.2.8 + # via -r requirements/_test.in +mpmath==1.3.0 + # via sympy +multidict==6.0.4 + # via + # -c requirements/_base.txt + # aiohttp + # yarl +networkx==3.2.1 + # via cfn-lint +openapi-schema-validator==0.6.2 + # via openapi-spec-validator +openapi-spec-validator==0.7.1 + # via moto +packaging==23.2 + # via + # docker + # pytest + # pytest-sugar +pathable==0.4.3 + # via jsonschema-path +pbr==6.0.0 + # via + # jschema-to-python + # sarif-om +pint==0.22 + # via -r requirements/_test.in +pluggy==1.3.0 + # via pytest +pprintpp==0.4.0 + # via pytest-icdiff +py-partiql-parser==0.4.2 + # via moto +pyasn1==0.5.0 + # via + # python-jose + # rsa +pycparser==2.21 + # via cffi +pydantic==1.10.13 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # aws-sam-translator +pyparsing==3.1.1 + # via moto +pytest==7.4.3 + # via + # -r requirements/_test.in + # pytest-aiohttp + # pytest-asyncio + # pytest-cov + # pytest-icdiff + # pytest-instafail + # pytest-mock + # pytest-sugar +pytest-aiohttp==1.0.5 + # via -r requirements/_test.in +pytest-asyncio==0.21.1 + # via pytest-aiohttp +pytest-cov==4.1.0 + # via -r requirements/_test.in +pytest-icdiff==0.8 + # via -r requirements/_test.in +pytest-instafail==0.5.0 + # via -r requirements/_test.in +pytest-mock==3.12.0 + # via -r requirements/_test.in +pytest-runner==6.0.0 + # via -r requirements/_test.in +pytest-sugar==0.9.7 + # via -r requirements/_test.in +python-dateutil==2.8.2 + # via + # -c requirements/_base.txt + # botocore + # faker + # moto +python-dotenv==1.0.0 + # via -r requirements/_test.in +python-jose==3.3.0 + # via + # moto + # python-jose +pyyaml==6.0.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -r requirements/_test.in + # cfn-lint + # jsonschema-path + # moto + # responses +referencing==0.29.3 + # via + # -c requirements/_base.txt + # jsonschema + # jsonschema-path + # jsonschema-specifications +regex==2023.10.3 + # via cfn-lint +requests==2.31.0 + # via + # docker + # jsonschema-path + # moto + # responses +responses==0.24.0 + # via moto +rfc3339-validator==0.1.4 + # via openapi-schema-validator +rpds-py==0.12.0 + # via + # -c requirements/_base.txt + # jsonschema + # referencing +rsa==4.9 + # via + # -c requirements/../../../requirements/constraints.txt + # python-jose +s3transfer==0.7.0 + # via + # -c requirements/_base.txt + # boto3 +sarif-om==1.0.4 + # via cfn-lint +six==1.16.0 + # via + # -c requirements/_base.txt + # ecdsa + # junit-xml + # python-dateutil + # rfc3339-validator +sshpubkeys==3.3.1 + # via moto +sympy==1.12 + # via cfn-lint +termcolor==2.3.0 + # via pytest-sugar +tomli==2.0.1 + # via + # coverage + # pytest +typing-extensions==4.8.0 + # via + # -c requirements/_base.txt + # aws-sam-translator + # pint + # pydantic +urllib3==2.0.7 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # botocore + # docker + # requests + # responses +websocket-client==1.6.4 + # via docker +werkzeug==3.0.1 + # via + # flask + # moto +wrapt==1.16.0 + # via + # -c requirements/_base.txt + # aws-xray-sdk +xmltodict==0.13.0 + # via moto +yarl==1.9.2 + # via + # -c requirements/_base.txt + # aiohttp + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/aws-library/requirements/_tools.in b/packages/aws-library/requirements/_tools.in new file mode 100644 index 00000000000..1def82c12a3 --- /dev/null +++ b/packages/aws-library/requirements/_tools.in @@ -0,0 +1,5 @@ +--constraint ../../../requirements/constraints.txt +--constraint _base.txt +--constraint _test.txt + +--requirement ../../../requirements/devenv.txt diff --git a/packages/aws-library/requirements/_tools.txt b/packages/aws-library/requirements/_tools.txt new file mode 100644 index 00000000000..944a95eebf6 --- /dev/null +++ b/packages/aws-library/requirements/_tools.txt @@ -0,0 +1,92 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements/_tools.txt --strip-extras requirements/_tools.in +# +astroid==3.0.1 + # via pylint +black==23.11.0 + # via -r requirements/../../../requirements/devenv.txt +build==1.0.3 + # via pip-tools +bump2version==1.0.1 + # via -r requirements/../../../requirements/devenv.txt +cfgv==3.4.0 + # via pre-commit +click==8.1.7 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # black + # pip-tools +dill==0.3.7 + # via pylint +distlib==0.3.7 + # via virtualenv +filelock==3.13.1 + # via virtualenv +identify==2.5.31 + # via pre-commit +isort==5.12.0 + # via + # -r requirements/../../../requirements/devenv.txt + # pylint +mccabe==0.7.0 + # via pylint +mypy-extensions==1.0.0 + # via black +nodeenv==1.8.0 + # via pre-commit +packaging==23.2 + # via + # -c requirements/_test.txt + # black + # build +pathspec==0.11.2 + # via black +pip-tools==7.3.0 + # via -r requirements/../../../requirements/devenv.txt +platformdirs==3.11.0 + # via + # black + # pylint + # virtualenv +pre-commit==3.5.0 + # via -r requirements/../../../requirements/devenv.txt +pylint==3.0.2 + # via -r requirements/../../../requirements/devenv.txt +pyproject-hooks==1.0.0 + # via build +pyyaml==6.0.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -c requirements/_test.txt + # pre-commit +ruff==0.1.5 + # via -r requirements/../../../requirements/devenv.txt +tomli==2.0.1 + # via + # -c requirements/_test.txt + # black + # build + # pip-tools + # pylint + # pyproject-hooks +tomlkit==0.12.2 + # via pylint +typing-extensions==4.8.0 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # astroid + # black +virtualenv==20.24.6 + # via pre-commit +wheel==0.41.3 + # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/packages/aws-library/requirements/ci.txt b/packages/aws-library/requirements/ci.txt new file mode 100644 index 00000000000..604f51d486e --- /dev/null +++ b/packages/aws-library/requirements/ci.txt @@ -0,0 +1,20 @@ +# Shortcut to install all packages for the contigous integration (CI) of 'models-library' +# +# - As ci.txt but w/ tests +# +# Usage: +# pip install -r requirements/ci.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt + +# installs this repo's packages +../pytest-simcore/ +../service-library/ +../settings-library/ + +# current module +. diff --git a/packages/aws-library/requirements/dev.txt b/packages/aws-library/requirements/dev.txt new file mode 100644 index 00000000000..b183acca92c --- /dev/null +++ b/packages/aws-library/requirements/dev.txt @@ -0,0 +1,20 @@ +# Shortcut to install all packages needed to develop 'models-library' +# +# - As ci.txt but with current and repo packages in develop (edit) mode +# +# Usage: +# pip install -r requirements/dev.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt + +# installs this repo's packages +--editable ../pytest-simcore/ +--editable ../service-library/ +--editable ../settings-library/ + +# current module +--editable . diff --git a/packages/aws-library/setup.cfg b/packages/aws-library/setup.cfg new file mode 100644 index 00000000000..e138d387a7d --- /dev/null +++ b/packages/aws-library/setup.cfg @@ -0,0 +1,17 @@ +[bumpversion] +current_version = 0.1.0 +commit = True +message = packages/aws-library version: {current_version} → {new_version} +tag = False +commit_args = --no-verify + +[bumpversion:file:VERSION] + +[bdist_wheel] +universal = 1 + +[aliases] +test = pytest + +[tool:pytest] +asyncio_mode = auto diff --git a/packages/aws-library/setup.py b/packages/aws-library/setup.py new file mode 100644 index 00000000000..d4ffa1a620d --- /dev/null +++ b/packages/aws-library/setup.py @@ -0,0 +1,58 @@ +import re +import sys +from pathlib import Path + +from setuptools import find_packages, setup + + +def read_reqs(reqs_path: Path) -> set[str]: + return { + r + for r in re.findall( + r"(^[^#\n-][\w\[,\]]+[-~>=<.\w]*)", + reqs_path.read_text(), + re.MULTILINE, + ) + if isinstance(r, str) + } + + +CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + +INSTALL_REQUIREMENTS = tuple( + read_reqs(CURRENT_DIR / "requirements" / "_base.in") +) # WEAK requirements + +TEST_REQUIREMENTS = tuple( + read_reqs(CURRENT_DIR / "requirements" / "_test.txt") +) # STRICT requirements + + +SETUP = { + "name": "simcore-aws-library", + "version": Path(CURRENT_DIR / "VERSION").read_text().strip(), + "author": "Sylvain Anderegg (sanderegg)", + "description": "Core service library for AWS APIs", + "python_requires": "~=3.10", + "classifiers": [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3.10", + ], + "long_description": Path(CURRENT_DIR / "README.md").read_text(), + "license": "MIT license", + "install_requires": INSTALL_REQUIREMENTS, + "packages": find_packages(where="src"), + "package_dir": {"": "src"}, + "include_package_data": True, + "test_suite": "tests", + "tests_require": TEST_REQUIREMENTS, + "extras_require": {"test": TEST_REQUIREMENTS}, + "zip_safe": False, +} + + +if __name__ == "__main__": + setup(**SETUP) diff --git a/packages/aws-library/src/aws_library/__init__.py b/packages/aws-library/src/aws_library/__init__.py new file mode 100644 index 00000000000..8abb76e0ad2 --- /dev/null +++ b/packages/aws-library/src/aws_library/__init__.py @@ -0,0 +1,3 @@ +import pkg_resources + +__version__: str = pkg_resources.get_distribution("simcore-aws-library").version diff --git a/packages/aws-library/src/aws_library/ec2/__init__.py b/packages/aws-library/src/aws_library/ec2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/aws-library/src/aws_library/ec2/client.py b/packages/aws-library/src/aws_library/ec2/client.py new file mode 100644 index 00000000000..a88cf93cbd7 --- /dev/null +++ b/packages/aws-library/src/aws_library/ec2/client.py @@ -0,0 +1,284 @@ +import contextlib +import logging +from dataclasses import dataclass +from typing import cast + +import aioboto3 +import botocore.exceptions +from aiobotocore.session import ClientCreatorContext +from aiocache import cached +from pydantic import ByteSize +from servicelib.logging_utils import log_context +from settings_library.ec2 import EC2Settings +from types_aiobotocore_ec2 import EC2Client +from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType +from types_aiobotocore_ec2.type_defs import FilterTypeDef + +from .errors import ( + EC2InstanceNotFoundError, + EC2InstanceTypeInvalidError, + EC2RuntimeError, + EC2TooManyInstancesError, +) +from .models import ( + EC2InstanceConfig, + EC2InstanceData, + EC2InstanceType, + EC2Tags, + Resources, +) +from .utils import compose_user_data + +_logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class SimcoreEC2API: + client: EC2Client + session: aioboto3.Session + exit_stack: contextlib.AsyncExitStack + + @classmethod + async def create(cls, settings: EC2Settings) -> "SimcoreEC2API": + session = aioboto3.Session() + session_client = session.client( + "ec2", + endpoint_url=settings.EC2_ENDPOINT, + aws_access_key_id=settings.EC2_ACCESS_KEY_ID, + aws_secret_access_key=settings.EC2_SECRET_ACCESS_KEY, + region_name=settings.EC2_REGION_NAME, + ) + assert isinstance(session_client, ClientCreatorContext) # nosec + exit_stack = contextlib.AsyncExitStack() + ec2_client = cast( + EC2Client, await exit_stack.enter_async_context(session_client) + ) + return cls(ec2_client, session, exit_stack) + + async def close(self) -> None: + await self.exit_stack.aclose() + + async def ping(self) -> bool: + try: + await self.client.describe_account_attributes() + return True + except Exception: # pylint: disable=broad-except + return False + + @cached(noself=True) + async def get_ec2_instance_capabilities( + self, + instance_type_names: set[InstanceTypeType], + ) -> list[EC2InstanceType]: + """returns the ec2 instance types from a list of instance type names + NOTE: the order might differ! + Arguments: + instance_type_names -- the types to filter with + + Raises: + Ec2InstanceTypeInvalidError: some invalid types were used as filter + ClustersKeeperRuntimeError: unexpected error communicating with EC2 + + """ + try: + instance_types = await self.client.describe_instance_types( + InstanceTypes=list(instance_type_names) + ) + list_instances: list[EC2InstanceType] = [] + for instance in instance_types.get("InstanceTypes", []): + with contextlib.suppress(KeyError): + list_instances.append( + EC2InstanceType( + name=instance["InstanceType"], + cpus=instance["VCpuInfo"]["DefaultVCpus"], + ram=ByteSize( + int(instance["MemoryInfo"]["SizeInMiB"]) * 1024 * 1024 + ), + ) + ) + return list_instances + except botocore.exceptions.ClientError as exc: + if exc.response.get("Error", {}).get("Code", "") == "InvalidInstanceType": + raise EC2InstanceTypeInvalidError from exc + raise EC2RuntimeError from exc # pragma: no cover + + async def start_aws_instance( + self, + instance_config: EC2InstanceConfig, + number_of_instances: int, + max_number_of_instances: int = 10, + ) -> list[EC2InstanceData]: + with log_context( + _logger, + logging.INFO, + msg=f"launching {number_of_instances} AWS instance(s) {instance_config.type.name} with {instance_config.tags=}", + ): + # first check the max amount is not already reached + current_instances = await self.get_instances( + key_names=[instance_config.key_name], tags=instance_config.tags + ) + if len(current_instances) + number_of_instances > max_number_of_instances: + raise EC2TooManyInstancesError(num_instances=max_number_of_instances) + + instances = await self.client.run_instances( + ImageId=instance_config.ami_id, + MinCount=number_of_instances, + MaxCount=number_of_instances, + InstanceType=instance_config.type.name, + InstanceInitiatedShutdownBehavior="terminate", + KeyName=instance_config.key_name, + TagSpecifications=[ + { + "ResourceType": "instance", + "Tags": [ + {"Key": tag_key, "Value": tag_value} + for tag_key, tag_value in instance_config.tags.items() + ], + } + ], + UserData=compose_user_data(instance_config.startup_script), + NetworkInterfaces=[ + { + "AssociatePublicIpAddress": True, + "DeviceIndex": 0, + "SubnetId": instance_config.subnet_id, + "Groups": instance_config.security_group_ids, + } + ], + ) + instance_ids = [i["InstanceId"] for i in instances["Instances"]] + _logger.info( + "New instances launched: %s, waiting for them to start now...", + instance_ids, + ) + + # wait for the instance to be in a pending state + # NOTE: reference to EC2 states https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-lifecycle.html + waiter = self.client.get_waiter("instance_exists") + await waiter.wait(InstanceIds=instance_ids) + _logger.info("instances %s exists now.", instance_ids) + + # get the private IPs + instances = await self.client.describe_instances(InstanceIds=instance_ids) + instance_datas = [ + EC2InstanceData( + launch_time=instance["LaunchTime"], + id=instance["InstanceId"], + aws_private_dns=instance["PrivateDnsName"], + aws_public_ip=instance["PublicIpAddress"] + if "PublicIpAddress" in instance + else None, + type=instance["InstanceType"], + state=instance["State"]["Name"], + tags={tag["Key"]: tag["Value"] for tag in instance["Tags"]}, + resources=Resources( + cpus=instance_config.type.cpus, ram=instance_config.type.ram + ), + ) + for instance in instances["Reservations"][0]["Instances"] + ] + _logger.info( + "%s is available, happy computing!!", + f"{instance_datas=}", + ) + return instance_datas + + async def get_instances( + self, + *, + key_names: list[str], + tags: EC2Tags, + state_names: list[InstanceStateNameType] | None = None, + ) -> list[EC2InstanceData]: + # NOTE: be careful: Name=instance-state-name,Values=["pending", "running"] means pending OR running + # NOTE2: AND is done by repeating Name=instance-state-name,Values=pending Name=instance-state-name,Values=running + if state_names is None: + state_names = ["pending", "running"] + + filters: list[FilterTypeDef] = [ + { + "Name": "key-name", + "Values": key_names, + }, + {"Name": "instance-state-name", "Values": state_names}, + ] + filters.extend( + [{"Name": f"tag:{key}", "Values": [value]} for key, value in tags.items()] + ) + + instances = await self.client.describe_instances(Filters=filters) + all_instances = [] + for reservation in instances["Reservations"]: + assert "Instances" in reservation # nosec + for instance in reservation["Instances"]: + assert "LaunchTime" in instance # nosec + assert "InstanceId" in instance # nosec + assert "PrivateDnsName" in instance # nosec + assert "InstanceType" in instance # nosec + assert "State" in instance # nosec + assert "Name" in instance["State"] # nosec + ec2_instance_types = await self.get_ec2_instance_capabilities( + {instance["InstanceType"]} + ) + assert len(ec2_instance_types) == 1 # nosec + assert "Tags" in instance # nosec + all_instances.append( + EC2InstanceData( + launch_time=instance["LaunchTime"], + id=instance["InstanceId"], + aws_private_dns=instance["PrivateDnsName"], + aws_public_ip=instance["PublicIpAddress"] + if "PublicIpAddress" in instance + else None, + type=instance["InstanceType"], + state=instance["State"]["Name"], + resources=Resources( + cpus=ec2_instance_types[0].cpus, + ram=ec2_instance_types[0].ram, + ), + tags={tag["Key"]: tag["Value"] for tag in instance["Tags"]}, + ) + ) + _logger.debug( + "received: %s instances with %s", f"{len(all_instances)}", f"{state_names=}" + ) + return all_instances + + async def terminate_instances(self, instance_datas: list[EC2InstanceData]) -> None: + try: + with log_context( + _logger, + logging.INFO, + msg=f"terminating instances {[i.id for i in instance_datas]}", + ): + await self.client.terminate_instances( + InstanceIds=[i.id for i in instance_datas] + ) + except botocore.exceptions.ClientError as exc: + if ( + exc.response.get("Error", {}).get("Code", "") + == "InvalidInstanceID.NotFound" + ): + raise EC2InstanceNotFoundError from exc + raise # pragma: no cover + + async def set_instances_tags( + self, instances: list[EC2InstanceData], *, tags: EC2Tags + ) -> None: + try: + with log_context( + _logger, + logging.DEBUG, + msg=f"setting {tags=} on instances '[{[i.id for i in instances]}]'", + ): + await self.client.create_tags( + Resources=[i.id for i in instances], + Tags=[ + {"Key": tag_key, "Value": tag_value} + for tag_key, tag_value in tags.items() + ], + ) + except botocore.exceptions.ClientError as exc: + if exc.response.get("Error", {}).get("Code", "") == "InvalidID": + raise EC2InstanceNotFoundError from exc + raise # pragma: no cover diff --git a/packages/aws-library/src/aws_library/ec2/errors.py b/packages/aws-library/src/aws_library/ec2/errors.py new file mode 100644 index 00000000000..2253fd17680 --- /dev/null +++ b/packages/aws-library/src/aws_library/ec2/errors.py @@ -0,0 +1,19 @@ +from pydantic.errors import PydanticErrorMixin + + +class EC2RuntimeError(PydanticErrorMixin, RuntimeError): + msg_template: str = "EC2 client unexpected error" + + +class EC2InstanceNotFoundError(EC2RuntimeError): + msg_template: str = "EC2 instance was not found" + + +class EC2InstanceTypeInvalidError(EC2RuntimeError): + msg_template: str = "EC2 instance type invalid" + + +class EC2TooManyInstancesError(EC2RuntimeError): + msg_template: str = ( + "The maximum amount of instances {num_instances} is already reached!" + ) diff --git a/packages/aws-library/src/aws_library/ec2/models.py b/packages/aws-library/src/aws_library/ec2/models.py new file mode 100644 index 00000000000..d124763cebd --- /dev/null +++ b/packages/aws-library/src/aws_library/ec2/models.py @@ -0,0 +1,76 @@ +import datetime +from dataclasses import dataclass +from typing import TypeAlias + +from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt +from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType + + +class Resources(BaseModel): + cpus: NonNegativeFloat + ram: ByteSize + + @classmethod + def create_as_empty(cls) -> "Resources": + return cls(cpus=0, ram=ByteSize(0)) + + def __ge__(self, other: "Resources") -> bool: + return self.cpus >= other.cpus and self.ram >= other.ram + + def __gt__(self, other: "Resources") -> bool: + return self.cpus > other.cpus or self.ram > other.ram + + def __add__(self, other: "Resources") -> "Resources": + return Resources.construct( + **{ + key: a + b + for (key, a), b in zip( + self.dict().items(), other.dict().values(), strict=True + ) + } + ) + + def __sub__(self, other: "Resources") -> "Resources": + return Resources.construct( + **{ + key: a - b + for (key, a), b in zip( + self.dict().items(), other.dict().values(), strict=True + ) + } + ) + + +@dataclass(frozen=True) +class EC2InstanceType: + name: InstanceTypeType + cpus: PositiveInt + ram: ByteSize + + +InstancePrivateDNSName: TypeAlias = str +EC2Tags: TypeAlias = dict[str, str] + + +@dataclass(frozen=True) +class EC2InstanceData: + launch_time: datetime.datetime + id: str # noqa: A003 + aws_private_dns: InstancePrivateDNSName + aws_public_ip: str | None + type: InstanceTypeType # noqa: A003 + state: InstanceStateNameType + resources: Resources + tags: EC2Tags + + +@dataclass(frozen=True) +class EC2InstanceConfig: + type: EC2InstanceType + tags: EC2Tags + startup_script: str + + ami_id: str + key_name: str + security_group_ids: list[str] + subnet_id: str diff --git a/packages/aws-library/src/aws_library/ec2/utils.py b/packages/aws-library/src/aws_library/ec2/utils.py new file mode 100644 index 00000000000..43e27fe1b6d --- /dev/null +++ b/packages/aws-library/src/aws_library/ec2/utils.py @@ -0,0 +1,10 @@ +from textwrap import dedent + + +def compose_user_data(docker_join_bash_command: str) -> str: + return dedent( + f"""\ +#!/bin/bash +{docker_join_bash_command} +""" + ) diff --git a/packages/aws-library/tests/conftest.py b/packages/aws-library/tests/conftest.py new file mode 100644 index 00000000000..9d04f458812 --- /dev/null +++ b/packages/aws-library/tests/conftest.py @@ -0,0 +1,23 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-import +from pathlib import Path + +import aws_library +import pytest + +pytest_plugins = [ + "pytest_simcore.aws_server", + "pytest_simcore.aws_ec2_service", + "pytest_simcore.environment_configs", + "pytest_simcore.repository_paths", + "pytest_simcore.pydantic_models", + "pytest_simcore.pytest_global_environs", +] + + +@pytest.fixture(scope="session") +def package_dir() -> Path: + pdir = Path(aws_library.__file__).resolve().parent + assert pdir.exists() + return pdir diff --git a/packages/aws-library/tests/test_ec2_client.py b/packages/aws-library/tests/test_ec2_client.py new file mode 100644 index 00000000000..1e85be088d1 --- /dev/null +++ b/packages/aws-library/tests/test_ec2_client.py @@ -0,0 +1,430 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + + +from collections.abc import AsyncIterator, Callable +from typing import cast, get_args + +import botocore.exceptions +import pytest +from aws_library.ec2.client import SimcoreEC2API +from aws_library.ec2.errors import ( + EC2InstanceNotFoundError, + EC2InstanceTypeInvalidError, + EC2TooManyInstancesError, +) +from aws_library.ec2.models import ( + EC2InstanceConfig, + EC2InstanceData, + EC2InstanceType, + EC2Tags, +) +from faker import Faker +from moto.server import ThreadedMotoServer +from settings_library.ec2 import EC2Settings +from types_aiobotocore_ec2 import EC2Client +from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType + + +def _ec2_allowed_types() -> list[InstanceTypeType]: + return ["t2.nano", "m5.12xlarge", "g4dn.4xlarge"] + + +@pytest.fixture(scope="session") +def ec2_allowed_instances() -> list[InstanceTypeType]: + return _ec2_allowed_types() + + +@pytest.fixture +async def simcore_ec2_api( + mocked_ec2_server_settings: EC2Settings, +) -> AsyncIterator[SimcoreEC2API]: + ec2 = await SimcoreEC2API.create(settings=mocked_ec2_server_settings) + assert ec2 + assert ec2.client + assert ec2.exit_stack + assert ec2.session + yield ec2 + await ec2.close() + + +async def test_ec2_client_lifespan(simcore_ec2_api: SimcoreEC2API): + ... + + +async def test_aiobotocore_ec2_client_when_ec2_server_goes_up_and_down( + mocked_aws_server: ThreadedMotoServer, + ec2_client: EC2Client, +): + # passes without exception + await ec2_client.describe_account_attributes(DryRun=True) + mocked_aws_server.stop() + with pytest.raises(botocore.exceptions.EndpointConnectionError): + await ec2_client.describe_account_attributes(DryRun=True) + + # restart + mocked_aws_server.start() + # passes without exception + await ec2_client.describe_account_attributes(DryRun=True) + + +async def test_ping( + mocked_aws_server: ThreadedMotoServer, + simcore_ec2_api: SimcoreEC2API, +): + assert await simcore_ec2_api.ping() is True + mocked_aws_server.stop() + assert await simcore_ec2_api.ping() is False + mocked_aws_server.start() + assert await simcore_ec2_api.ping() is True + + +@pytest.fixture +def ec2_instance_config( + fake_ec2_instance_type: EC2InstanceType, + faker: Faker, + aws_subnet_id: str, + aws_security_group_id: str, + aws_ami_id: str, +) -> EC2InstanceConfig: + return EC2InstanceConfig( + type=fake_ec2_instance_type, + tags=faker.pydict(allowed_types=(str,)), + startup_script=faker.pystr(), + ami_id=aws_ami_id, + key_name=faker.pystr(), + security_group_ids=[aws_security_group_id], + subnet_id=aws_subnet_id, + ) + + +async def test_get_ec2_instance_capabilities( + simcore_ec2_api: SimcoreEC2API, + ec2_allowed_instances: list[InstanceTypeType], +): + instance_types: list[ + EC2InstanceType + ] = await simcore_ec2_api.get_ec2_instance_capabilities( + cast( + set[InstanceTypeType], + set(ec2_allowed_instances), + ) + ) + assert instance_types + assert len(instance_types) == len(ec2_allowed_instances) + + # all the instance names are found and valid + assert all(i.name in ec2_allowed_instances for i in instance_types) + for instance_type_name in ec2_allowed_instances: + assert any(i.name == instance_type_name for i in instance_types) + + +async def test_get_ec2_instance_capabilities_empty_list_returns_all_options( + simcore_ec2_api: SimcoreEC2API, +): + instance_types = await simcore_ec2_api.get_ec2_instance_capabilities(set()) + assert instance_types + # NOTE: this might need adaptation when moto is updated + assert 700 < len(instance_types) < 800 + + +async def test_get_ec2_instance_capabilities_with_invalid_type_raises( + simcore_ec2_api: SimcoreEC2API, + faker: Faker, +): + with pytest.raises(EC2InstanceTypeInvalidError): + await simcore_ec2_api.get_ec2_instance_capabilities(set(faker.pystr())) + + +@pytest.fixture(params=_ec2_allowed_types()) +async def fake_ec2_instance_type( + simcore_ec2_api: SimcoreEC2API, + request: pytest.FixtureRequest, +) -> EC2InstanceType: + instance_type_name: InstanceTypeType = request.param + instance_types: list[ + EC2InstanceType + ] = await simcore_ec2_api.get_ec2_instance_capabilities({instance_type_name}) + + assert len(instance_types) == 1 + return instance_types[0] + + +async def _assert_no_instances_in_ec2(ec2_client: EC2Client) -> None: + all_instances = await ec2_client.describe_instances() + assert not all_instances["Reservations"] + + +async def _assert_instances_in_ec2( + ec2_client: EC2Client, + *, + expected_num_reservations: int, + expected_num_instances: int, + expected_instance_type: EC2InstanceType, + expected_tags: EC2Tags, + expected_state: str, +) -> None: + all_instances = await ec2_client.describe_instances() + assert len(all_instances["Reservations"]) == expected_num_reservations + for reservation in all_instances["Reservations"]: + assert "Instances" in reservation + assert len(reservation["Instances"]) == expected_num_instances + for instance in reservation["Instances"]: + assert "InstanceType" in instance + assert instance["InstanceType"] == expected_instance_type.name + assert "Tags" in instance + assert instance["Tags"] == [ + {"Key": key, "Value": value} for key, value in expected_tags.items() + ] + assert "State" in instance + assert "Name" in instance["State"] + assert instance["State"]["Name"] == expected_state + + +async def test_start_aws_instance( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + ec2_instance_config: EC2InstanceConfig, +): + await _assert_no_instances_in_ec2(ec2_client) + + number_of_instances = 1 + + # let's create a first reservation and check that it is correctly created in EC2 + await simcore_ec2_api.start_aws_instance( + ec2_instance_config, number_of_instances=number_of_instances + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=number_of_instances, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, + expected_state="running", + ) + + # create a second reservation + await simcore_ec2_api.start_aws_instance( + ec2_instance_config, number_of_instances=number_of_instances + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=2, + expected_num_instances=number_of_instances, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, + expected_state="running", + ) + + +@pytest.mark.parametrize("max_num_instances", [13]) +async def test_start_aws_instance_is_limited_in_number_of_instances( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + ec2_instance_config: EC2InstanceConfig, + max_num_instances: int, +): + await _assert_no_instances_in_ec2(ec2_client) + + # create many instances in one go shall fail + with pytest.raises(EC2TooManyInstancesError): + await simcore_ec2_api.start_aws_instance( + ec2_instance_config, + number_of_instances=max_num_instances + 1, + max_number_of_instances=max_num_instances, + ) + await _assert_no_instances_in_ec2(ec2_client) + + # create instances 1 by 1 + for _ in range(max_num_instances): + await simcore_ec2_api.start_aws_instance( + ec2_instance_config, + number_of_instances=1, + max_number_of_instances=max_num_instances, + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=max_num_instances, + expected_num_instances=1, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, + expected_state="running", + ) + + # now creating one more shall fail + with pytest.raises(EC2TooManyInstancesError): + await simcore_ec2_api.start_aws_instance( + ec2_instance_config, + number_of_instances=1, + max_number_of_instances=max_num_instances, + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=max_num_instances, + expected_num_instances=1, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, + expected_state="running", + ) + + +async def test_get_instances( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + faker: Faker, + ec2_instance_config: EC2InstanceConfig, +): + # we have nothing running now in ec2 + await _assert_no_instances_in_ec2(ec2_client) + assert ( + await simcore_ec2_api.get_instances( + key_names=[ec2_instance_config.key_name], tags={} + ) + == [] + ) + + # create some instance + _MAX_NUM_INSTANCES = 10 + num_instances = faker.pyint(min_value=1, max_value=_MAX_NUM_INSTANCES) + created_instances = await simcore_ec2_api.start_aws_instance( + ec2_instance_config, number_of_instances=num_instances + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, + expected_state="running", + ) + # this returns all the entries using thes key names + instance_received = await simcore_ec2_api.get_instances( + key_names=[ec2_instance_config.key_name], tags={} + ) + assert created_instances == instance_received + + # passing the tags will return the same + instance_received = await simcore_ec2_api.get_instances( + key_names=[ec2_instance_config.key_name], tags=ec2_instance_config.tags + ) + assert created_instances == instance_received + + # asking for running state will also return the same + instance_received = await simcore_ec2_api.get_instances( + key_names=[ec2_instance_config.key_name], + tags=ec2_instance_config.tags, + state_names=["running"], + ) + assert created_instances == instance_received + + # asking for other states shall return nothing + for state in get_args(InstanceStateNameType): + instance_received = await simcore_ec2_api.get_instances( + key_names=[ec2_instance_config.key_name], + tags=ec2_instance_config.tags, + state_names=[state], + ) + if state == "running": + assert created_instances == instance_received + else: + assert not instance_received + + +async def test_terminate_instance( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + faker: Faker, + ec2_instance_config: EC2InstanceConfig, +): + # we have nothing running now in ec2 + await _assert_no_instances_in_ec2(ec2_client) + # create some instance + _NUM_INSTANCES = 10 + num_instances = faker.pyint(min_value=1, max_value=_NUM_INSTANCES) + created_instances = await simcore_ec2_api.start_aws_instance( + ec2_instance_config, number_of_instances=num_instances + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, + expected_state="running", + ) + + # terminate the instance + await simcore_ec2_api.terminate_instances(created_instances) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, + expected_state="terminated", + ) + # calling it several times is ok, the instance stays a while + await simcore_ec2_api.terminate_instances(created_instances) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, + expected_state="terminated", + ) + + +async def test_terminate_instance_not_existing_raises( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + fake_ec2_instance_data: Callable[..., EC2InstanceData], +): + await _assert_no_instances_in_ec2(ec2_client) + with pytest.raises(EC2InstanceNotFoundError): + await simcore_ec2_api.terminate_instances([fake_ec2_instance_data()]) + + +async def test_set_instance_tags( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + faker: Faker, + ec2_instance_config: EC2InstanceConfig, +): + await _assert_no_instances_in_ec2(ec2_client) + # create some instance + _MAX_NUM_INSTANCES = 10 + num_instances = faker.pyint(min_value=1, max_value=_MAX_NUM_INSTANCES) + created_instances = await simcore_ec2_api.start_aws_instance( + ec2_instance_config, number_of_instances=num_instances + ) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags, + expected_state="running", + ) + + new_tags = faker.pydict(allowed_types=(str,)) + await simcore_ec2_api.set_instances_tags(created_instances, tags=new_tags) + await _assert_instances_in_ec2( + ec2_client, + expected_num_reservations=1, + expected_num_instances=num_instances, + expected_instance_type=ec2_instance_config.type, + expected_tags=ec2_instance_config.tags | new_tags, + expected_state="running", + ) + + +async def test_set_instance_tags_not_existing_raises( + simcore_ec2_api: SimcoreEC2API, + ec2_client: EC2Client, + fake_ec2_instance_data: Callable[..., EC2InstanceData], +): + await _assert_no_instances_in_ec2(ec2_client) + with pytest.raises(EC2InstanceNotFoundError): + await simcore_ec2_api.set_instances_tags([fake_ec2_instance_data()], tags={}) diff --git a/packages/aws-library/tests/test_ec2_models.py b/packages/aws-library/tests/test_ec2_models.py new file mode 100644 index 00000000000..aac4a1d7863 --- /dev/null +++ b/packages/aws-library/tests/test_ec2_models.py @@ -0,0 +1,124 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +import pytest +from aws_library.ec2.models import Resources +from pydantic import ByteSize + + +@pytest.mark.parametrize( + "a,b,a_greater_or_equal_than_b", + [ + ( + Resources(cpus=0.2, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.1, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.1, ram=ByteSize(1)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.05, ram=ByteSize(1)), + Resources(cpus=0.1, ram=ByteSize(0)), + False, + ), + ( + Resources(cpus=0.1, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(1)), + False, + ), + ], +) +def test_resources_ge_operator( + a: Resources, b: Resources, a_greater_or_equal_than_b: bool +): + assert (a >= b) is a_greater_or_equal_than_b + + +@pytest.mark.parametrize( + "a,b,a_greater_than_b", + [ + ( + Resources(cpus=0.2, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.1, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(0)), + False, + ), + ( + Resources(cpus=0.1, ram=ByteSize(1)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.05, ram=ByteSize(1)), + Resources(cpus=0.1, ram=ByteSize(0)), + True, + ), + ( + Resources(cpus=0.1, ram=ByteSize(0)), + Resources(cpus=0.1, ram=ByteSize(1)), + False, + ), + ], +) +def test_resources_gt_operator(a: Resources, b: Resources, a_greater_than_b: bool): + assert (a > b) is a_greater_than_b + + +@pytest.mark.parametrize( + "a,b,result", + [ + ( + Resources(cpus=0, ram=ByteSize(0)), + Resources(cpus=1, ram=ByteSize(34)), + Resources(cpus=1, ram=ByteSize(34)), + ), + ( + Resources(cpus=0.1, ram=ByteSize(-1)), + Resources(cpus=1, ram=ByteSize(34)), + Resources(cpus=1.1, ram=ByteSize(33)), + ), + ], +) +def test_resources_add(a: Resources, b: Resources, result: Resources): + assert a + b == result + a += b + assert a == result + + +def test_resources_create_as_empty(): + assert Resources.create_as_empty() == Resources(cpus=0, ram=ByteSize(0)) + + +@pytest.mark.parametrize( + "a,b,result", + [ + ( + Resources(cpus=0, ram=ByteSize(0)), + Resources(cpus=1, ram=ByteSize(34)), + Resources.construct(cpus=-1, ram=ByteSize(-34)), + ), + ( + Resources(cpus=0.1, ram=ByteSize(-1)), + Resources(cpus=1, ram=ByteSize(34)), + Resources.construct(cpus=-0.9, ram=ByteSize(-35)), + ), + ], +) +def test_resources_sub(a: Resources, b: Resources, result: Resources): + assert a - b == result + a -= b + assert a == result diff --git a/packages/aws-library/tests/test_ec2_utils.py b/packages/aws-library/tests/test_ec2_utils.py new file mode 100644 index 00000000000..ce03ef066b9 --- /dev/null +++ b/packages/aws-library/tests/test_ec2_utils.py @@ -0,0 +1,7 @@ +from aws_library.ec2.utils import compose_user_data +from faker import Faker + + +def test_compose_user_data(faker: Faker): + assert compose_user_data(faker.pystr()).startswith("#!/bin/bash\n") + assert compose_user_data(faker.pystr()).endswith("\n") diff --git a/packages/models-library/src/models_library/api_schemas_clusters_keeper/ec2_instances.py b/packages/models-library/src/models_library/api_schemas_clusters_keeper/ec2_instances.py index 373a12af823..16f47d2a3fd 100644 --- a/packages/models-library/src/models_library/api_schemas_clusters_keeper/ec2_instances.py +++ b/packages/models-library/src/models_library/api_schemas_clusters_keeper/ec2_instances.py @@ -4,7 +4,7 @@ @dataclass(frozen=True) -class EC2InstanceType: +class EC2InstanceTypeGet: name: str cpus: PositiveInt ram: ByteSize diff --git a/packages/pytest-simcore/setup.py b/packages/pytest-simcore/setup.py index 61874d47ea8..b9ee764e695 100644 --- a/packages/pytest-simcore/setup.py +++ b/packages/pytest-simcore/setup.py @@ -25,6 +25,7 @@ "aiohttp", "aioredis", "docker", + "moto[server]", "python-socketio", "PyYAML", "sqlalchemy[postgresql_psycopg2binary]", diff --git a/packages/pytest-simcore/src/pytest_simcore/aioresponses_mocker.py b/packages/pytest-simcore/src/pytest_simcore/aioresponses_mocker.py index 11beaa7e325..c91edc673cb 100644 --- a/packages/pytest-simcore/src/pytest_simcore/aioresponses_mocker.py +++ b/packages/pytest-simcore/src/pytest_simcore/aioresponses_mocker.py @@ -1,7 +1,7 @@ import pytest from aioresponses import aioresponses as AioResponsesMock -from .helpers.utils_docker import get_localhost_ip +from .helpers.utils_host import get_localhost_ip # WARNING: any request done through the client will go through aioresponses. It is # unfortunate but that means any valid request (like calling the test server) prefix must be set as passthrough. diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py b/packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py new file mode 100644 index 00000000000..641664a5292 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/aws_ec2_service.py @@ -0,0 +1,149 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-import + +import contextlib +import datetime +import random +from collections.abc import AsyncIterator, Callable +from typing import cast + +import aioboto3 +import pytest +from aiobotocore.session import ClientCreatorContext +from aws_library.ec2.models import EC2InstanceData, Resources +from faker import Faker +from pydantic import ByteSize +from settings_library.ec2 import EC2Settings +from types_aiobotocore_ec2.client import EC2Client + + +@pytest.fixture +async def ec2_client( + mocked_ec2_server_settings: EC2Settings, +) -> AsyncIterator[EC2Client]: + session = aioboto3.Session() + exit_stack = contextlib.AsyncExitStack() + session_client = session.client( + "ec2", + endpoint_url=mocked_ec2_server_settings.EC2_ENDPOINT, + aws_access_key_id=mocked_ec2_server_settings.EC2_ACCESS_KEY_ID, + aws_secret_access_key=mocked_ec2_server_settings.EC2_SECRET_ACCESS_KEY, + region_name=mocked_ec2_server_settings.EC2_REGION_NAME, + ) + assert isinstance(session_client, ClientCreatorContext) + ec2_client = cast(EC2Client, await exit_stack.enter_async_context(session_client)) + + yield ec2_client + + await exit_stack.aclose() + + +@pytest.fixture(scope="session") +def vpc_cidr_block() -> str: + return "10.0.0.0/16" + + +@pytest.fixture +async def aws_vpc_id( + ec2_client: EC2Client, + vpc_cidr_block: str, +) -> AsyncIterator[str]: + vpc = await ec2_client.create_vpc( + CidrBlock=vpc_cidr_block, + ) + vpc_id = vpc["Vpc"]["VpcId"] # type: ignore + print(f"--> Created Vpc in AWS with {vpc_id=}") + yield vpc_id + + await ec2_client.delete_vpc(VpcId=vpc_id) + print(f"<-- Deleted Vpc in AWS with {vpc_id=}") + + +@pytest.fixture(scope="session") +def subnet_cidr_block() -> str: + return "10.0.1.0/24" + + +@pytest.fixture +async def aws_subnet_id( + aws_vpc_id: str, + ec2_client: EC2Client, + subnet_cidr_block: str, +) -> AsyncIterator[str]: + subnet = await ec2_client.create_subnet( + CidrBlock=subnet_cidr_block, VpcId=aws_vpc_id + ) + assert "Subnet" in subnet + assert "SubnetId" in subnet["Subnet"] + subnet_id = subnet["Subnet"]["SubnetId"] + print(f"--> Created Subnet in AWS with {subnet_id=}") + + yield subnet_id + + # all the instances in the subnet must be terminated before that works + instances_in_subnet = await ec2_client.describe_instances( + Filters=[{"Name": "subnet-id", "Values": [subnet_id]}] + ) + if instances_in_subnet["Reservations"]: + print(f"--> terminating {len(instances_in_subnet)} instances in subnet") + await ec2_client.terminate_instances( + InstanceIds=[ + instance["Instances"][0]["InstanceId"] # type: ignore + for instance in instances_in_subnet["Reservations"] + ] + ) + print(f"<-- terminated {len(instances_in_subnet)} instances in subnet") + + await ec2_client.delete_subnet(SubnetId=subnet_id) + subnets = await ec2_client.describe_subnets() + print(f"<-- Deleted Subnet in AWS with {subnet_id=}") + print(f"current {subnets=}") + + +@pytest.fixture +async def aws_security_group_id( + faker: Faker, + aws_vpc_id: str, + ec2_client: EC2Client, +) -> AsyncIterator[str]: + security_group = await ec2_client.create_security_group( + Description=faker.text(), GroupName=faker.pystr(), VpcId=aws_vpc_id + ) + security_group_id = security_group["GroupId"] + print(f"--> Created Security Group in AWS with {security_group_id=}") + yield security_group_id + await ec2_client.delete_security_group(GroupId=security_group_id) + print(f"<-- Deleted Security Group in AWS with {security_group_id=}") + + +@pytest.fixture +async def aws_ami_id( + ec2_client: EC2Client, +) -> str: + images = await ec2_client.describe_images() + image = random.choice(images["Images"]) # noqa: S311 + assert "ImageId" in image + return image["ImageId"] + + +@pytest.fixture +def fake_ec2_instance_data(faker: Faker) -> Callable[..., EC2InstanceData]: + def _creator(**overrides) -> EC2InstanceData: + return EC2InstanceData( + **( + { + "launch_time": faker.date_time(tzinfo=datetime.timezone.utc), + "id": faker.uuid4(), + "aws_private_dns": f"ip-{faker.ipv4().replace('.', '-')}.ec2.internal", + "aws_public_ip": faker.ipv4(), + "type": faker.pystr(), + "state": faker.pystr(), + "resources": Resources(cpus=4.0, ram=ByteSize(1024 * 1024)), + "tags": faker.pydict(allowed_types=(str,)), + } + | overrides + ) + ) + + return _creator diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_server.py b/packages/pytest-simcore/src/pytest_simcore/aws_server.py new file mode 100644 index 00000000000..48baff0e4c2 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/aws_server.py @@ -0,0 +1,86 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-import + +from collections.abc import Iterator + +import pytest +import requests +from aiohttp.test_utils import unused_port +from moto.server import ThreadedMotoServer +from settings_library.ec2 import EC2Settings + +from .helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from .helpers.utils_host import get_localhost_ip + + +@pytest.fixture(scope="module") +def mocked_aws_server() -> Iterator[ThreadedMotoServer]: + """creates a moto-server that emulates AWS services in place + NOTE: Never use a bucket with underscores it fails!! + """ + server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port()) + # pylint: disable=protected-access + print( + f"--> started mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 + ) + print( + f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]" # noqa: SLF001 + ) + server.start() + yield server + server.stop() + print( + f"<-- stopped mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 + ) + + +@pytest.fixture +def reset_aws_server_state(mocked_aws_server: ThreadedMotoServer) -> Iterator[None]: + # NOTE: reset_aws_server_state [http://docs.getmoto.org/en/latest/docs/server_mode.html#reset-api] + yield + # pylint: disable=protected-access + requests.post( + f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}/moto-api/reset", # noqa: SLF001 + timeout=10, + ) + print( + f"<-- cleaned mock AWS server on {mocked_aws_server._ip_address}:{mocked_aws_server._port}" # noqa: SLF001 + ) + + +@pytest.fixture +def mocked_ec2_server_settings( + mocked_aws_server: ThreadedMotoServer, + reset_aws_server_state: None, +) -> EC2Settings: + return EC2Settings( + EC2_ACCESS_KEY_ID="xxx", + EC2_ENDPOINT=f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 + EC2_SECRET_ACCESS_KEY="xxx", # noqa: S106 + ) + + +@pytest.fixture +def mocked_ec2_server_envs( + mocked_ec2_server_settings: EC2Settings, + monkeypatch: pytest.MonkeyPatch, +) -> EnvVarsDict: + changed_envs: EnvVarsDict = mocked_ec2_server_settings.dict() + return setenvs_from_dict(monkeypatch, changed_envs) + + +@pytest.fixture +async def mocked_s3_server_envs( + mocked_aws_server: ThreadedMotoServer, + reset_aws_server_state: None, + monkeypatch: pytest.MonkeyPatch, +) -> EnvVarsDict: + changed_envs = { + "S3_SECURE": "false", + "S3_ENDPOINT": f"{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 + "S3_ACCESS_KEY": "xxx", + "S3_SECRET_KEY": "xxx", + "S3_BUCKET_NAME": "pytestbucket", + } + return setenvs_from_dict(monkeypatch, changed_envs) diff --git a/packages/pytest-simcore/src/pytest_simcore/aws_services.py b/packages/pytest-simcore/src/pytest_simcore/aws_services.py deleted file mode 100644 index b96bbfd6c98..00000000000 --- a/packages/pytest-simcore/src/pytest_simcore/aws_services.py +++ /dev/null @@ -1,83 +0,0 @@ -# pylint: disable=protected-access -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable - -import asyncio -from typing import AsyncIterator, Iterator - -import pytest -from aiobotocore.session import get_session -from aiohttp.test_utils import unused_port -from moto.server import ThreadedMotoServer -from pytest_simcore.helpers.utils_docker import get_localhost_ip - - -@pytest.fixture(scope="module") -def mocked_s3_server() -> Iterator[ThreadedMotoServer]: - """creates a moto-server that emulates AWS services in place - NOTE: Never use a bucket with underscores it fails!! - """ - server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port()) - # pylint: disable=protected-access - print(f"--> started mock S3 server on {server._ip_address}:{server._port}") - print( - f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]" - ) - server.start() - yield server - server.stop() - print(f"<-- stopped mock S3 server on {server._ip_address}:{server._port}") - - -async def _clean_bucket_content(aiobotore_s3_client, bucket: str): - response = await aiobotore_s3_client.list_objects_v2(Bucket=bucket) - while response["KeyCount"] > 0: - await aiobotore_s3_client.delete_objects( - Bucket=bucket, - Delete={ - "Objects": [ - {"Key": obj["Key"]} for obj in response["Contents"] if "Key" in obj - ] - }, - ) - response = await aiobotore_s3_client.list_objects_v2(Bucket=bucket) - - -async def _remove_all_buckets(aiobotore_s3_client): - response = await aiobotore_s3_client.list_buckets() - bucket_names = [ - bucket["Name"] for bucket in response["Buckets"] if "Name" in bucket - ] - await asyncio.gather( - *(_clean_bucket_content(aiobotore_s3_client, bucket) for bucket in bucket_names) - ) - await asyncio.gather( - *(aiobotore_s3_client.delete_bucket(Bucket=bucket) for bucket in bucket_names) - ) - - -@pytest.fixture -async def mocked_s3_server_envs( - mocked_s3_server: ThreadedMotoServer, monkeypatch: pytest.MonkeyPatch -) -> AsyncIterator[None]: - monkeypatch.setenv("S3_SECURE", "false") - monkeypatch.setenv( - "S3_ENDPOINT", - f"{mocked_s3_server._ip_address}:{mocked_s3_server._port}", # pylint: disable=protected-access - ) - monkeypatch.setenv("S3_ACCESS_KEY", "xxx") - monkeypatch.setenv("S3_SECRET_KEY", "xxx") - monkeypatch.setenv("S3_BUCKET_NAME", "pytestbucket") - - yield - - # cleanup the buckets - session = get_session() - async with session.create_client( - "s3", - endpoint_url=f"http://{mocked_s3_server._ip_address}:{mocked_s3_server._port}", # pylint: disable=protected-access - aws_secret_access_key="xxx", - aws_access_key_id="xxx", - ) as client: - await _remove_all_buckets(client) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index db34d158938..424f6bb7da2 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -15,9 +15,10 @@ import shutil import subprocess import sys +from collections.abc import Iterator from copy import deepcopy from pathlib import Path -from typing import Any, Iterator +from typing import Any import pytest import yaml @@ -30,11 +31,8 @@ ) from .helpers.constants import HEADER_STR from .helpers.typing_env import EnvVarsDict -from .helpers.utils_docker import ( - get_localhost_ip, - run_docker_compose_config, - save_docker_infos, -) +from .helpers.utils_docker import run_docker_compose_config, save_docker_infos +from .helpers.utils_host import get_localhost_ip @pytest.fixture(scope="session") @@ -79,9 +77,9 @@ def testing_environ_vars(env_devel_file: Path) -> EnvVarsDict: env_devel["API_SERVER_DEV_FEATURES_ENABLED"] = "1" - if not "DOCKER_REGISTRY" in os.environ: + if "DOCKER_REGISTRY" not in os.environ: env_devel["DOCKER_REGISTRY"] = "local" - if not "DOCKER_IMAGE_TAG" in os.environ: + if "DOCKER_IMAGE_TAG" not in os.environ: env_devel["DOCKER_IMAGE_TAG"] = "production" return {key: value for key, value in env_devel.items() if value is not None} @@ -264,8 +262,7 @@ def core_docker_compose_file( @pytest.fixture(scope="module") def ops_services_selection(request) -> list[str]: """Selection of services from the ops stack""" - ops_services = getattr(request.module, FIXTURE_CONFIG_OPS_SERVICES_SELECTION, []) - return ops_services + return getattr(request.module, FIXTURE_CONFIG_OPS_SERVICES_SELECTION, []) @pytest.fixture(scope="module") @@ -370,7 +367,7 @@ def _filter_services_and_dump( with docker_compose_path.open("wt") as fh: if "TRAVIS" in os.environ: # in travis we do not have access to file - print(f"{str(docker_compose_path):-^100}") + print(f"{docker_compose_path!s:-^100}") yaml.dump(content, sys.stdout, default_flow_style=False) print("-" * 100) else: diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py index e063e23adb2..d1627698946 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py @@ -15,7 +15,7 @@ import tenacity from pytest import FixtureRequest -from .helpers.utils_docker import get_localhost_ip +from .helpers.utils_host import get_localhost_ip log = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py index 37b8c0379d8..e7a7fbe7c54 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py @@ -24,7 +24,7 @@ from .helpers.constants import HEADER_STR, MINUTE from .helpers.typing_env import EnvVarsDict from .helpers.utils_dict import copy_from_dict -from .helpers.utils_docker import get_localhost_ip +from .helpers.utils_host import get_localhost_ip log = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py index 3d5502e6bfb..29b81200a7e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py @@ -2,7 +2,6 @@ import logging import os import re -import socket import subprocess from enum import Enum from pathlib import Path @@ -46,19 +45,6 @@ class ContainerStatus(str, Enum): log = logging.getLogger(__name__) -def get_localhost_ip(default="127.0.0.1") -> str: - """Return the IP address for localhost""" - local_ip = default - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - # doesn't even have to be reachable - s.connect(("10.255.255.255", 1)) - local_ip = s.getsockname()[0] - finally: - s.close() - return f"{local_ip}" - - @retry( wait=wait_fixed(2), stop=stop_after_attempt(10), diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_host.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_host.py new file mode 100644 index 00000000000..e259053694e --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_host.py @@ -0,0 +1,14 @@ +import socket + + +def get_localhost_ip(default="127.0.0.1") -> str: + """Return the IP address for localhost""" + local_ip = default + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + # doesn't even have to be reachable + s.connect(("10.255.255.255", 1)) + local_ip = s.getsockname()[0] + finally: + s.close() + return f"{local_ip}" diff --git a/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py b/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py index 901c2789519..62baba3623e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/httpbin_service.py @@ -21,7 +21,7 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from .helpers.utils_docker import get_localhost_ip +from .helpers.utils_host import get_localhost_ip @pytest.fixture(scope="session") diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index c24c3d639dc..7ff284f9213 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -3,7 +3,8 @@ # pylint: disable=unused-variable import logging -from typing import Any, Iterator +from collections.abc import Iterator +from typing import Any import pytest from minio import Minio @@ -16,7 +17,8 @@ from tenacity.stop import stop_after_attempt from tenacity.wait import wait_fixed -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip log = logging.getLogger(__name__) @@ -69,7 +71,6 @@ def minio_config( @pytest.fixture(scope="module") def minio_service(minio_config: dict[str, str]) -> Iterator[Minio]: - client = Minio(**minio_config["client"]) for attempt in Retrying( diff --git a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py index ec250c00876..0c05f587652 100644 --- a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py @@ -15,7 +15,8 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip from .helpers.utils_postgres import PostgresTestConfig, migrated_pg_tables_context _TEMPLATE_DB_TO_RESTORE = "template_simcore_db" diff --git a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py index 3e90d2a7bcd..d190d711ff6 100644 --- a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py @@ -16,7 +16,8 @@ from tenacity.wait import wait_fixed from .helpers.typing_env import EnvVarsDict -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip _logger = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/redis_service.py b/packages/pytest-simcore/src/pytest_simcore/redis_service.py index 20cabcfe006..9651255198b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/redis_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/redis_service.py @@ -3,18 +3,20 @@ # pylint:disable=redefined-outer-name import logging -from typing import AsyncIterator +from collections.abc import AsyncIterator import pytest import tenacity from redis.asyncio import Redis, from_url +from settings_library.basic_types import PortInt from settings_library.redis import RedisDatabase, RedisSettings from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed from yarl import URL -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip log = logging.getLogger(__name__) @@ -33,13 +35,13 @@ async def redis_settings( "simcore_redis", testing_environ_vars["REDIS_PORT"] ) # test runner is running on the host computer - settings = RedisSettings(REDIS_HOST=get_localhost_ip(), REDIS_PORT=int(port)) + settings = RedisSettings(REDIS_HOST=get_localhost_ip(), REDIS_PORT=PortInt(port)) await wait_till_redis_responsive(settings.build_redis_dsn(RedisDatabase.RESOURCES)) return settings -@pytest.fixture(scope="function") +@pytest.fixture() def redis_service( redis_settings: RedisSettings, monkeypatch: pytest.MonkeyPatch, @@ -53,7 +55,7 @@ def redis_service( return redis_settings -@pytest.fixture(scope="function") +@pytest.fixture() async def redis_client( redis_settings: RedisSettings, ) -> AsyncIterator[Redis]: @@ -70,7 +72,7 @@ async def redis_client( await client.close(close_connection_pool=True) -@pytest.fixture(scope="function") +@pytest.fixture() async def redis_locks_client( redis_settings: RedisSettings, ) -> AsyncIterator[Redis]: @@ -98,6 +100,7 @@ async def wait_till_redis_responsive(redis_url: URL | str) -> None: try: if not await client.ping(): - raise ConnectionError(f"{redis_url=} not available") + msg = f"{redis_url=} not available" + raise ConnectionError(msg) finally: await client.close(close_connection_pool=True) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index 34081e39670..406e1ac269e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -20,7 +20,8 @@ from .helpers.constants import MINUTE from .helpers.typing_env import EnvVarsDict -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip log = logging.getLogger(__name__) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py index 2e1c6f4dc86..9b0032c51fd 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -2,8 +2,8 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name import os +from collections.abc import Callable, Iterable from copy import deepcopy -from typing import Callable, Iterable import aiohttp import pytest @@ -15,7 +15,8 @@ from servicelib.minio_utils import MinioRetryPolicyUponInitialization from yarl import URL -from .helpers.utils_docker import get_localhost_ip, get_service_published_port +from .helpers.utils_docker import get_service_published_port +from .helpers.utils_host import get_localhost_ip @pytest.fixture(scope="module") @@ -38,7 +39,7 @@ def storage_endpoint(docker_stack: dict, testing_environ_vars: dict) -> Iterable os.environ = old_environ -@pytest.fixture(scope="function") +@pytest.fixture() async def storage_service( minio_service: Minio, storage_endpoint: URL, docker_stack: dict ) -> URL: diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/clusters_keeper/ec2_instances.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/clusters_keeper/ec2_instances.py index 2663c82d520..433eee07fa3 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/clusters_keeper/ec2_instances.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/clusters_keeper/ec2_instances.py @@ -1,5 +1,5 @@ from models_library.api_schemas_clusters_keeper import CLUSTERS_KEEPER_RPC_NAMESPACE -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType +from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from models_library.rabbitmq_basic_types import RPCMethodName from ..._client_rpc import RabbitMQRPCClient @@ -8,14 +8,14 @@ async def get_instance_type_details( client: RabbitMQRPCClient, *, instance_type_names: set[str] -) -> list[EC2InstanceType]: +) -> list[EC2InstanceTypeGet]: """**Remote method** Raises: RPCServerError -- if anything happens remotely """ - instance_types: list[EC2InstanceType] = await client.request( + instance_types: list[EC2InstanceTypeGet] = await client.request( CLUSTERS_KEEPER_RPC_NAMESPACE, RPCMethodName("get_instance_type_details"), timeout_s=RPC_REMOTE_METHOD_TIMEOUT_S, diff --git a/packages/settings-library/src/settings_library/ec2.py b/packages/settings-library/src/settings_library/ec2.py new file mode 100644 index 00000000000..710db9c6e4f --- /dev/null +++ b/packages/settings-library/src/settings_library/ec2.py @@ -0,0 +1,26 @@ +from typing import Any, ClassVar + +from pydantic import Field + +from .base import BaseCustomSettings + + +class EC2Settings(BaseCustomSettings): + EC2_ACCESS_KEY_ID: str + EC2_ENDPOINT: str | None = Field( + default=None, description="do not define if using standard AWS" + ) + EC2_REGION_NAME: str = "us-east-1" + EC2_SECRET_ACCESS_KEY: str + + class Config(BaseCustomSettings.Config): + schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + "examples": [ + { + "EC2_ACCESS_KEY_ID": "my_access_key_id", + "EC2_ENDPOINT": "http://my_ec2_endpoint.com", + "EC2_REGION_NAME": "us-east-1", + "EC2_SECRET_ACCESS_KEY": "my_secret_access_key", + } + ], + } diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index f691a844b28..62a11df75ac 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -18,7 +18,7 @@ pytest_plugins = [ - "pytest_simcore.aws_services", + "pytest_simcore.aws_server", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", "pytest_simcore.file_extra", diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py b/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py index 7c38863c096..12353b6cd6a 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py @@ -147,12 +147,12 @@ async def test_upload_file_to_presigned_links_raises_aws_s3_400_request_time_out @pytest.fixture async def aiobotocore_s3_client( - mocked_s3_server: ThreadedMotoServer, + mocked_aws_server: ThreadedMotoServer, ) -> AsyncIterator[AioBaseClient]: session = get_session() async with session.create_client( "s3", - endpoint_url=f"http://{mocked_s3_server._ip_address}:{mocked_s3_server._port}", # pylint: disable=protected-access + endpoint_url=f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access aws_secret_access_key="xxx", aws_access_key_id="xxx", ) as client: @@ -195,7 +195,7 @@ def file_id(faker: Faker) -> str: @pytest.fixture async def create_upload_links( - mocked_s3_server: ThreadedMotoServer, + mocked_aws_server: ThreadedMotoServer, aiobotocore_s3_client: AioBaseClient, faker: Faker, bucket: str, diff --git a/services/agent/requirements/_base.in b/services/agent/requirements/_base.in index c195fb17f5c..ea2c803f81b 100644 --- a/services/agent/requirements/_base.in +++ b/services/agent/requirements/_base.in @@ -14,4 +14,5 @@ aiodocker fastapi packaging pydantic +python-dotenv uvicorn diff --git a/services/agent/requirements/_base.txt b/services/agent/requirements/_base.txt index 2619c197119..c3d0645fe49 100644 --- a/services/agent/requirements/_base.txt +++ b/services/agent/requirements/_base.txt @@ -115,6 +115,8 @@ pyrsistent==0.19.2 # via jsonschema python-dateutil==2.8.2 # via arrow +python-dotenv==1.0.0 + # via -r requirements/_base.in pyyaml==6.0.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt diff --git a/services/agent/requirements/_test.txt b/services/agent/requirements/_test.txt index af900eaf1d6..ac1398a936d 100644 --- a/services/agent/requirements/_test.txt +++ b/services/agent/requirements/_test.txt @@ -9,7 +9,9 @@ aioboto3==9.6.0 # -c requirements/./constraints.txt # -r requirements/_test.in aiobotocore==2.3.0 - # via aioboto3 + # via + # aioboto3 + # aiobotocore aiohttp==3.8.5 # via # -c requirements/../../../requirements/constraints.txt @@ -229,7 +231,9 @@ python-dateutil==2.8.2 # faker # moto python-jose==3.3.0 - # via moto + # via + # moto + # python-jose pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt diff --git a/services/agent/tests/conftest.py b/services/agent/tests/conftest.py index 92ec5d411c0..97da425621a 100644 --- a/services/agent/tests/conftest.py +++ b/services/agent/tests/conftest.py @@ -22,7 +22,7 @@ pytestmark = pytest.mark.asyncio pytest_plugins = [ - "pytest_simcore.aws_services", + "pytest_simcore.aws_server", "pytest_simcore.repository_paths", ] @@ -188,9 +188,9 @@ def caplog_info_debug(caplog: LogCaptureFixture) -> Iterable[LogCaptureFixture]: @pytest.fixture(scope="module") -def mocked_s3_server_url(mocked_s3_server: ThreadedMotoServer) -> HttpUrl: +def mocked_s3_server_url(mocked_aws_server: ThreadedMotoServer) -> HttpUrl: # pylint: disable=protected-access return parse_obj_as( HttpUrl, - f"http://{mocked_s3_server._ip_address}:{mocked_s3_server._port}", # noqa: SLF001 + f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # noqa: SLF001 ) diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index b1046bafb27..3aa9a1d6cf1 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -36,8 +36,8 @@ from pydantic import HttpUrl, parse_obj_as from pytest import MonkeyPatch # noqa: PT013 from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from pytest_simcore.simcore_webserver_projects_rest_api import GET_PROJECT from requests.auth import HTTPBasicAuth from respx import MockRouter @@ -269,7 +269,6 @@ def mocked_webserver_service_api_base( assert_all_called=False, assert_all_mocked=True, ) as respx_mock: - # healthcheck_readiness_probe, healthcheck_liveness_probe response_body = { "name": "webserver", diff --git a/services/autoscaling/requirements/_base.in b/services/autoscaling/requirements/_base.in index 09555b35987..ae362ec2744 100644 --- a/services/autoscaling/requirements/_base.in +++ b/services/autoscaling/requirements/_base.in @@ -9,14 +9,13 @@ # intra-repo required dependencies --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in +--requirement ../../../packages/aws-library/requirements/_base.in # service-library[fastapi] --requirement ../../../packages/service-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_fastapi.in aiocache aiodocker -aioboto3 dask[distributed] fastapi packaging -types-aiobotocore[ec2] diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index 0c81d2664c3..bb189297c0e 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -9,13 +9,15 @@ aio-pika==9.3.0 # -c requirements/../../../packages/service-library/requirements/./_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aioboto3==12.0.0 - # via -r requirements/_base.in + # via -r requirements/../../../packages/aws-library/requirements/_base.in aiobotocore==2.7.0 # via # aioboto3 # aiobotocore aiocache==0.12.2 - # via -r requirements/_base.in + # via + # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/_base.in aiodebug==2.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in @@ -31,6 +33,7 @@ aiofiles==23.2.1 # -r requirements/../../../packages/service-library/requirements/_base.in aiohttp==3.8.6 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -79,6 +82,7 @@ botocore-stubs==1.31.77 # via types-aiobotocore certifi==2023.7.22 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -121,6 +125,7 @@ exceptiongroup==1.1.3 # via anyio fastapi==0.99.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -148,6 +153,7 @@ httpcore==0.18.0 # via httpx httpx==0.25.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -170,6 +176,7 @@ importlib-metadata==6.8.0 # dask jinja2==3.1.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -236,6 +243,7 @@ psutil==5.9.5 # distributed pydantic==1.10.13 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -246,6 +254,7 @@ pydantic==1.10.13 # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in @@ -266,6 +275,7 @@ python-dateutil==2.8.2 # botocore pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -282,6 +292,7 @@ pyyaml==6.0.1 # distributed redis==5.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -323,6 +334,7 @@ sortedcontainers==2.4.0 # distributed starlette==0.27.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -363,7 +375,7 @@ typer==0.9.0 # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in types-aiobotocore==2.7.0 - # via -r requirements/_base.in + # via -r requirements/../../../packages/aws-library/requirements/_base.in types-aiobotocore-ec2==2.7.0 # via types-aiobotocore types-awscrt==0.19.8 @@ -382,6 +394,7 @@ typing-extensions==4.8.0 # uvicorn urllib3==1.26.16 # via + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt diff --git a/services/autoscaling/requirements/ci.txt b/services/autoscaling/requirements/ci.txt index d2b1c3498e4..4adf7f199a7 100644 --- a/services/autoscaling/requirements/ci.txt +++ b/services/autoscaling/requirements/ci.txt @@ -11,11 +11,12 @@ --requirement _test.txt # installs this repo's packages +../../packages/aws-library +../../packages/dask-task-models-library ../../packages/models-library ../../packages/pytest-simcore ../../packages/service-library[fastapi] ../../packages/settings-library -../../packages/dask-task-models-library # installs current package . diff --git a/services/autoscaling/requirements/dev.txt b/services/autoscaling/requirements/dev.txt index 8cbfdbe8e3f..432e7ef62e9 100644 --- a/services/autoscaling/requirements/dev.txt +++ b/services/autoscaling/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/aws-library --editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] diff --git a/services/autoscaling/requirements/prod.txt b/services/autoscaling/requirements/prod.txt index 7d635391e63..9452305726e 100644 --- a/services/autoscaling/requirements/prod.txt +++ b/services/autoscaling/requirements/prod.txt @@ -10,6 +10,7 @@ --requirement _base.txt # installs this repo's packages +../../packages/aws-library ../../packages/models-library ../../packages/service-library[fastapi] ../../packages/settings-library diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py index 057d59aa912..be378286fad 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py @@ -21,6 +21,7 @@ ) from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings +from settings_library.ec2 import EC2Settings from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from settings_library.utils_logging import MixinLoggingSettings @@ -29,15 +30,6 @@ from .._meta import API_VERSION, API_VTAG, APP_NAME -class EC2Settings(BaseCustomSettings): - EC2_ACCESS_KEY_ID: str - EC2_ENDPOINT: str | None = Field( - default=None, description="do not define if using standard AWS" - ) - EC2_REGION_NAME: str = "us-east-1" - EC2_SECRET_ACCESS_KEY: str - - class EC2InstancesSettings(BaseCustomSettings): EC2_INSTANCES_ALLOWED_TYPES: list[str] = Field( ..., diff --git a/services/autoscaling/src/simcore_service_autoscaling/models.py b/services/autoscaling/src/simcore_service_autoscaling/models.py index 8be0f953fe8..9e3ffec2df9 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/models.py +++ b/services/autoscaling/src/simcore_service_autoscaling/models.py @@ -1,65 +1,8 @@ -import datetime from dataclasses import dataclass, field from typing import Any, TypeAlias +from aws_library.ec2.models import EC2InstanceData from models_library.generated_models.docker_rest_api import Node -from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt -from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType - - -class Resources(BaseModel): - cpus: NonNegativeFloat - ram: ByteSize - - @classmethod - def create_as_empty(cls) -> "Resources": - return cls(cpus=0, ram=ByteSize(0)) - - def __ge__(self, other: "Resources") -> bool: - return self.cpus >= other.cpus and self.ram >= other.ram - - def __gt__(self, other: "Resources") -> bool: - return self.cpus > other.cpus or self.ram > other.ram - - def __add__(self, other: "Resources") -> "Resources": - return Resources.construct( - **{ - key: a + b - for (key, a), b in zip( - self.dict().items(), other.dict().values(), strict=True - ) - } - ) - - def __sub__(self, other: "Resources") -> "Resources": - return Resources.construct( - **{ - key: a - b - for (key, a), b in zip( - self.dict().items(), other.dict().values(), strict=True - ) - } - ) - - -@dataclass(frozen=True) -class EC2InstanceType: - name: InstanceTypeType - cpus: PositiveInt - ram: ByteSize - - -InstancePrivateDNSName = str - - -@dataclass(frozen=True) -class EC2InstanceData: - launch_time: datetime.datetime - id: str - aws_private_dns: InstancePrivateDNSName - type: InstanceTypeType - state: InstanceStateNameType - resources: Resources @dataclass(frozen=True) diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py index 5dcd7de627c..382b7f85c6c 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py @@ -7,6 +7,12 @@ from typing import cast import arrow +from aws_library.ec2.models import ( + EC2InstanceConfig, + EC2InstanceData, + EC2InstanceType, + Resources, +) from fastapi import FastAPI from models_library.generated_models.docker_rest_api import ( Availability, @@ -24,13 +30,7 @@ Ec2TooManyInstancesError, ) from ..core.settings import ApplicationSettings, get_application_settings -from ..models import ( - AssociatedInstance, - Cluster, - EC2InstanceData, - EC2InstanceType, - Resources, -) +from ..models import AssociatedInstance, Cluster from ..utils import utils_docker, utils_ec2 from ..utils.auto_scaling_core import ( associate_ec2_instances_with_nodes, @@ -58,12 +58,13 @@ async def _analyze_current_cluster( # get the EC2 instances we have existing_ec2_instances = await get_ec2_client(app).get_instances( - app_settings.AUTOSCALING_EC2_INSTANCES, auto_scaling_mode.get_ec2_tags(app) + key_names=[app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME], + tags=auto_scaling_mode.get_ec2_tags(app), ) terminated_ec2_instances = await get_ec2_client(app).get_instances( - app_settings.AUTOSCALING_EC2_INSTANCES, - auto_scaling_mode.get_ec2_tags(app), + key_names=[app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME], + tags=auto_scaling_mode.get_ec2_tags(app), state_names=["terminated"], ) @@ -112,7 +113,7 @@ def _node_not_ready(node: Node) -> bool: async def _cleanup_disconnected_nodes(app: FastAPI, cluster: Cluster) -> Cluster: if cluster.disconnected_nodes: await utils_docker.remove_nodes( - get_docker_client(app), cluster.disconnected_nodes + get_docker_client(app), nodes=cluster.disconnected_nodes ) return dataclasses.replace(cluster, disconnected_nodes=[]) @@ -407,11 +408,17 @@ async def _start_instances( results = await asyncio.gather( *[ ec2_client.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - instance_type=instance_type, - tags=instance_tags, - startup_script=instance_startup_script, + EC2InstanceConfig( + type=instance_type, + tags=instance_tags, + startup_script=instance_startup_script, + ami_id=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_AMI_ID, + key_name=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME, + security_group_ids=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_SECURITY_GROUP_IDS, + subnet_id=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_SUBNET_ID, + ), number_of_instances=instance_num, + max_number_of_instances=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES, ) for instance_type, instance_num in needed_instances.items() ], @@ -590,7 +597,7 @@ async def _try_scale_down_cluster(app: FastAPI, cluster: Cluster) -> Cluster: await utils_docker.remove_nodes( get_docker_client(app), - [i.node for i in terminateable_instances], + nodes=[i.node for i in terminateable_instances], force=True, ) terminated_instance_ids = [i.ec2_instance.id for i in terminateable_instances] diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py index 2adf6268e8c..08ae67d435f 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py @@ -2,13 +2,14 @@ from collections.abc import Iterable from dataclasses import dataclass +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from fastapi import FastAPI from models_library.docker import DockerLabelKey from models_library.generated_models.docker_rest_api import Node as DockerNode from servicelib.logging_utils import LogLevelInt from types_aiobotocore_ec2.literals import InstanceTypeType -from ..models import AssociatedInstance, EC2InstanceData, EC2InstanceType, Resources +from ..models import AssociatedInstance @dataclass diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py index 0fe64471056..f627e0398a1 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py @@ -2,6 +2,7 @@ import logging from collections.abc import Iterable +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from fastapi import FastAPI from models_library.docker import ( DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY, @@ -14,13 +15,7 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import ( - AssociatedInstance, - DaskTask, - EC2InstanceData, - EC2InstanceType, - Resources, -) +from ..models import AssociatedInstance, DaskTask from ..utils import computational_scaling as utils from ..utils import utils_docker, utils_ec2 from . import dask @@ -75,6 +70,7 @@ async def try_assigning_task_to_instances( *, notify_progress: bool ) -> bool: + assert type_to_instance_map # nosec return await utils.try_assigning_task_to_instances( app, pending_task, diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py index 6a2d5814c85..b2107f2a043 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py @@ -1,5 +1,6 @@ from collections.abc import Iterable +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from fastapi import FastAPI from models_library.docker import DockerLabelKey from models_library.generated_models.docker_rest_api import Node, Task @@ -7,7 +8,7 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import AssociatedInstance, EC2InstanceData, EC2InstanceType, Resources +from ..models import AssociatedInstance from ..utils import dynamic_scaling as utils from ..utils import utils_docker, utils_ec2 from ..utils.rabbitmq import log_tasks_message, progress_tasks_message diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py b/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py index d94c855a60d..a40dacbb17e 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/dask.py @@ -4,6 +4,7 @@ from typing import Any, Final, TypeAlias import distributed +from aws_library.ec2.models import EC2InstanceData, Resources from pydantic import AnyUrl, ByteSize, parse_obj_as from ..core.errors import ( @@ -11,14 +12,7 @@ DaskSchedulerNotFoundError, DaskWorkerNotFoundError, ) -from ..models import ( - AssociatedInstance, - DaskTask, - DaskTaskId, - DaskTaskResources, - EC2InstanceData, - Resources, -) +from ..models import AssociatedInstance, DaskTask, DaskTaskId, DaskTaskResources from ..utils.auto_scaling_core import ( node_host_name_from_ec2_private_dns, node_ip_from_ec2_private_dns, diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py b/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py index f8c0eb595e1..c3464eb6f0b 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/ec2.py @@ -1,232 +1,17 @@ -import contextlib import logging -from dataclasses import dataclass from typing import cast -import aioboto3 -import botocore.exceptions -from aiobotocore.session import ClientCreatorContext -from aiocache import cached +from aws_library.ec2.client import SimcoreEC2API from fastapi import FastAPI -from pydantic import ByteSize, parse_obj_as -from servicelib.logging_utils import log_context +from settings_library.ec2 import EC2Settings from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay from tenacity.wait import wait_random_exponential -from types_aiobotocore_ec2 import EC2Client -from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType -from types_aiobotocore_ec2.type_defs import FilterTypeDef -from ..core.errors import ( - ConfigurationError, - Ec2InstanceNotFoundError, - Ec2NotConnectedError, - Ec2TooManyInstancesError, -) -from ..core.settings import EC2InstancesSettings, EC2Settings -from ..models import EC2InstanceData, EC2InstanceType, Resources -from ..utils.utils_ec2 import compose_user_data +from ..core.errors import ConfigurationError, Ec2NotConnectedError -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class AutoscalingEC2: - client: EC2Client - session: aioboto3.Session - exit_stack: contextlib.AsyncExitStack - - @classmethod - async def create(cls, settings: EC2Settings) -> "AutoscalingEC2": - session = aioboto3.Session() - session_client = session.client( - "ec2", - endpoint_url=settings.EC2_ENDPOINT, - aws_access_key_id=settings.EC2_ACCESS_KEY_ID, - aws_secret_access_key=settings.EC2_SECRET_ACCESS_KEY, - region_name=settings.EC2_REGION_NAME, - ) - assert isinstance(session_client, ClientCreatorContext) # nosec - exit_stack = contextlib.AsyncExitStack() - ec2_client = cast( - EC2Client, await exit_stack.enter_async_context(session_client) - ) - return cls(ec2_client, session, exit_stack) - - async def close(self) -> None: - await self.exit_stack.aclose() - - async def ping(self) -> bool: - try: - await self.client.describe_account_attributes() - return True - except Exception: # pylint: disable=broad-except - return False - - @cached(noself=True) - async def get_ec2_instance_capabilities( - self, - instance_type_names: set[InstanceTypeType], - ) -> list[EC2InstanceType]: - """instance_type_names must be a set of unique values""" - instance_types = await self.client.describe_instance_types( - InstanceTypes=list(instance_type_names) - ) - list_instances: list[EC2InstanceType] = [] - for instance in instance_types.get("InstanceTypes", []): - with contextlib.suppress(KeyError): - list_instances.append( - EC2InstanceType( - name=instance["InstanceType"], - cpus=instance["VCpuInfo"]["DefaultVCpus"], - ram=parse_obj_as( - ByteSize, f"{instance['MemoryInfo']['SizeInMiB']}MiB" - ), - ) - ) - return list_instances - - async def start_aws_instance( - self, - instance_settings: EC2InstancesSettings, - instance_type: EC2InstanceType, - tags: dict[str, str], - startup_script: str, - number_of_instances: int, - ) -> list[EC2InstanceData]: - with log_context( - logger, - logging.INFO, - msg=f"launching {number_of_instances} AWS instance(s) {instance_type.name} with {tags=}", - ): - # first check the max amount is not already reached - current_instances = await self.get_instances(instance_settings, tags) - if ( - len(current_instances) + number_of_instances - > instance_settings.EC2_INSTANCES_MAX_INSTANCES - ): - raise Ec2TooManyInstancesError( - num_instances=instance_settings.EC2_INSTANCES_MAX_INSTANCES - ) - - instances = await self.client.run_instances( - ImageId=instance_settings.EC2_INSTANCES_AMI_ID, - MinCount=number_of_instances, - MaxCount=number_of_instances, - InstanceType=instance_type.name, - InstanceInitiatedShutdownBehavior="terminate", - KeyName=instance_settings.EC2_INSTANCES_KEY_NAME, - SubnetId=instance_settings.EC2_INSTANCES_SUBNET_ID, - TagSpecifications=[ - { - "ResourceType": "instance", - "Tags": [ - {"Key": tag_key, "Value": tag_value} - for tag_key, tag_value in tags.items() - ], - } - ], - UserData=compose_user_data(startup_script), - SecurityGroupIds=instance_settings.EC2_INSTANCES_SECURITY_GROUP_IDS, - ) - instance_ids = [i["InstanceId"] for i in instances["Instances"]] - logger.info( - "New instances launched: %s, waiting for them to start now...", - instance_ids, - ) - - # wait for the instance to be in a pending state - # NOTE: reference to EC2 states https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-lifecycle.html - waiter = self.client.get_waiter("instance_exists") - await waiter.wait(InstanceIds=instance_ids) - logger.info("instances %s exists now.", instance_ids) - - # get the private IPs - instances = await self.client.describe_instances(InstanceIds=instance_ids) - instance_datas = [ - EC2InstanceData( - launch_time=instance["LaunchTime"], - id=instance["InstanceId"], - aws_private_dns=instance["PrivateDnsName"], - type=instance["InstanceType"], - state=instance["State"]["Name"], - resources=Resources(cpus=instance_type.cpus, ram=instance_type.ram), - ) - for instance in instances["Reservations"][0]["Instances"] - ] - logger.info( - "%s is available, happy computing!!", - f"{instance_datas=}", - ) - return instance_datas - - async def get_instances( - self, - instance_settings: EC2InstancesSettings, - tags: dict[str, str], - *, - state_names: list[InstanceStateNameType] | None = None, - ) -> list[EC2InstanceData]: - # NOTE: be careful: Name=instance-state-name,Values=["pending", "running"] means pending OR running - # NOTE2: AND is done by repeating Name=instance-state-name,Values=pending Name=instance-state-name,Values=running - if state_names is None: - state_names = ["pending", "running"] - - filters: list[FilterTypeDef] = [ - { - "Name": "key-name", - "Values": [instance_settings.EC2_INSTANCES_KEY_NAME], - }, - {"Name": "instance-state-name", "Values": state_names}, - ] - filters.extend( - [{"Name": f"tag:{key}", "Values": [value]} for key, value in tags.items()] - ) - - instances = await self.client.describe_instances(Filters=filters) - all_instances = [] - for reservation in instances["Reservations"]: - assert "Instances" in reservation # nosec - for instance in reservation["Instances"]: - assert "LaunchTime" in instance # nosec - assert "InstanceId" in instance # nosec - assert "PrivateDnsName" in instance # nosec - assert "InstanceType" in instance # nosec - assert "State" in instance # nosec - assert "Name" in instance["State"] # nosec - ec2_instance_types = await self.get_ec2_instance_capabilities( - {instance["InstanceType"]} - ) - assert len(ec2_instance_types) == 1 # nosec - all_instances.append( - EC2InstanceData( - launch_time=instance["LaunchTime"], - id=instance["InstanceId"], - aws_private_dns=instance["PrivateDnsName"], - type=instance["InstanceType"], - state=instance["State"]["Name"], - resources=Resources( - cpus=ec2_instance_types[0].cpus, - ram=ec2_instance_types[0].ram, - ), - ) - ) - logger.debug("received: %s", f"{all_instances=}") - return all_instances - - async def terminate_instances(self, instance_datas: list[EC2InstanceData]) -> None: - try: - await self.client.terminate_instances( - InstanceIds=[i.id for i in instance_datas] - ) - except botocore.exceptions.ClientError as exc: - if ( - exc.response.get("Error", {}).get("Code", "") - == "InvalidInstanceID.NotFound" - ): - raise Ec2InstanceNotFoundError from exc - raise # pragma: no cover +_logger = logging.getLogger(__name__) def setup(app: FastAPI) -> None: @@ -235,16 +20,16 @@ async def on_startup() -> None: settings: EC2Settings | None = app.state.settings.AUTOSCALING_EC2_ACCESS if not settings: - logger.warning("EC2 client is de-activated in the settings") + _logger.warning("EC2 client is de-activated in the settings") return - app.state.ec2_client = client = await AutoscalingEC2.create(settings) + app.state.ec2_client = client = await SimcoreEC2API.create(settings) async for attempt in AsyncRetrying( reraise=True, stop=stop_after_delay(120), wait=wait_random_exponential(max=30), - before_sleep=before_sleep_log(logger, logging.WARNING), + before_sleep=before_sleep_log(_logger, logging.WARNING), ): with attempt: connected = await client.ping() @@ -253,15 +38,15 @@ async def on_startup() -> None: async def on_shutdown() -> None: if app.state.ec2_client: - await cast(AutoscalingEC2, app.state.ec2_client).close() + await cast(SimcoreEC2API, app.state.ec2_client).close() app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) -def get_ec2_client(app: FastAPI) -> AutoscalingEC2: +def get_ec2_client(app: FastAPI) -> SimcoreEC2API: if not app.state.ec2_client: raise ConfigurationError( msg="EC2 client is not available. Please check the configuration." ) - return cast(AutoscalingEC2, app.state.ec2_client) + return cast(SimcoreEC2API, app.state.ec2_client) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py index 7d9ab4f310d..701f9d1ae39 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py @@ -3,12 +3,13 @@ import re from typing import Final +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from models_library.generated_models.docker_rest_api import Node from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.errors import Ec2InstanceInvalidError, Ec2InvalidDnsNameError from ..core.settings import ApplicationSettings -from ..models import AssociatedInstance, EC2InstanceData, EC2InstanceType, Resources +from ..models import AssociatedInstance from ..modules.auto_scaling_mode_base import BaseAutoscaling from . import utils_docker @@ -24,7 +25,8 @@ def node_host_name_from_ec2_private_dns( Ec2InvalidDnsNameError: if the dns name does not follow the expected pattern """ if match := re.match(_EC2_INTERNAL_DNS_RE, ec2_instance_data.aws_private_dns): - return match.group("host_name") + host_name: str = match.group("host_name") + return host_name raise Ec2InvalidDnsNameError(aws_private_dns_name=ec2_instance_data.aws_private_dns) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py b/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py index f5cb43c4acb..8fcefb5c8de 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py @@ -3,19 +3,14 @@ from collections.abc import Iterable from typing import Final +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from dask_task_models_library.constants import DASK_TASK_EC2_RESOURCE_RESTRICTION_KEY from fastapi import FastAPI from servicelib.utils_formatting import timedelta_as_minute_second from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import ( - AssociatedInstance, - DaskTask, - EC2InstanceData, - EC2InstanceType, - Resources, -) +from ..models import AssociatedInstance, DaskTask _logger = logging.getLogger(__name__) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py b/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py index 549b59bb38c..93af22f7f4e 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py @@ -2,12 +2,13 @@ import logging from collections.abc import Iterable +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from fastapi import FastAPI from models_library.generated_models.docker_rest_api import Task from servicelib.utils_formatting import timedelta_as_minute_second from ..core.settings import get_application_settings -from ..models import AssociatedInstance, EC2InstanceData, EC2InstanceType, Resources +from ..models import AssociatedInstance from . import utils_docker from .rabbitmq import log_tasks_message, progress_tasks_message diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py b/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py index e8bce0ba5aa..d8c88c50a86 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/rabbitmq.py @@ -1,6 +1,7 @@ import asyncio import logging +from aws_library.ec2.models import Resources from fastapi import FastAPI from models_library.docker import StandardSimcoreDockerLabels from models_library.generated_models.docker_rest_api import Task @@ -13,7 +14,7 @@ from servicelib.logging_utils import log_catch from ..core.settings import ApplicationSettings, get_application_settings -from ..models import Cluster, Resources +from ..models import Cluster from ..modules.rabbitmq import post_message logger = logging.getLogger(__name__) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py index 65b8ce8af05..edea9cee264 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py @@ -13,6 +13,7 @@ from typing import Final, cast import yaml +from aws_library.ec2.models import EC2InstanceData, Resources from models_library.docker import ( DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY, DockerGenericTag, @@ -33,7 +34,6 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import ApplicationSettings -from ..models import EC2InstanceData, Resources from ..modules.docker import AutoscalingDocker logger = logging.getLogger(__name__) @@ -77,7 +77,7 @@ async def get_worker_nodes(docker_client: AutoscalingDocker) -> list[Node]: async def remove_nodes( - docker_client: AutoscalingDocker, nodes: list[Node], force: bool = False + docker_client: AutoscalingDocker, *, nodes: list[Node], force: bool = False ) -> list[Node]: """removes docker nodes that are in the down state (unless force is used and they will be forcibly removed)""" diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py index 672a8c76733..88c18314460 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py @@ -8,10 +8,11 @@ from collections.abc import Callable from textwrap import dedent +from aws_library.ec2.models import EC2InstanceType, Resources + from .._meta import VERSION from ..core.errors import ConfigurationError, Ec2InstanceNotFoundError from ..core.settings import ApplicationSettings -from ..models import EC2InstanceType, Resources logger = logging.getLogger(__name__) diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index e388acba225..fb6dca9f814 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -6,10 +6,8 @@ import dataclasses import datetime import json -import random -from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from collections.abc import AsyncIterator, Awaitable, Callable from copy import deepcopy -from datetime import timezone from pathlib import Path from typing import Any, Final, cast from unittest import mock @@ -19,10 +17,10 @@ import httpx import psutil import pytest -import requests import simcore_service_autoscaling -from aiohttp.test_utils import unused_port from asgi_lifespan import LifespanManager +from aws_library.ec2.client import SimcoreEC2API +from aws_library.ec2.models import EC2InstanceData from deepdiff import DeepDiff from faker import Faker from fakeredis.aioredis import FakeRedis @@ -37,26 +35,25 @@ ResourceObject, Service, ) -from moto.server import ThreadedMotoServer from pydantic import ByteSize, PositiveInt, parse_obj_as from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from settings_library.rabbit import RabbitSettings from simcore_service_autoscaling.core.application import create_app from simcore_service_autoscaling.core.settings import ApplicationSettings, EC2Settings -from simcore_service_autoscaling.models import Cluster, DaskTaskResources, Resources +from simcore_service_autoscaling.models import Cluster, DaskTaskResources from simcore_service_autoscaling.modules.docker import AutoscalingDocker -from simcore_service_autoscaling.modules.ec2 import AutoscalingEC2, EC2InstanceData from tenacity import retry from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from types_aiobotocore_ec2.client import EC2Client from types_aiobotocore_ec2.literals import InstanceTypeType pytest_plugins = [ + "pytest_simcore.aws_server", + "pytest_simcore.aws_ec2_service", "pytest_simcore.dask_scheduler", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", @@ -114,6 +111,30 @@ def app_environment( return mock_env_devel_environment | envs +@pytest.fixture +def mocked_ec2_instances_envs( + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + aws_security_group_id: str, + aws_subnet_id: str, + aws_ami_id: str, + aws_allowed_ec2_instance_type_names: list[InstanceTypeType], +) -> EnvVarsDict: + envs = setenvs_from_dict( + monkeypatch, + { + "EC2_INSTANCES_KEY_NAME": "osparc-pytest", + "EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps([aws_security_group_id]), + "EC2_INSTANCES_SUBNET_ID": aws_subnet_id, + "EC2_INSTANCES_AMI_ID": aws_ami_id, + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( + aws_allowed_ec2_instance_type_names + ), + }, + ) + return app_environment | envs + + @pytest.fixture def disable_dynamic_service_background_task(mocker: MockerFixture) -> None: mocker.patch( @@ -460,53 +481,6 @@ async def assert_for_service_state( ) -@pytest.fixture(scope="module") -def mocked_aws_server() -> Iterator[ThreadedMotoServer]: - """creates a moto-server that emulates AWS services in place - NOTE: Never use a bucket with underscores it fails!! - """ - server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port()) - # pylint: disable=protected-access - print( - f"--> started mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 - ) - print( - f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]" # noqa: SLF001 - ) - server.start() - yield server - server.stop() - print( - f"<-- stopped mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 - ) - - -@pytest.fixture -def reset_aws_server_state(mocked_aws_server: ThreadedMotoServer) -> Iterator[None]: - # NOTE: reset_aws_server_state [http://docs.getmoto.org/en/latest/docs/server_mode.html#reset-api] - yield - # pylint: disable=protected-access - requests.post( - f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}/moto-api/reset", # noqa: SLF001 - timeout=10, - ) - - -@pytest.fixture -def mocked_aws_server_envs( - app_environment: EnvVarsDict, - mocked_aws_server: ThreadedMotoServer, - reset_aws_server_state: None, - monkeypatch: pytest.MonkeyPatch, -) -> EnvVarsDict: - changed_envs: EnvVarsDict = { - "EC2_ENDPOINT": f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 - "EC2_ACCESS_KEY_ID": "xxx", - "EC2_SECRET_ACCESS_KEY": "xxx", - } - return app_environment | setenvs_from_dict(monkeypatch, changed_envs) - - @pytest.fixture(scope="session") def aws_allowed_ec2_instance_type_names() -> list[InstanceTypeType]: return [ @@ -532,125 +506,17 @@ def aws_allowed_ec2_instance_type_names_env( return app_environment | setenvs_from_dict(monkeypatch, changed_envs) -@pytest.fixture(scope="session") -def vpc_cidr_block() -> str: - return "10.0.0.0/16" - - -@pytest.fixture -async def aws_vpc_id( - mocked_aws_server_envs: None, - app_environment: EnvVarsDict, - monkeypatch: pytest.MonkeyPatch, - ec2_client: EC2Client, - vpc_cidr_block: str, -) -> AsyncIterator[str]: - vpc = await ec2_client.create_vpc( - CidrBlock=vpc_cidr_block, - ) - vpc_id = vpc["Vpc"]["VpcId"] # type: ignore - print(f"--> Created Vpc in AWS with {vpc_id=}") - yield vpc_id - - await ec2_client.delete_vpc(VpcId=vpc_id) - print(f"<-- Deleted Vpc in AWS with {vpc_id=}") - - -@pytest.fixture(scope="session") -def subnet_cidr_block() -> str: - return "10.0.1.0/24" - - -@pytest.fixture -async def aws_subnet_id( - monkeypatch: pytest.MonkeyPatch, - aws_vpc_id: str, - ec2_client: EC2Client, - subnet_cidr_block: str, -) -> AsyncIterator[str]: - subnet = await ec2_client.create_subnet( - CidrBlock=subnet_cidr_block, VpcId=aws_vpc_id - ) - assert "Subnet" in subnet - assert "SubnetId" in subnet["Subnet"] - subnet_id = subnet["Subnet"]["SubnetId"] - print(f"--> Created Subnet in AWS with {subnet_id=}") - - monkeypatch.setenv("EC2_INSTANCES_SUBNET_ID", subnet_id) - yield subnet_id - - # all the instances in the subnet must be terminated before that works - instances_in_subnet = await ec2_client.describe_instances( - Filters=[{"Name": "subnet-id", "Values": [subnet_id]}] - ) - if instances_in_subnet["Reservations"]: - print(f"--> terminating {len(instances_in_subnet)} instances in subnet") - await ec2_client.terminate_instances( - InstanceIds=[ - instance["Instances"][0]["InstanceId"] # type: ignore - for instance in instances_in_subnet["Reservations"] - ] - ) - print(f"<-- terminated {len(instances_in_subnet)} instances in subnet") - - await ec2_client.delete_subnet(SubnetId=subnet_id) - subnets = await ec2_client.describe_subnets() - print(f"<-- Deleted Subnet in AWS with {subnet_id=}") - print(f"current {subnets=}") - - -@pytest.fixture -async def aws_security_group_id( - monkeypatch: pytest.MonkeyPatch, - faker: Faker, - aws_vpc_id: str, - ec2_client: EC2Client, -) -> AsyncIterator[str]: - security_group = await ec2_client.create_security_group( - Description=faker.text(), GroupName=faker.pystr(), VpcId=aws_vpc_id - ) - security_group_id = security_group["GroupId"] - print(f"--> Created Security Group in AWS with {security_group_id=}") - monkeypatch.setenv( - "EC2_INSTANCES_SECURITY_GROUP_IDS", json.dumps([security_group_id]) - ) - yield security_group_id - await ec2_client.delete_security_group(GroupId=security_group_id) - print(f"<-- Deleted Security Group in AWS with {security_group_id=}") - - -@pytest.fixture -async def aws_ami_id( - app_environment: EnvVarsDict, - mocked_aws_server_envs: None, - monkeypatch: pytest.MonkeyPatch, - ec2_client: EC2Client, -) -> str: - images = await ec2_client.describe_images() - image = random.choice(images["Images"]) # noqa: S311 - ami_id = image["ImageId"] # type: ignore - monkeypatch.setenv("EC2_INSTANCES_AMI_ID", ami_id) - return ami_id - - @pytest.fixture async def autoscaling_ec2( app_environment: EnvVarsDict, -) -> AsyncIterator[AutoscalingEC2]: +) -> AsyncIterator[SimcoreEC2API]: settings = EC2Settings.create_from_envs() - ec2 = await AutoscalingEC2.create(settings) + ec2 = await SimcoreEC2API.create(settings) assert ec2 yield ec2 await ec2.close() -@pytest.fixture -async def ec2_client( - autoscaling_ec2: AutoscalingEC2, -) -> EC2Client: - return autoscaling_ec2.client - - @pytest.fixture def host_cpu_count() -> int: return psutil.cpu_count() @@ -679,26 +545,6 @@ def aws_instance_private_dns() -> str: return "ip-10-23-40-12.ec2.internal" -@pytest.fixture -def fake_ec2_instance_data(faker: Faker) -> Callable[..., EC2InstanceData]: - def _creator(**overrides) -> EC2InstanceData: - return EC2InstanceData( - **( - { - "launch_time": faker.date_time(tzinfo=timezone.utc), - "id": faker.uuid4(), - "aws_private_dns": f"ip-{faker.ipv4().replace('.', '-')}.ec2.internal", - "type": faker.pystr(), - "state": faker.pystr(), - "resources": Resources(cpus=4.0, ram=ByteSize(1024 * 1024)), - } - | overrides - ) - ) - - return _creator - - @pytest.fixture def fake_localhost_ec2_instance_data( fake_ec2_instance_data: Callable[..., EC2InstanceData] diff --git a/services/autoscaling/tests/unit/test_api_health.py b/services/autoscaling/tests/unit/test_api_health.py index 1ad9bbda13c..9ac0dccac01 100644 --- a/services/autoscaling/tests/unit/test_api_health.py +++ b/services/autoscaling/tests/unit/test_api_health.py @@ -20,7 +20,7 @@ def app_environment( app_environment: EnvVarsDict, enabled_rabbitmq: None, - mocked_aws_server_envs: None, + mocked_ec2_server_envs: EnvVarsDict, mocked_redis_server: None, ) -> EnvVarsDict: return app_environment diff --git a/services/autoscaling/tests/unit/test_core_settings.py b/services/autoscaling/tests/unit/test_core_settings.py index 45376d5963e..8576d5fdb35 100644 --- a/services/autoscaling/tests/unit/test_core_settings.py +++ b/services/autoscaling/tests/unit/test_core_settings.py @@ -56,16 +56,16 @@ def test_invalid_EC2_INSTANCES_TIME_BEFORE_TERMINATION( assert settings.AUTOSCALING_EC2_INSTANCES assert settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION assert ( - settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION - == datetime.timedelta(minutes=59) + datetime.timedelta(minutes=59) + == settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION ) monkeypatch.setenv("EC2_INSTANCES_TIME_BEFORE_TERMINATION", "-1:05:00") settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES assert ( - settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION - == datetime.timedelta(minutes=0) + datetime.timedelta(minutes=0) + == settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION ) @@ -74,7 +74,7 @@ def test_EC2_INSTANCES_PRE_PULL_IMAGES( ): settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES - assert settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES == [] + assert not settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES # passing an invalid image tag name will fail monkeypatch.setenv( @@ -97,9 +97,9 @@ def test_EC2_INSTANCES_PRE_PULL_IMAGES( ) settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES - assert settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES == [ + assert [ "nginx:latest", "itisfoundation/my-very-nice-service:latest", "simcore/services/dynamic/another-nice-one:2.4.5", "asd", - ] + ] == settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES diff --git a/services/autoscaling/tests/unit/test_models.py b/services/autoscaling/tests/unit/test_models.py index fdb9362591a..f859ff591d6 100644 --- a/services/autoscaling/tests/unit/test_models.py +++ b/services/autoscaling/tests/unit/test_models.py @@ -3,92 +3,14 @@ # pylint: disable=unused-variable -from typing import Any, Awaitable, Callable +from collections.abc import Awaitable, Callable +from typing import Any import aiodocker import pytest from models_library.docker import DockerLabelKey, StandardSimcoreDockerLabels from models_library.generated_models.docker_rest_api import Service, Task -from pydantic import ByteSize, ValidationError, parse_obj_as -from simcore_service_autoscaling.models import Resources - - -@pytest.mark.parametrize( - "a,b,a_greater_or_equal_than_b", - [ - ( - Resources(cpus=0.2, ram=ByteSize(0)), - Resources(cpus=0.1, ram=ByteSize(0)), - True, - ), - ( - Resources(cpus=0.1, ram=ByteSize(0)), - Resources(cpus=0.1, ram=ByteSize(0)), - True, - ), - ( - Resources(cpus=0.1, ram=ByteSize(1)), - Resources(cpus=0.1, ram=ByteSize(0)), - True, - ), - ( - Resources(cpus=0.05, ram=ByteSize(1)), - Resources(cpus=0.1, ram=ByteSize(0)), - False, - ), - ( - Resources(cpus=0.1, ram=ByteSize(0)), - Resources(cpus=0.1, ram=ByteSize(1)), - False, - ), - ], -) -def test_resources_ge_operator( - a: Resources, b: Resources, a_greater_or_equal_than_b: bool -): - assert (a >= b) is a_greater_or_equal_than_b - - -@pytest.mark.parametrize( - "a,b,result", - [ - ( - Resources(cpus=0, ram=ByteSize(0)), - Resources(cpus=1, ram=ByteSize(34)), - Resources(cpus=1, ram=ByteSize(34)), - ), - ( - Resources(cpus=0.1, ram=ByteSize(-1)), - Resources(cpus=1, ram=ByteSize(34)), - Resources(cpus=1.1, ram=ByteSize(33)), - ), - ], -) -def test_resources_add(a: Resources, b: Resources, result: Resources): - assert a + b == result - a += b - assert a == result - - -@pytest.mark.parametrize( - "a,b,result", - [ - ( - Resources(cpus=0, ram=ByteSize(0)), - Resources(cpus=1, ram=ByteSize(34)), - Resources.construct(cpus=-1, ram=ByteSize(-34)), - ), - ( - Resources(cpus=0.1, ram=ByteSize(-1)), - Resources(cpus=1, ram=ByteSize(34)), - Resources.construct(cpus=-0.9, ram=ByteSize(-35)), - ), - ], -) -def test_resources_sub(a: Resources, b: Resources, result: Resources): - assert a - b == result - a -= b - assert a == result +from pydantic import ValidationError, parse_obj_as async def test_get_simcore_service_docker_labels_from_task_with_missing_labels_raises( diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py index 0482437bb5d..7d4f1755ff3 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py @@ -18,6 +18,7 @@ import distributed import pytest +from aws_library.ec2.models import Resources from dask_task_models_library.constants import DASK_TASK_EC2_RESOURCE_RESTRICTION_KEY from faker import Faker from fastapi import FastAPI @@ -34,7 +35,6 @@ AssociatedInstance, Cluster, EC2InstanceData, - Resources, ) from simcore_service_autoscaling.modules.auto_scaling_core import ( _deactivate_empty_nodes, @@ -66,14 +66,12 @@ def local_dask_scheduler_server_envs( @pytest.fixture def minimal_configuration( docker_swarm: None, + mocked_ec2_server_envs: EnvVarsDict, enabled_computational_mode: EnvVarsDict, local_dask_scheduler_server_envs: EnvVarsDict, + mocked_ec2_instances_envs: EnvVarsDict, disabled_rabbitmq: None, disable_dynamic_service_background_task: None, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], mocked_redis_server: None, ) -> None: ... @@ -90,11 +88,6 @@ def dask_workers_config() -> dict[str, Any]: } -@pytest.fixture -def empty_cluster(cluster: Callable[..., Cluster]) -> Cluster: - return cluster() - - async def _assert_ec2_instances( ec2_client: EC2Client, *, @@ -192,15 +185,6 @@ def mock_find_node_with_name( ) -@pytest.fixture -def mock_cluster_used_resources(mocker: MockerFixture) -> mock.Mock: - return mocker.patch( - "simcore_service_autoscaling.modules.auto_scaling_core.utils_docker.compute_cluster_used_resources", - autospec=True, - return_value=Resources.create_as_empty(), - ) - - @pytest.fixture def mock_compute_node_used_resources(mocker: MockerFixture) -> mock.Mock: return mocker.patch( @@ -220,7 +204,7 @@ def mock_rabbitmq_post_message(mocker: MockerFixture) -> Iterator[mock.Mock]: @pytest.fixture def mock_terminate_instances(mocker: MockerFixture) -> Iterator[mock.Mock]: return mocker.patch( - "simcore_service_autoscaling.modules.ec2.AutoscalingEC2.terminate_instances", + "simcore_service_autoscaling.modules.ec2.SimcoreEC2API.terminate_instances", autospec=True, ) @@ -232,7 +216,7 @@ def mock_start_aws_instance( fake_ec2_instance_data: Callable[..., EC2InstanceData], ) -> Iterator[mock.Mock]: return mocker.patch( - "simcore_service_autoscaling.modules.ec2.AutoscalingEC2.start_aws_instance", + "simcore_service_autoscaling.modules.ec2.SimcoreEC2API.start_aws_instance", autospec=True, return_value=fake_ec2_instance_data(aws_private_dns=aws_instance_private_dns), ) @@ -556,7 +540,9 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 - datetime.timedelta(seconds=1) ).isoformat() await auto_scale_cluster(app=initialized_app, auto_scaling_mode=auto_scaling_mode) - mocked_docker_remove_node.assert_called_once_with(mock.ANY, [fake_node], force=True) + mocked_docker_remove_node.assert_called_once_with( + mock.ANY, nodes=[fake_node], force=True + ) await _assert_ec2_instances( ec2_client, num_reservations=1, diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py index cb41d2d3278..3f4caf35d1d 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py @@ -16,6 +16,7 @@ import aiodocker import pytest +from aws_library.ec2.models import EC2InstanceData, Resources from faker import Faker from fastapi import FastAPI from models_library.docker import ( @@ -35,7 +36,7 @@ from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_autoscaling.core.settings import ApplicationSettings -from simcore_service_autoscaling.models import AssociatedInstance, Cluster, Resources +from simcore_service_autoscaling.models import AssociatedInstance, Cluster from simcore_service_autoscaling.modules.auto_scaling_core import ( _activate_drained_nodes, _deactivate_empty_nodes, @@ -50,7 +51,6 @@ AutoscalingDocker, get_docker_client, ) -from simcore_service_autoscaling.modules.ec2 import EC2InstanceData from types_aiobotocore_ec2.client import EC2Client from types_aiobotocore_ec2.literals import InstanceTypeType @@ -58,7 +58,7 @@ @pytest.fixture def mock_terminate_instances(mocker: MockerFixture) -> Iterator[mock.Mock]: return mocker.patch( - "simcore_service_autoscaling.modules.ec2.AutoscalingEC2.terminate_instances", + "simcore_service_autoscaling.modules.ec2.SimcoreEC2API.terminate_instances", autospec=True, ) @@ -70,7 +70,7 @@ def mock_start_aws_instance( fake_ec2_instance_data: Callable[..., EC2InstanceData], ) -> Iterator[mock.Mock]: return mocker.patch( - "simcore_service_autoscaling.modules.ec2.AutoscalingEC2.start_aws_instance", + "simcore_service_autoscaling.modules.ec2.SimcoreEC2API.start_aws_instance", autospec=True, return_value=fake_ec2_instance_data(aws_private_dns=aws_instance_private_dns), ) @@ -114,15 +114,6 @@ def mock_remove_nodes(mocker: MockerFixture) -> mock.Mock: ) -@pytest.fixture -def mock_cluster_used_resources(mocker: MockerFixture) -> mock.Mock: - return mocker.patch( - "simcore_service_autoscaling.modules.auto_scaling_core.utils_docker.compute_cluster_used_resources", - autospec=True, - return_value=Resources.create_as_empty(), - ) - - @pytest.fixture def mock_compute_node_used_resources(mocker: MockerFixture) -> mock.Mock: return mocker.patch( @@ -197,13 +188,11 @@ async def drained_host_node( @pytest.fixture def minimal_configuration( docker_swarm: None, + mocked_ec2_server_envs: EnvVarsDict, enabled_dynamic_mode: EnvVarsDict, + mocked_ec2_instances_envs: EnvVarsDict, disabled_rabbitmq: None, disable_dynamic_service_background_task: None, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], mocked_redis_server: None, ) -> None: ... @@ -676,7 +665,9 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 - datetime.timedelta(seconds=1) ).isoformat() await auto_scale_cluster(app=initialized_app, auto_scaling_mode=auto_scaling_mode) - mocked_docker_remove_node.assert_called_once_with(mock.ANY, [fake_node], force=True) + mocked_docker_remove_node.assert_called_once_with( + mock.ANY, nodes=[fake_node], force=True + ) await _assert_ec2_instances( ec2_client, num_reservations=1, diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py index ceb7816531b..7f3663ef7dc 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py @@ -20,7 +20,7 @@ def app_environment( app_environment: EnvVarsDict, disabled_rabbitmq: None, - mocked_aws_server_envs: None, + mocked_ec2_server_envs: None, mocked_redis_server: None, monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: diff --git a/services/autoscaling/tests/unit/test_modules_dask.py b/services/autoscaling/tests/unit/test_modules_dask.py index 561ed1755d3..09b23f84b33 100644 --- a/services/autoscaling/tests/unit/test_modules_dask.py +++ b/services/autoscaling/tests/unit/test_modules_dask.py @@ -9,6 +9,7 @@ import distributed import pytest +from aws_library.ec2.models import Resources from faker import Faker from pydantic import AnyUrl, ByteSize, parse_obj_as from simcore_service_autoscaling.core.errors import ( @@ -21,7 +22,6 @@ DaskTaskId, DaskTaskResources, EC2InstanceData, - Resources, ) from simcore_service_autoscaling.modules.dask import ( DaskTask, diff --git a/services/autoscaling/tests/unit/test_modules_ec2.py b/services/autoscaling/tests/unit/test_modules_ec2.py index b64b8517f5c..0be93d22055 100644 --- a/services/autoscaling/tests/unit/test_modules_ec2.py +++ b/services/autoscaling/tests/unit/test_modules_ec2.py @@ -2,68 +2,10 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -from collections.abc import Callable -from typing import cast - -import botocore.exceptions import pytest -from faker import Faker from fastapi import FastAPI -from moto.server import ThreadedMotoServer -from pydantic import ByteSize, parse_obj_as -from pytest_simcore.helpers.utils_envs import EnvVarsDict -from simcore_service_autoscaling.core.errors import ( - ConfigurationError, - Ec2InstanceNotFoundError, - Ec2TooManyInstancesError, -) -from simcore_service_autoscaling.core.settings import ApplicationSettings, EC2Settings -from simcore_service_autoscaling.models import EC2InstanceType -from simcore_service_autoscaling.modules.ec2 import ( - AutoscalingEC2, - EC2InstanceData, - get_ec2_client, -) -from types_aiobotocore_ec2 import EC2Client -from types_aiobotocore_ec2.literals import InstanceTypeType - - -@pytest.fixture -def ec2_settings( - app_environment: EnvVarsDict, -) -> EC2Settings: - return EC2Settings.create_from_envs() - - -@pytest.fixture -def app_settings( - app_environment: EnvVarsDict, -) -> ApplicationSettings: - return ApplicationSettings.create_from_envs() - - -async def test_ec2_client_lifespan(ec2_settings: EC2Settings): - ec2 = await AutoscalingEC2.create(settings=ec2_settings) - assert ec2 - assert ec2.client - assert ec2.exit_stack - assert ec2.session - - await ec2.close() - - -async def test_ec2_client_raises_when_no_connection_available(ec2_client: EC2Client): - with pytest.raises( - botocore.exceptions.ClientError, match=r".+ AWS was not able to validate .+" - ): - await ec2_client.describe_account_attributes(DryRun=True) - - -async def test_ec2_client_with_mock_server( - mocked_aws_server_envs: None, ec2_client: EC2Client -): - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) +from simcore_service_autoscaling.core.errors import ConfigurationError +from simcore_service_autoscaling.modules.ec2 import get_ec2_client async def test_ec2_does_not_initialize_if_deactivated( @@ -73,271 +15,6 @@ async def test_ec2_does_not_initialize_if_deactivated( initialized_app: FastAPI, ): assert hasattr(initialized_app.state, "ec2_client") - assert initialized_app.state.ec2_client == None + assert initialized_app.state.ec2_client is None with pytest.raises(ConfigurationError): get_ec2_client(initialized_app) - - -async def test_ec2_client_when_ec2_server_goes_up_and_down( - mocked_aws_server: ThreadedMotoServer, - mocked_aws_server_envs: None, - ec2_client: EC2Client, -): - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) - mocked_aws_server.stop() - with pytest.raises(botocore.exceptions.EndpointConnectionError): - await ec2_client.describe_account_attributes(DryRun=True) - - # restart - mocked_aws_server.start() - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) - - -async def test_ping( - mocked_aws_server: ThreadedMotoServer, - mocked_aws_server_envs: None, - aws_allowed_ec2_instance_type_names_env: list[str], - app_settings: ApplicationSettings, - autoscaling_ec2: AutoscalingEC2, -): - assert await autoscaling_ec2.ping() is True - mocked_aws_server.stop() - assert await autoscaling_ec2.ping() is False - mocked_aws_server.start() - assert await autoscaling_ec2.ping() is True - - -async def test_get_ec2_instance_capabilities( - mocked_aws_server_envs: None, - aws_allowed_ec2_instance_type_names_env: list[str], - app_settings: ApplicationSettings, - autoscaling_ec2: AutoscalingEC2, -): - assert app_settings.AUTOSCALING_EC2_INSTANCES - assert app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES - instance_types = await autoscaling_ec2.get_ec2_instance_capabilities( - cast( - set[InstanceTypeType], - set(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES), - ) - ) - assert instance_types - assert len(instance_types) == len( - app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES - ) - - # all the instance names are found and valid - assert all( - i.name in app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES - for i in instance_types - ) - for ( - instance_type_name - ) in app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES: - assert any(i.name == instance_type_name for i in instance_types) - - -@pytest.fixture -async def fake_ec2_instance_type( - mocked_aws_server_envs: None, - ec2_client: EC2Client, -) -> EC2InstanceType: - instance_type_name: InstanceTypeType = parse_obj_as(InstanceTypeType, "c3.8xlarge") - instance_types = await ec2_client.describe_instance_types( - InstanceTypes=[instance_type_name] - ) - assert instance_types - assert "InstanceTypes" in instance_types - assert instance_types["InstanceTypes"] - assert "MemoryInfo" in instance_types["InstanceTypes"][0] - assert "SizeInMiB" in instance_types["InstanceTypes"][0]["MemoryInfo"] - assert "VCpuInfo" in instance_types["InstanceTypes"][0] - assert "DefaultVCpus" in instance_types["InstanceTypes"][0]["VCpuInfo"] - - return EC2InstanceType( - name=instance_type_name, - cpus=instance_types["InstanceTypes"][0]["VCpuInfo"]["DefaultVCpus"], - ram=parse_obj_as( - ByteSize, - f"{instance_types['InstanceTypes'][0]['MemoryInfo']['SizeInMiB']}MiB", - ), - ) - - -async def test_start_aws_instance( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - autoscaling_ec2: AutoscalingEC2, - app_settings: ApplicationSettings, - faker: Faker, - fake_ec2_instance_type: EC2InstanceType, -): - assert app_settings.AUTOSCALING_EC2_ACCESS - assert app_settings.AUTOSCALING_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - await autoscaling_ec2.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - # check we have that now in ec2 - all_instances = await ec2_client.describe_instances() - assert len(all_instances["Reservations"]) == 1 - running_instance = all_instances["Reservations"][0] - assert "Instances" in running_instance - assert len(running_instance["Instances"]) == 1 - running_instance = running_instance["Instances"][0] - assert "InstanceType" in running_instance - assert running_instance["InstanceType"] == fake_ec2_instance_type.name - assert "Tags" in running_instance - assert running_instance["Tags"] == [ - {"Key": key, "Value": value} for key, value in tags.items() - ] - - -async def test_start_aws_instance_is_limited_in_number_of_instances( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - autoscaling_ec2: AutoscalingEC2, - app_settings: ApplicationSettings, - faker: Faker, - fake_ec2_instance_type: EC2InstanceType, -): - assert app_settings.AUTOSCALING_EC2_ACCESS - assert app_settings.AUTOSCALING_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - - # create as many instances as we can - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - for _ in range(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES): - await autoscaling_ec2.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - # now creating one more shall fail - with pytest.raises(Ec2TooManyInstancesError): - await autoscaling_ec2.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - -async def test_get_instances( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - autoscaling_ec2: AutoscalingEC2, - app_settings: ApplicationSettings, - faker: Faker, - fake_ec2_instance_type: EC2InstanceType, -): - assert app_settings.AUTOSCALING_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - assert ( - await autoscaling_ec2.get_instances(app_settings.AUTOSCALING_EC2_INSTANCES, {}) - == [] - ) - - # create some instance - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - created_instances = await autoscaling_ec2.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - assert len(created_instances) == 1 - - instance_received = await autoscaling_ec2.get_instances( - app_settings.AUTOSCALING_EC2_INSTANCES, - tags=tags, - ) - assert created_instances == instance_received - - -async def test_terminate_instance( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - autoscaling_ec2: AutoscalingEC2, - app_settings: ApplicationSettings, - faker: Faker, - fake_ec2_instance_type: EC2InstanceType, -): - assert app_settings.AUTOSCALING_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - # create some instance - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - created_instances = await autoscaling_ec2.start_aws_instance( - app_settings.AUTOSCALING_EC2_INSTANCES, - fake_ec2_instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - assert len(created_instances) == 1 - - # terminate the instance - await autoscaling_ec2.terminate_instances(created_instances) - # calling it several times is ok, the instance stays a while - await autoscaling_ec2.terminate_instances(created_instances) - - -async def test_terminate_instance_not_existing_raises( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - autoscaling_ec2: AutoscalingEC2, - app_settings: ApplicationSettings, - fake_ec2_instance_data: Callable[..., EC2InstanceData], -): - assert app_settings.AUTOSCALING_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - with pytest.raises(Ec2InstanceNotFoundError): - await autoscaling_ec2.terminate_instances([fake_ec2_instance_data()]) diff --git a/services/autoscaling/tests/unit/test_utils_computational_scaling.py b/services/autoscaling/tests/unit/test_utils_computational_scaling.py index 17c3e5f5d9d..945b883187e 100644 --- a/services/autoscaling/tests/unit/test_utils_computational_scaling.py +++ b/services/autoscaling/tests/unit/test_utils_computational_scaling.py @@ -8,6 +8,7 @@ from unittest import mock import pytest +from aws_library.ec2.models import EC2InstanceType, Resources from faker import Faker from models_library.generated_models.docker_rest_api import Node as DockerNode from pydantic import ByteSize, parse_obj_as @@ -17,8 +18,6 @@ DaskTask, DaskTaskResources, EC2InstanceData, - EC2InstanceType, - Resources, ) from simcore_service_autoscaling.utils.computational_scaling import ( _DEFAULT_MAX_CPU, diff --git a/services/autoscaling/tests/unit/test_utils_docker.py b/services/autoscaling/tests/unit/test_utils_docker.py index 3a5985becf0..e9ee5c21c27 100644 --- a/services/autoscaling/tests/unit/test_utils_docker.py +++ b/services/autoscaling/tests/unit/test_utils_docker.py @@ -6,11 +6,13 @@ import datetime import itertools import random +from collections.abc import AsyncIterator, Awaitable, Callable from copy import deepcopy -from typing import Any, AsyncIterator, Awaitable, Callable +from typing import Any import aiodocker import pytest +from aws_library.ec2.models import Resources from deepdiff import DeepDiff from faker import Faker from models_library.docker import DockerGenericTag, DockerLabelKey @@ -24,7 +26,6 @@ from pydantic import ByteSize, parse_obj_as from pytest_mock.plugin import MockerFixture from servicelib.docker_utils import to_datetime -from simcore_service_autoscaling.models import Resources from simcore_service_autoscaling.modules.docker import AutoscalingDocker from simcore_service_autoscaling.utils.utils_docker import ( Node, @@ -72,7 +73,6 @@ async def _creator(labels: list[str]) -> None: "Labels": {f"{label}": "true" for label in labels}, }, ) - return yield _creator # revert labels @@ -145,14 +145,14 @@ async def test_worker_nodes( async def test_remove_monitored_down_nodes_with_empty_list_does_nothing( autoscaling_docker: AutoscalingDocker, ): - assert await remove_nodes(autoscaling_docker, []) == [] + assert await remove_nodes(autoscaling_docker, nodes=[]) == [] async def test_remove_monitored_down_nodes_of_non_down_node_does_nothing( autoscaling_docker: AutoscalingDocker, host_node: Node, ): - assert await remove_nodes(autoscaling_docker, [host_node]) == [] + assert await remove_nodes(autoscaling_docker, nodes=[host_node]) == [] @pytest.fixture @@ -174,7 +174,7 @@ async def test_remove_monitored_down_nodes_of_down_node( assert fake_docker_node.Status fake_docker_node.Status.State = NodeState.down assert fake_docker_node.Status.State == NodeState.down - assert await remove_nodes(autoscaling_docker, [fake_docker_node]) == [ + assert await remove_nodes(autoscaling_docker, nodes=[fake_docker_node]) == [ fake_docker_node ] # NOTE: this is the same as calling with aiodocker.Docker() as docker: docker.nodes.remove() @@ -190,7 +190,7 @@ async def test_remove_monitored_down_node_with_unexpected_state_does_nothing( assert fake_docker_node.Status fake_docker_node.Status = None assert not fake_docker_node.Status - assert await remove_nodes(autoscaling_docker, [fake_docker_node]) == [] + assert await remove_nodes(autoscaling_docker, nodes=[fake_docker_node]) == [] async def test_pending_service_task_with_insufficient_resources_with_no_service( diff --git a/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py b/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py index ccc283144b3..d27bc0dd28f 100644 --- a/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py +++ b/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py @@ -8,12 +8,11 @@ from datetime import timedelta import pytest +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType from faker import Faker from models_library.generated_models.docker_rest_api import Task from pydantic import ByteSize from pytest_mock import MockerFixture -from simcore_service_autoscaling.models import EC2InstanceType -from simcore_service_autoscaling.modules.ec2 import EC2InstanceData from simcore_service_autoscaling.utils.dynamic_scaling import ( try_assigning_task_to_pending_instances, ) diff --git a/services/autoscaling/tests/unit/test_utils_ec2.py b/services/autoscaling/tests/unit/test_utils_ec2.py index dc9ea43d8be..8697c8d5f11 100644 --- a/services/autoscaling/tests/unit/test_utils_ec2.py +++ b/services/autoscaling/tests/unit/test_utils_ec2.py @@ -6,15 +6,14 @@ import random import pytest +from aws_library.ec2.models import EC2InstanceType, Resources from faker import Faker from pydantic import ByteSize from simcore_service_autoscaling.core.errors import ( ConfigurationError, Ec2InstanceNotFoundError, ) -from simcore_service_autoscaling.models import Resources from simcore_service_autoscaling.utils.utils_ec2 import ( - EC2InstanceType, closest_instance_policy, compose_user_data, find_best_fitting_ec2_instance, @@ -60,10 +59,11 @@ async def test_find_best_fitting_ec2_instance_closest_instance_policy_with_resou [ ( Resources(cpus=n, ram=ByteSize(n)), - EC2InstanceType(name="fake", cpus=n, ram=ByteSize(n)), + EC2InstanceType(name="c5ad.12xlarge", cpus=n, ram=ByteSize(n)), ) for n in range(1, 30) ], + ids=str, ) async def test_find_best_fitting_ec2_instance_closest_instance_policy( needed_resources: Resources, @@ -77,7 +77,7 @@ async def test_find_best_fitting_ec2_instance_closest_instance_policy( ) SKIPPED_KEYS = ["name"] - for k in found_instance.__dict__.keys(): + for k in found_instance.__dict__: if k not in SKIPPED_KEYS: assert getattr(found_instance, k) == getattr(expected_ec2_instance, k) diff --git a/services/clusters-keeper/requirements/_base.in b/services/clusters-keeper/requirements/_base.in index 25eb0daeb8a..dc3b222d6db 100644 --- a/services/clusters-keeper/requirements/_base.in +++ b/services/clusters-keeper/requirements/_base.in @@ -9,14 +9,13 @@ # intra-repo required dependencies --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in +--requirement ../../../packages/aws-library/requirements/_base.in # service-library[fastapi] --requirement ../../../packages/service-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_fastapi.in -aioboto3 dask[distributed] fastapi packaging -types-aiobotocore[ec2] diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 2202b2fea87..3436e8e8028 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -7,25 +7,39 @@ aio-pika==9.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aioboto3==12.0.0 - # via -r requirements/_base.in + # via -r requirements/../../../packages/aws-library/requirements/_base.in aiobotocore==2.7.0 - # via aioboto3 + # via + # aioboto3 + # aiobotocore +aiocache==0.12.2 + # via -r requirements/../../../packages/aws-library/requirements/_base.in aiodebug==2.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiodocker==0.21.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiofiles==23.2.1 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiohttp==3.8.6 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -50,6 +64,9 @@ anyio==4.0.0 arrow==1.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in @@ -74,6 +91,12 @@ botocore-stubs==1.31.80 # via types-aiobotocore certifi==2023.7.22 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -116,6 +139,12 @@ exceptiongroup==1.1.3 # via anyio fastapi==0.99.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -143,6 +172,12 @@ httpcore==1.0.1 # via httpx httpx==0.25.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -165,6 +200,12 @@ importlib-metadata==6.8.0 # dask jinja2==3.1.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -182,6 +223,8 @@ jmespath==1.0.1 # botocore jsonschema==4.19.2 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in @@ -210,6 +253,8 @@ multidict==6.0.4 # yarl orjson==3.9.10 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in @@ -231,6 +276,12 @@ psutil==5.9.5 # distributed pydantic==1.10.13 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -241,6 +292,12 @@ pydantic==1.10.13 # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in @@ -254,6 +311,7 @@ pygments==2.16.1 pyinstrument==4.6.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.8.2 # via @@ -261,6 +319,12 @@ python-dateutil==2.8.2 # botocore pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -272,11 +336,18 @@ pyyaml==6.0.1 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # dask # distributed redis==5.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -287,15 +358,19 @@ redis==5.0.1 # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/./constraints.txt # -c requirements/../../../packages/service-library/requirements/././constraints.txt # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications rich==13.6.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -317,6 +392,12 @@ sortedcontainers==2.4.0 # distributed starlette==0.27.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt @@ -334,11 +415,13 @@ tblib==2.0.0 tenacity==8.2.3 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in toolz==0.12.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # dask # distributed @@ -350,14 +433,17 @@ tornado==6.3.3 tqdm==4.66.1 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in typer==0.9.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in types-aiobotocore==2.7.0 - # via -r requirements/_base.in + # via -r requirements/../../../packages/aws-library/requirements/_base.in types-aiobotocore-ec2==2.7.0 # via types-aiobotocore types-awscrt==0.19.8 @@ -376,6 +462,12 @@ typing-extensions==4.8.0 # uvicorn urllib3==1.26.16 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt diff --git a/services/clusters-keeper/requirements/_test.txt b/services/clusters-keeper/requirements/_test.txt index 85dd0cf4dcc..411492a8d69 100644 --- a/services/clusters-keeper/requirements/_test.txt +++ b/services/clusters-keeper/requirements/_test.txt @@ -264,7 +264,9 @@ python-dateutil==2.8.2 python-dotenv==1.0.0 # via -r requirements/_test.in python-jose==3.3.0 - # via moto + # via + # moto + # python-jose pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt diff --git a/services/clusters-keeper/requirements/ci.txt b/services/clusters-keeper/requirements/ci.txt index 1a24f9bf9fe..972b44964e6 100644 --- a/services/clusters-keeper/requirements/ci.txt +++ b/services/clusters-keeper/requirements/ci.txt @@ -11,6 +11,7 @@ --requirement _test.txt # installs this repo's packages +../../packages/aws-library ../../packages/models-library ../../packages/pytest-simcore ../../packages/service-library[fastapi] diff --git a/services/clusters-keeper/requirements/dev.txt b/services/clusters-keeper/requirements/dev.txt index f8a28b5d52d..5324f4c79f7 100644 --- a/services/clusters-keeper/requirements/dev.txt +++ b/services/clusters-keeper/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/aws-library --editable ../../packages/models-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] diff --git a/services/clusters-keeper/requirements/prod.txt b/services/clusters-keeper/requirements/prod.txt index 8e90729c4fc..f009eafa289 100644 --- a/services/clusters-keeper/requirements/prod.txt +++ b/services/clusters-keeper/requirements/prod.txt @@ -10,6 +10,7 @@ --requirement _base.txt # installs this repo's packages +../../packages/aws-library ../../packages/models-library ../../packages/service-library[fastapi] ../../packages/settings-library diff --git a/services/clusters-keeper/setup.py b/services/clusters-keeper/setup.py index 63b4d8ed7c5..8a67e39e66d 100755 --- a/services/clusters-keeper/setup.py +++ b/services/clusters-keeper/setup.py @@ -30,6 +30,7 @@ def read_reqs(reqs_path: Path) -> set[str]: PROD_REQUIREMENTS = tuple( read_reqs(CURRENT_DIR / "requirements" / "_base.txt") | { + "simcore-aws-library", "simcore-models-library", "simcore-service-library[fastapi]", "simcore-settings-library", diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index 4b69f776900..491cdeec854 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -1,6 +1,6 @@ import datetime from functools import cached_property -from typing import cast +from typing import Any, ClassVar, Final, cast from fastapi import FastAPI from models_library.basic_types import ( @@ -12,6 +12,7 @@ from pydantic import Field, NonNegativeInt, PositiveInt, parse_obj_as, validator from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings +from settings_library.ec2 import EC2Settings from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from settings_library.utils_logging import MixinLoggingSettings @@ -19,14 +20,23 @@ from .._meta import API_VERSION, API_VTAG, APP_NAME +CLUSTERS_KEEPER_ENV_PREFIX: Final[str] = "CLUSTERS_KEEPER_" -class EC2ClustersKeeperSettings(BaseCustomSettings): - EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID: str - EC2_CLUSTERS_KEEPER_ENDPOINT: str | None = Field( - default=None, description="do not define if using standard AWS" - ) - EC2_CLUSTERS_KEEPER_REGION_NAME: str = "us-east-1" - EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY: str + +class ClustersKeeperEC2Settings(EC2Settings): + class Config(EC2Settings.Config): + env_prefix = CLUSTERS_KEEPER_ENV_PREFIX + + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + { + f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_ACCESS_KEY_ID": "my_access_key_id", + f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_ENDPOINT": "http://my_ec2_endpoint.com", + f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_REGION_NAME": "us-east-1", + f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_SECRET_ACCESS_KEY": "my_secret_access_key", + } + ], + } class WorkersEC2InstancesSettings(BaseCustomSettings): @@ -181,7 +191,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) - CLUSTERS_KEEPER_EC2_ACCESS: EC2ClustersKeeperSettings | None = Field( + CLUSTERS_KEEPER_EC2_ACCESS: ClustersKeeperEC2Settings | None = Field( auto_default_from_env=True ) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/models.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/models.py deleted file mode 100644 index 6217940b44d..00000000000 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/models.py +++ /dev/null @@ -1,19 +0,0 @@ -import datetime -from dataclasses import dataclass -from typing import TypeAlias - -from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType - -InstancePrivateDNSName = str -EC2Tags: TypeAlias = dict[str, str] - - -@dataclass(frozen=True) -class EC2InstanceData: - launch_time: datetime.datetime - id: str # noqa: A003 - aws_private_dns: InstancePrivateDNSName - aws_public_ip: str | None - type: InstanceTypeType # noqa: A003 - state: InstanceStateNameType - tags: EC2Tags diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py index 895db73fdc8..a3a5a7229aa 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py @@ -2,6 +2,7 @@ import logging from typing import cast +from aws_library.ec2.models import EC2InstanceConfig, EC2InstanceData, EC2InstanceType from fastapi import FastAPI from models_library.users import UserID from models_library.wallets import WalletID @@ -10,7 +11,6 @@ from ..core.errors import Ec2InstanceNotFoundError from ..core.settings import get_application_settings -from ..models import EC2InstanceData from ..utils.clusters import create_startup_script from ..utils.ec2 import ( HEARTBEAT_TAG_KEY, @@ -30,16 +30,23 @@ async def create_cluster( ec2_client = get_ec2_client(app) app_settings = get_application_settings(app) assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES # nosec - return await ec2_client.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - instance_type=cast( - InstanceTypeType, - next( - iter( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - ) - ), - ), + ec2_instance_types: list[ + EC2InstanceType + ] = await ec2_client.get_ec2_instance_capabilities( + instance_type_names={ + cast( + InstanceTypeType, + next( + iter( + app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES + ) + ), + ) + } + ) + assert len(ec2_instance_types) == 1 # nosec + instance_config = EC2InstanceConfig( + type=ec2_instance_types[0], tags=creation_ec2_tags(app_settings, user_id=user_id, wallet_id=wallet_id), startup_script=create_startup_script( app_settings, @@ -47,20 +54,30 @@ async def create_cluster( app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=False ), ), + ami_id=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_AMI_ID, + key_name=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_KEY_NAME, + security_group_ids=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS, + subnet_id=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_SUBNET_ID, + ) + new_ec2_instance_data: list[EC2InstanceData] = await ec2_client.start_aws_instance( + instance_config, number_of_instances=1, + max_number_of_instances=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_MAX_INSTANCES, ) + return new_ec2_instance_data async def get_all_clusters(app: FastAPI) -> list[EC2InstanceData]: app_settings = get_application_settings(app) assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES # nosec - return await get_ec2_client(app).get_instances( + ec2_instance_data: list[EC2InstanceData] = await get_ec2_client(app).get_instances( key_names=[ app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_KEY_NAME ], tags=all_created_ec2_instances_filter(app_settings), state_names=["running"], ) + return ec2_instance_data async def get_cluster( @@ -86,7 +103,7 @@ async def get_cluster_workers( ) -> list[EC2InstanceData]: app_settings = get_application_settings(app) assert app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES # nosec - return await get_ec2_client(app).get_instances( + ec2_instance_data: list[EC2InstanceData] = await get_ec2_client(app).get_instances( key_names=[ app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_KEY_NAME ], @@ -94,6 +111,7 @@ async def get_cluster_workers( "Name": f"{get_cluster_name(app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=False)}*" }, ) + return ec2_instance_data async def cluster_heartbeat( diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py index 81fb41ff967..ce02d45166c 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py @@ -2,12 +2,12 @@ import logging import arrow +from aws_library.ec2.models import EC2InstanceData from fastapi import FastAPI from models_library.users import UserID from models_library.wallets import WalletID from ..core.settings import get_application_settings -from ..models import EC2InstanceData from ..modules.clusters import ( delete_clusters, get_all_clusters, @@ -25,7 +25,8 @@ def _get_instance_last_heartbeat(instance: EC2InstanceData) -> datetime.datetime if last_heartbeat := instance.tags.get(HEARTBEAT_TAG_KEY, None): last_heartbeat_time: datetime.datetime = arrow.get(last_heartbeat).datetime return last_heartbeat_time - return instance.launch_time + launch_time: datetime.datetime = instance.launch_time + return launch_time async def _find_terminateable_instances( diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py index 6a66adf6f00..bd74dc85f56 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/ec2.py @@ -1,293 +1,25 @@ -import contextlib import logging -from dataclasses import dataclass from typing import cast -import aioboto3 -import botocore.exceptions -from aiobotocore.session import ClientCreatorContext +from aws_library.ec2.client import SimcoreEC2API from fastapi import FastAPI -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType -from pydantic import ByteSize -from servicelib.logging_utils import log_context +from settings_library.ec2 import EC2Settings from tenacity._asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay from tenacity.wait import wait_random_exponential -from types_aiobotocore_ec2 import EC2Client -from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType -from types_aiobotocore_ec2.type_defs import FilterTypeDef -from ..core.errors import ( - ClustersKeeperRuntimeError, - ConfigurationError, - Ec2InstanceNotFoundError, - Ec2InstanceTypeInvalidError, - Ec2NotConnectedError, - Ec2TooManyInstancesError, -) -from ..core.settings import ( - EC2ClustersKeeperSettings, - PrimaryEC2InstancesSettings, - get_application_settings, -) -from ..models import EC2InstanceData, EC2Tags -from ..utils.ec2 import compose_user_data +from ..core.errors import ConfigurationError, Ec2NotConnectedError +from ..core.settings import get_application_settings logger = logging.getLogger(__name__) -@dataclass(frozen=True) -class ClustersKeeperEC2: - client: EC2Client - session: aioboto3.Session - exit_stack: contextlib.AsyncExitStack - - @classmethod - async def create(cls, settings: EC2ClustersKeeperSettings) -> "ClustersKeeperEC2": - session = aioboto3.Session() - session_client = session.client( - "ec2", - endpoint_url=settings.EC2_CLUSTERS_KEEPER_ENDPOINT, - aws_access_key_id=settings.EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID, - aws_secret_access_key=settings.EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY, - region_name=settings.EC2_CLUSTERS_KEEPER_REGION_NAME, - ) - assert isinstance(session_client, ClientCreatorContext) # nosec - exit_stack = contextlib.AsyncExitStack() - ec2_client = cast( - EC2Client, await exit_stack.enter_async_context(session_client) - ) - return cls(ec2_client, session, exit_stack) - - async def close(self) -> None: - await self.exit_stack.aclose() - - async def ping(self) -> bool: - try: - await self.client.describe_account_attributes() - return True - except Exception: # pylint: disable=broad-except - return False - - async def get_ec2_instance_capabilities( - self, - instance_type_names: set[InstanceTypeType], - ) -> list[EC2InstanceType]: - """returns the ec2 instance types from a list of instance type names - NOTE: the order might differ! - Arguments: - instance_type_names -- the types to filter with - - Raises: - Ec2InstanceTypeInvalidError: some invalid types were used as filter - ClustersKeeperRuntimeError: unexpected error communicating with EC2 - - """ - try: - instance_types = await self.client.describe_instance_types( - InstanceTypes=list(instance_type_names) - ) - list_instances: list[EC2InstanceType] = [] - for instance in instance_types.get("InstanceTypes", []): - with contextlib.suppress(KeyError): - list_instances.append( - EC2InstanceType( - name=instance["InstanceType"], - cpus=instance["VCpuInfo"]["DefaultVCpus"], - ram=ByteSize( - int(instance["MemoryInfo"]["SizeInMiB"]) * 1024 * 1024 - ), - ) - ) - return list_instances - except botocore.exceptions.ClientError as exc: - if exc.response.get("Error", {}).get("Code", "") == "InvalidInstanceType": - raise Ec2InstanceTypeInvalidError from exc - raise ClustersKeeperRuntimeError from exc # pragma: no cover - - async def start_aws_instance( - self, - instance_settings: PrimaryEC2InstancesSettings, - instance_type: InstanceTypeType, - tags: EC2Tags, - startup_script: str, - number_of_instances: int, - ) -> list[EC2InstanceData]: - with log_context( - logger, - logging.INFO, - msg=f"launching {number_of_instances} AWS instance(s) {instance_type} with {tags=}", - ): - # first check the max amount is not already reached - current_instances = await self.get_instances( - key_names=[instance_settings.PRIMARY_EC2_INSTANCES_KEY_NAME], tags=tags - ) - if ( - len(current_instances) + number_of_instances - > instance_settings.PRIMARY_EC2_INSTANCES_MAX_INSTANCES - ): - raise Ec2TooManyInstancesError( - num_instances=instance_settings.PRIMARY_EC2_INSTANCES_MAX_INSTANCES - ) - - instances = await self.client.run_instances( - ImageId=instance_settings.PRIMARY_EC2_INSTANCES_AMI_ID, - MinCount=number_of_instances, - MaxCount=number_of_instances, - InstanceType=instance_type, - InstanceInitiatedShutdownBehavior="terminate", - KeyName=instance_settings.PRIMARY_EC2_INSTANCES_KEY_NAME, - TagSpecifications=[ - { - "ResourceType": "instance", - "Tags": [ - {"Key": tag_key, "Value": tag_value} - for tag_key, tag_value in tags.items() - ], - } - ], - UserData=compose_user_data(startup_script), - NetworkInterfaces=[ - { - "AssociatePublicIpAddress": True, - "DeviceIndex": 0, - "SubnetId": instance_settings.PRIMARY_EC2_INSTANCES_SUBNET_ID, - "Groups": instance_settings.PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS, - } - ], - ) - instance_ids = [i["InstanceId"] for i in instances["Instances"]] - logger.info( - "New instances launched: %s, waiting for them to start now...", - instance_ids, - ) - - # wait for the instance to be in a pending state - # NOTE: reference to EC2 states https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-lifecycle.html - waiter = self.client.get_waiter("instance_exists") - await waiter.wait(InstanceIds=instance_ids) - logger.info("instances %s exists now.", instance_ids) - - # get the private IPs - instances = await self.client.describe_instances(InstanceIds=instance_ids) - instance_datas = [ - EC2InstanceData( - launch_time=instance["LaunchTime"], - id=instance["InstanceId"], - aws_private_dns=instance["PrivateDnsName"], - aws_public_ip=instance["PublicIpAddress"] - if "PublicIpAddress" in instance - else None, - type=instance["InstanceType"], - state=instance["State"]["Name"], - tags={tag["Key"]: tag["Value"] for tag in instance["Tags"]}, - ) - for instance in instances["Reservations"][0]["Instances"] - ] - logger.info( - "%s is available, happy computing!!", - f"{instance_datas=}", - ) - return instance_datas - - async def get_instances( - self, - *, - key_names: list[str], - tags: EC2Tags, - state_names: list[InstanceStateNameType] | None = None, - ) -> list[EC2InstanceData]: - # NOTE: be careful: Name=instance-state-name,Values=["pending", "running"] means pending OR running - # NOTE2: AND is done by repeating Name=instance-state-name,Values=pending Name=instance-state-name,Values=running - if state_names is None: - state_names = ["pending", "running"] - - filters: list[FilterTypeDef] = [ - { - "Name": "key-name", - "Values": key_names, - }, - {"Name": "instance-state-name", "Values": state_names}, - ] - filters.extend( - [{"Name": f"tag:{key}", "Values": [value]} for key, value in tags.items()] - ) - - instances = await self.client.describe_instances(Filters=filters) - all_instances = [] - for reservation in instances["Reservations"]: - assert "Instances" in reservation # nosec - for instance in reservation["Instances"]: - assert "LaunchTime" in instance # nosec - assert "InstanceId" in instance # nosec - assert "PrivateDnsName" in instance # nosec - assert "InstanceType" in instance # nosec - assert "State" in instance # nosec - assert "Name" in instance["State"] # nosec - assert "Tags" in instance # nosec - all_instances.append( - EC2InstanceData( - launch_time=instance["LaunchTime"], - id=instance["InstanceId"], - aws_private_dns=instance["PrivateDnsName"], - aws_public_ip=instance["PublicIpAddress"] - if "PublicIpAddress" in instance - else None, - type=instance["InstanceType"], - state=instance["State"]["Name"], - tags={ - tag["Key"]: tag["Value"] - for tag in instance["Tags"] - if all(k in tag for k in ["Key", "Value"]) - }, - ) - ) - logger.debug( - "received: %s instances with %s", f"{len(all_instances)}", f"{state_names=}" - ) - return all_instances - - async def terminate_instances(self, instance_datas: list[EC2InstanceData]) -> None: - try: - with log_context( - logger, - logging.INFO, - msg=f"terminating instances {[i.id for i in instance_datas]}", - ): - await self.client.terminate_instances( - InstanceIds=[i.id for i in instance_datas] - ) - except botocore.exceptions.ClientError as exc: - if ( - exc.response.get("Error", {}).get("Code", "") - == "InvalidInstanceID.NotFound" - ): - raise Ec2InstanceNotFoundError from exc - raise # pragma: no cover - - async def set_instances_tags( - self, instances: list[EC2InstanceData], *, tags: EC2Tags - ) -> None: - with log_context( - logger, - logging.DEBUG, - msg=f"setting {tags=} on instances '[{[i.id for i in instances]}]'", - ): - await self.client.create_tags( - Resources=[i.id for i in instances], - Tags=[ - {"Key": tag_key, "Value": tag_value} - for tag_key, tag_value in tags.items() - ], - ) - - def setup(app: FastAPI) -> None: async def on_startup() -> None: app.state.ec2_client = None - settings: EC2ClustersKeeperSettings | None = get_application_settings( + settings: EC2Settings | None = get_application_settings( app ).CLUSTERS_KEEPER_EC2_ACCESS @@ -295,7 +27,7 @@ async def on_startup() -> None: logger.warning("EC2 client is de-activated in the settings") return - app.state.ec2_client = client = await ClustersKeeperEC2.create(settings) + app.state.ec2_client = client = await SimcoreEC2API.create(settings) async for attempt in AsyncRetrying( reraise=True, @@ -310,15 +42,15 @@ async def on_startup() -> None: async def on_shutdown() -> None: if app.state.ec2_client: - await cast(ClustersKeeperEC2, app.state.ec2_client).close() + await cast(SimcoreEC2API, app.state.ec2_client).close() app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) -def get_ec2_client(app: FastAPI) -> ClustersKeeperEC2: +def get_ec2_client(app: FastAPI) -> SimcoreEC2API: if not app.state.ec2_client: raise ConfigurationError( msg="EC2 client is not available. Please check the configuration." ) - return cast(ClustersKeeperEC2, app.state.ec2_client) + return cast(SimcoreEC2API, app.state.ec2_client) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/clusters.py index babb123e210..ce6ab0b54c1 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/clusters.py @@ -1,3 +1,4 @@ +from aws_library.ec2.models import EC2InstanceData from fastapi import FastAPI from models_library.api_schemas_clusters_keeper.clusters import OnDemandCluster from models_library.users import UserID @@ -5,7 +6,6 @@ from servicelib.rabbitmq import RPCRouter from ..core.errors import Ec2InstanceNotFoundError -from ..models import EC2InstanceData from ..modules import clusters from ..modules.dask import ping_scheduler from ..modules.redis import get_redis_client diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py index 758564cf9dc..a68760ad762 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/rpc/ec2_instances.py @@ -1,5 +1,8 @@ +from dataclasses import asdict + +from aws_library.ec2.models import EC2InstanceType from fastapi import FastAPI -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType +from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from servicelib.rabbitmq import RPCRouter from ..modules.ec2 import get_ec2_client @@ -10,5 +13,8 @@ @router.expose() async def get_instance_type_details( app: FastAPI, *, instance_type_names: set[str] -) -> list[EC2InstanceType]: - return await get_ec2_client(app).get_ec2_instance_capabilities(instance_type_names) +) -> list[EC2InstanceTypeGet]: + instance_capabilities: list[EC2InstanceType] = await get_ec2_client( + app + ).get_ec2_instance_capabilities(instance_type_names) + return [EC2InstanceTypeGet(**asdict(t)) for t in instance_capabilities] diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py index f139a36dc19..64cdf4fcb10 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py @@ -3,6 +3,7 @@ import functools from typing import Any, Final +from aws_library.ec2.models import EC2InstanceData from models_library.api_schemas_clusters_keeper.clusters import ( ClusterState, OnDemandCluster, @@ -14,7 +15,6 @@ from .._meta import PACKAGE_DATA_FOLDER from ..core.settings import ApplicationSettings -from ..models import EC2InstanceData from .dask import get_scheduler_url _DOCKER_COMPOSE_FILE_NAME: Final[str] = "docker-compose.yml" @@ -40,8 +40,8 @@ def _convert_to_env_list(entries: list[Any]) -> str: environment_variables = [ f"DOCKER_IMAGE_TAG={app_settings.CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG}", - f"EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID}", - f"EC2_CLUSTERS_KEEPER_ENDPOINT={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_CLUSTERS_KEEPER_ENDPOINT}", + f"EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ACCESS_KEY_ID}", + f"EC2_CLUSTERS_KEEPER_ENDPOINT={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ENDPOINT}", f"WORKERS_EC2_INSTANCES_ALLOWED_TYPES={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_ALLOWED_TYPES)}", f"WORKERS_EC2_INSTANCES_AMI_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_AMI_ID}", f"WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS)}", @@ -52,8 +52,8 @@ def _convert_to_env_list(entries: list[Any]) -> str: f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}", f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}", f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}", - f"EC2_CLUSTERS_KEEPER_REGION_NAME={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_CLUSTERS_KEEPER_REGION_NAME}", - f"EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY}", + f"EC2_CLUSTERS_KEEPER_REGION_NAME={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_REGION_NAME}", + f"EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_SECRET_ACCESS_KEY}", f"LOG_LEVEL={app_settings.LOG_LEVEL}", ] diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py index 3db6435c820..ac5bb9400de 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py @@ -1,7 +1,6 @@ +from aws_library.ec2.models import EC2InstanceData from pydantic import AnyUrl, parse_obj_as -from ..models import EC2InstanceData - def get_scheduler_url(ec2_instance: EC2InstanceData) -> AnyUrl: url: AnyUrl = parse_obj_as(AnyUrl, f"tcp://{ec2_instance.aws_public_ip}:8786") diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py index 7b99b23f6c3..5774f42e0e4 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py @@ -1,12 +1,12 @@ from textwrap import dedent from typing import Final +from aws_library.ec2.models import EC2Tags from models_library.users import UserID from models_library.wallets import WalletID from .._meta import VERSION from ..core.settings import ApplicationSettings -from ..models import EC2Tags _APPLICATION_TAG_KEY: Final[str] = "io.simcore.clusters-keeper" _APPLICATION_VERSION_TAG: Final[EC2Tags] = { diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index 8fcb73ceff0..26c97e27084 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -4,46 +4,40 @@ import importlib.resources import json -import random from collections.abc import AsyncIterator, Awaitable, Callable, Iterator -from datetime import timezone from pathlib import Path from typing import Any import aiodocker import httpx import pytest -import requests import simcore_service_clusters_keeper import simcore_service_clusters_keeper.data import yaml -from aiohttp.test_utils import unused_port from asgi_lifespan import LifespanManager +from aws_library.ec2.client import SimcoreEC2API from faker import Faker from fakeredis.aioredis import FakeRedis from fastapi import FastAPI from models_library.users import UserID from models_library.wallets import WalletID -from moto.server import ThreadedMotoServer from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict from servicelib.rabbitmq import RabbitMQRPCClient +from settings_library.ec2 import EC2Settings from settings_library.rabbit import RabbitSettings from simcore_service_clusters_keeper.core.application import create_app from simcore_service_clusters_keeper.core.settings import ( + CLUSTERS_KEEPER_ENV_PREFIX, ApplicationSettings, - EC2ClustersKeeperSettings, -) -from simcore_service_clusters_keeper.modules.ec2 import ( - ClustersKeeperEC2, - EC2InstanceData, ) from simcore_service_clusters_keeper.utils.ec2 import get_cluster_name from types_aiobotocore_ec2.client import EC2Client from types_aiobotocore_ec2.literals import InstanceTypeType pytest_plugins = [ + "pytest_simcore.aws_server", + "pytest_simcore.aws_ec2_service", "pytest_simcore.dask_scheduler", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", @@ -76,6 +70,19 @@ def ec2_instances() -> list[InstanceTypeType]: return ["t2.nano", "m5.12xlarge"] +@pytest.fixture +def mocked_ec2_server_envs( + mocked_ec2_server_settings: EC2Settings, + monkeypatch: pytest.MonkeyPatch, +) -> EnvVarsDict: + # NOTE: overrides the EC2Settings with what clusters-keeper expects + changed_envs: EnvVarsDict = { + f"{CLUSTERS_KEEPER_ENV_PREFIX}{k}": v + for k, v in mocked_ec2_server_settings.dict().items() + } + return setenvs_from_dict(monkeypatch, changed_envs) + + @pytest.fixture def app_environment( mock_env_devel_environment: EnvVarsDict, @@ -88,8 +95,8 @@ def app_environment( monkeypatch, { "CLUSTERS_KEEPER_EC2_ACCESS": "{}", - "EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID": faker.pystr(), - "EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY": faker.pystr(), + "CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID": faker.pystr(), + "CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY": faker.pystr(), "CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES": "{}", "PRIMARY_EC2_INSTANCES_KEY_NAME": faker.pystr(), "PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps( @@ -111,6 +118,28 @@ def app_environment( return mock_env_devel_environment | envs +@pytest.fixture +def mocked_primary_ec2_instances_envs( + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + aws_security_group_id: str, + aws_subnet_id: str, + aws_ami_id: str, +) -> EnvVarsDict: + envs = setenvs_from_dict( + monkeypatch, + { + "PRIMARY_EC2_INSTANCES_KEY_NAME": "osparc-pytest", + "PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps( + [aws_security_group_id] + ), + "PRIMARY_EC2_INSTANCES_SUBNET_ID": aws_subnet_id, + "PRIMARY_EC2_INSTANCES_AMI_ID": aws_ami_id, + }, + ) + return app_environment | envs + + @pytest.fixture def disable_clusters_management_background_task( mocker: MockerFixture, @@ -175,53 +204,6 @@ async def async_client(initialized_app: FastAPI) -> AsyncIterator[httpx.AsyncCli yield client -@pytest.fixture(scope="module") -def mocked_aws_server() -> Iterator[ThreadedMotoServer]: - """creates a moto-server that emulates AWS services in place - NOTE: Never use a bucket with underscores it fails!! - """ - server = ThreadedMotoServer(ip_address=get_localhost_ip(), port=unused_port()) - # pylint: disable=protected-access - print( - f"--> started mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 - ) - print( - f"--> Dashboard available on [http://{server._ip_address}:{server._port}/moto-api/]" # noqa: SLF001 - ) - server.start() - yield server - server.stop() - print( - f"<-- stopped mock AWS server on {server._ip_address}:{server._port}" # noqa: SLF001 - ) - - -@pytest.fixture -def reset_aws_server_state(mocked_aws_server: ThreadedMotoServer) -> Iterator[None]: - # NOTE: reset_aws_server_state [http://docs.getmoto.org/en/latest/docs/server_mode.html#reset-api] - yield - # pylint: disable=protected-access - requests.post( - f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}/moto-api/reset", # noqa: SLF001 - timeout=10, - ) - - -@pytest.fixture -def mocked_aws_server_envs( - app_environment: EnvVarsDict, - mocked_aws_server: ThreadedMotoServer, - reset_aws_server_state: None, - monkeypatch: pytest.MonkeyPatch, -) -> EnvVarsDict: - changed_envs = { - "EC2_CLUSTERS_KEEPER_ENDPOINT": f"http://{mocked_aws_server._ip_address}:{mocked_aws_server._port}", # pylint: disable=protected-access # noqa: SLF001 - "EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID": "xxx", - "EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY": "xxx", - } - return app_environment | setenvs_from_dict(monkeypatch, changed_envs) - - @pytest.fixture def aws_allowed_ec2_instance_type_names_env( app_environment: EnvVarsDict, @@ -241,147 +223,17 @@ def aws_allowed_ec2_instance_type_names_env( return app_environment | setenvs_from_dict(monkeypatch, changed_envs) -@pytest.fixture(scope="session") -def vpc_cidr_block() -> str: - return "10.0.0.0/16" - - -@pytest.fixture -async def aws_vpc_id( - mocked_aws_server_envs: None, - app_environment: EnvVarsDict, - monkeypatch: pytest.MonkeyPatch, - ec2_client: EC2Client, - vpc_cidr_block: str, -) -> AsyncIterator[str]: - vpc = await ec2_client.create_vpc( - CidrBlock=vpc_cidr_block, - ) - vpc_id = vpc["Vpc"]["VpcId"] # type: ignore - print(f"--> Created Vpc in AWS with {vpc_id=}") - yield vpc_id - - await ec2_client.delete_vpc(VpcId=vpc_id) - print(f"<-- Deleted Vpc in AWS with {vpc_id=}") - - -@pytest.fixture(scope="session") -def subnet_cidr_block() -> str: - return "10.0.1.0/24" - - -@pytest.fixture -async def aws_subnet_id( - monkeypatch: pytest.MonkeyPatch, - aws_vpc_id: str, - ec2_client: EC2Client, - subnet_cidr_block: str, -) -> AsyncIterator[str]: - subnet = await ec2_client.create_subnet( - CidrBlock=subnet_cidr_block, VpcId=aws_vpc_id - ) - assert "Subnet" in subnet - assert "SubnetId" in subnet["Subnet"] - subnet_id = subnet["Subnet"]["SubnetId"] - print(f"--> Created Subnet in AWS with {subnet_id=}") - - monkeypatch.setenv("PRIMARY_EC2_INSTANCES_SUBNET_ID", subnet_id) - yield subnet_id - - # all the instances in the subnet must be terminated before that works - instances_in_subnet = await ec2_client.describe_instances( - Filters=[{"Name": "subnet-id", "Values": [subnet_id]}] - ) - if instances_in_subnet["Reservations"]: - print(f"--> terminating {len(instances_in_subnet)} instances in subnet") - await ec2_client.terminate_instances( - InstanceIds=[ - instance["Instances"][0]["InstanceId"] # type: ignore - for instance in instances_in_subnet["Reservations"] - ] - ) - print(f"<-- terminated {len(instances_in_subnet)} instances in subnet") - - await ec2_client.delete_subnet(SubnetId=subnet_id) - subnets = await ec2_client.describe_subnets() - print(f"<-- Deleted Subnet in AWS with {subnet_id=}") - print(f"current {subnets=}") - - -@pytest.fixture -async def aws_security_group_id( - monkeypatch: pytest.MonkeyPatch, - faker: Faker, - aws_vpc_id: str, - ec2_client: EC2Client, -) -> AsyncIterator[str]: - security_group = await ec2_client.create_security_group( - Description=faker.text(), GroupName=faker.pystr(), VpcId=aws_vpc_id - ) - security_group_id = security_group["GroupId"] - print(f"--> Created Security Group in AWS with {security_group_id=}") - monkeypatch.setenv( - "PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS", - json.dumps([security_group_id]), - ) - yield security_group_id - await ec2_client.delete_security_group(GroupId=security_group_id) - print(f"<-- Deleted Security Group in AWS with {security_group_id=}") - - -@pytest.fixture -async def aws_ami_id( - app_environment: EnvVarsDict, - mocked_aws_server_envs: None, - monkeypatch: pytest.MonkeyPatch, - ec2_client: EC2Client, -) -> str: - images = await ec2_client.describe_images() - image = random.choice(images["Images"]) # noqa S311 - ami_id = image["ImageId"] # type: ignore - monkeypatch.setenv("PRIMARY_EC2_INSTANCES_AMI_ID", ami_id) - return ami_id - - @pytest.fixture async def clusters_keeper_ec2( app_environment: EnvVarsDict, -) -> AsyncIterator[ClustersKeeperEC2]: - settings = EC2ClustersKeeperSettings.create_from_envs() - ec2 = await ClustersKeeperEC2.create(settings) +) -> AsyncIterator[SimcoreEC2API]: + settings = EC2Settings.create_from_envs() + ec2 = await SimcoreEC2API.create(settings) assert ec2 yield ec2 await ec2.close() -@pytest.fixture -async def ec2_client( - clusters_keeper_ec2: ClustersKeeperEC2, -) -> EC2Client: - return clusters_keeper_ec2.client - - -@pytest.fixture -def fake_ec2_instance_data(faker: Faker) -> Callable[..., EC2InstanceData]: - def _creator(**overrides) -> EC2InstanceData: - return EC2InstanceData( - **( - { - "launch_time": faker.date_time(tzinfo=timezone.utc), - "id": faker.uuid4(), - "aws_private_dns": faker.name(), - "aws_public_ip": faker.ipv4_public(), - "type": faker.pystr(), - "state": faker.pystr(), - "tags": faker.pydict(allowed_types=(str,)), - } - | overrides - ) - ) - - return _creator - - @pytest.fixture async def mocked_redis_server(mocker: MockerFixture) -> None: mock_redis = FakeRedis() diff --git a/services/clusters-keeper/tests/unit/test_api_health.py b/services/clusters-keeper/tests/unit/test_api_health.py index 411081de36d..dcd67cb4e98 100644 --- a/services/clusters-keeper/tests/unit/test_api_health.py +++ b/services/clusters-keeper/tests/unit/test_api_health.py @@ -20,7 +20,7 @@ def app_environment( app_environment: EnvVarsDict, enabled_rabbitmq: None, - mocked_aws_server_envs: None, + mocked_ec2_server_envs: EnvVarsDict, mocked_redis_server: None, ) -> EnvVarsDict: return app_environment diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters.py b/services/clusters-keeper/tests/unit/test_modules_clusters.py index 945bb781fde..6d0e82cd8da 100644 --- a/services/clusters-keeper/tests/unit/test_modules_clusters.py +++ b/services/clusters-keeper/tests/unit/test_modules_clusters.py @@ -5,22 +5,23 @@ import asyncio import datetime -from typing import Awaitable, Callable +from collections.abc import Awaitable, Callable import arrow import pytest +from aws_library.ec2.models import EC2InstanceData from faker import Faker from fastapi import FastAPI from models_library.users import UserID from models_library.wallets import WalletID from parse import Result, search +from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_clusters_keeper._meta import VERSION as APP_VERSION from simcore_service_clusters_keeper.core.errors import Ec2InstanceNotFoundError from simcore_service_clusters_keeper.core.settings import ( ApplicationSettings, get_application_settings, ) -from simcore_service_clusters_keeper.models import EC2InstanceData from simcore_service_clusters_keeper.modules.clusters import ( cluster_heartbeat, create_cluster, @@ -50,11 +51,9 @@ def wallet_id(faker: Faker) -> WalletID: def _base_configuration( docker_swarm: None, disabled_rabbitmq: None, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], mocked_redis_server: None, + mocked_ec2_server_envs: EnvVarsDict, + mocked_primary_ec2_instances_envs: EnvVarsDict, ) -> None: ... diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py b/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py index 4801c0233be..e09cc48c7dd 100644 --- a/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py +++ b/services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py @@ -9,6 +9,7 @@ import pytest from attr import dataclass +from aws_library.ec2.models import EC2InstanceData from faker import Faker from fastapi import FastAPI from models_library.users import UserID @@ -16,7 +17,6 @@ from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict -from simcore_service_clusters_keeper.models import EC2InstanceData from simcore_service_clusters_keeper.modules.clusters import ( cluster_heartbeat, create_cluster, @@ -58,12 +58,10 @@ def app_environment( @pytest.fixture def _base_configuration( - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], disabled_rabbitmq: None, mocked_redis_server: None, + mocked_ec2_server_envs: EnvVarsDict, + mocked_primary_ec2_instances_envs: EnvVarsDict, ) -> None: ... diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters_management_task.py b/services/clusters-keeper/tests/unit/test_modules_clusters_management_task.py index 18d440aa541..d738a5cc05e 100644 --- a/services/clusters-keeper/tests/unit/test_modules_clusters_management_task.py +++ b/services/clusters-keeper/tests/unit/test_modules_clusters_management_task.py @@ -36,7 +36,7 @@ def mock_background_task(mocker: MockerFixture) -> mock.Mock: async def test_clusters_management_task_created_and_deleted( disabled_rabbitmq: None, - mocked_aws_server_envs: None, + mocked_ec2_server_envs: EnvVarsDict, mocked_redis_server: None, mock_background_task: mock.Mock, initialized_app: FastAPI, diff --git a/services/clusters-keeper/tests/unit/test_modules_ec2.py b/services/clusters-keeper/tests/unit/test_modules_ec2.py index eb253cbe289..0820ada5818 100644 --- a/services/clusters-keeper/tests/unit/test_modules_ec2.py +++ b/services/clusters-keeper/tests/unit/test_modules_ec2.py @@ -2,71 +2,11 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -from collections.abc import Callable -from typing import cast -import botocore.exceptions import pytest -from faker import Faker from fastapi import FastAPI -from moto.server import ThreadedMotoServer -from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.utils_envs import EnvVarsDict -from simcore_service_clusters_keeper.core.errors import ( - ConfigurationError, - Ec2InstanceNotFoundError, - Ec2InstanceTypeInvalidError, - Ec2TooManyInstancesError, -) -from simcore_service_clusters_keeper.core.settings import ( - ApplicationSettings, - EC2ClustersKeeperSettings, -) -from simcore_service_clusters_keeper.modules.ec2 import ( - ClustersKeeperEC2, - EC2InstanceData, - get_ec2_client, -) -from types_aiobotocore_ec2 import EC2Client -from types_aiobotocore_ec2.literals import InstanceTypeType - - -@pytest.fixture -def ec2_settings( - app_environment: EnvVarsDict, -) -> EC2ClustersKeeperSettings: - return EC2ClustersKeeperSettings.create_from_envs() - - -@pytest.fixture -def app_settings( - app_environment: EnvVarsDict, -) -> ApplicationSettings: - return ApplicationSettings.create_from_envs() - - -async def test_ec2_client_lifespan(ec2_settings: EC2ClustersKeeperSettings): - ec2 = await ClustersKeeperEC2.create(settings=ec2_settings) - assert ec2 - assert ec2.client - assert ec2.exit_stack - assert ec2.session - - await ec2.close() - - -async def test_ec2_client_raises_when_no_connection_available(ec2_client: EC2Client): - with pytest.raises( - botocore.exceptions.ClientError, match=r".+ AWS was not able to validate .+" - ): - await ec2_client.describe_account_attributes(DryRun=True) - - -async def test_ec2_client_with_mock_server( - mocked_aws_server_envs: None, ec2_client: EC2Client -): - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) +from simcore_service_clusters_keeper.core.errors import ConfigurationError +from simcore_service_clusters_keeper.modules.ec2 import get_ec2_client async def test_ec2_does_not_initialize_if_deactivated( @@ -79,279 +19,3 @@ async def test_ec2_does_not_initialize_if_deactivated( assert initialized_app.state.ec2_client is None with pytest.raises(ConfigurationError): get_ec2_client(initialized_app) - - -async def test_ec2_client_when_ec2_server_goes_up_and_down( - mocked_aws_server: ThreadedMotoServer, - mocked_aws_server_envs: None, - ec2_client: EC2Client, -): - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) - mocked_aws_server.stop() - with pytest.raises(botocore.exceptions.EndpointConnectionError): - await ec2_client.describe_account_attributes(DryRun=True) - - # restart - mocked_aws_server.start() - # passes without exception - await ec2_client.describe_account_attributes(DryRun=True) - - -async def test_ping( - mocked_aws_server: ThreadedMotoServer, - mocked_aws_server_envs: None, - aws_allowed_ec2_instance_type_names_env: list[str], - app_settings: ApplicationSettings, - clusters_keeper_ec2: ClustersKeeperEC2, -): - assert await clusters_keeper_ec2.ping() is True - mocked_aws_server.stop() - assert await clusters_keeper_ec2.ping() is False - mocked_aws_server.start() - assert await clusters_keeper_ec2.ping() is True - - -async def test_get_ec2_instance_capabilities( - mocked_aws_server_envs: None, - aws_allowed_ec2_instance_type_names_env: list[str], - app_settings: ApplicationSettings, - clusters_keeper_ec2: ClustersKeeperEC2, -): - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - assert ( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - ) - instance_types = await clusters_keeper_ec2.get_ec2_instance_capabilities( - cast( - set[InstanceTypeType], - set( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - ), - ) - ) - assert instance_types - assert len(instance_types) == len( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - ) - - # all the instance names are found and valid - assert all( - i.name - in app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - for i in instance_types - ) - for ( - instance_type_name - ) in ( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ALLOWED_TYPES - ): - assert any(i.name == instance_type_name for i in instance_types) - - -async def test_get_ec2_instance_capabilities_with_empty_set_returns_all_options( - mocked_aws_server_envs: None, - clusters_keeper_ec2: ClustersKeeperEC2, -): - instance_types = await clusters_keeper_ec2.get_ec2_instance_capabilities(set()) - assert instance_types - # NOTE: this might need adaptation when moto is updated - assert 700 < len(instance_types) < 800 - - -async def test_get_ec2_instance_capabilities_with_invalid_names( - mocked_aws_server_envs: None, clusters_keeper_ec2: ClustersKeeperEC2, faker: Faker -): - with pytest.raises(Ec2InstanceTypeInvalidError): - await clusters_keeper_ec2.get_ec2_instance_capabilities( - faker.pyset(allowed_types=(str,)) - ) - - -async def test_start_aws_instance( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - clusters_keeper_ec2: ClustersKeeperEC2, - app_settings: ApplicationSettings, - faker: Faker, - mocker: MockerFixture, -): - assert app_settings.CLUSTERS_KEEPER_EC2_ACCESS - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - - instance_type = faker.pystr() - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - await clusters_keeper_ec2.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - # check we have that now in ec2 - all_instances = await ec2_client.describe_instances() - assert len(all_instances["Reservations"]) == 1 - running_instance = all_instances["Reservations"][0] - assert "Instances" in running_instance - assert len(running_instance["Instances"]) == 1 - running_instance = running_instance["Instances"][0] - assert "InstanceType" in running_instance - assert running_instance["InstanceType"] == instance_type - assert "Tags" in running_instance - assert running_instance["Tags"] == [ - {"Key": key, "Value": value} for key, value in tags.items() - ] - - -async def test_start_aws_instance_is_limited_in_number_of_instances( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - clusters_keeper_ec2: ClustersKeeperEC2, - app_settings: ApplicationSettings, - faker: Faker, - mocker: MockerFixture, -): - assert app_settings.CLUSTERS_KEEPER_EC2_ACCESS - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - - # create as many instances as we can - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - for _ in range( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_MAX_INSTANCES - ): - await clusters_keeper_ec2.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - faker.pystr(), - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - # now creating one more shall fail - with pytest.raises(Ec2TooManyInstancesError): - await clusters_keeper_ec2.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - faker.pystr(), - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - - -async def test_get_instances( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - clusters_keeper_ec2: ClustersKeeperEC2, - app_settings: ApplicationSettings, - faker: Faker, - mocker: MockerFixture, -): - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - assert ( - await clusters_keeper_ec2.get_instances( - key_names=[ - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_KEY_NAME - ], - tags={}, - ) - == [] - ) - - # create some instance - instance_type = faker.pystr() - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - created_instances = await clusters_keeper_ec2.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - assert len(created_instances) == 1 - - instance_received = await clusters_keeper_ec2.get_instances( - key_names=[ - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_KEY_NAME - ], - tags=tags, - ) - assert created_instances == instance_received - - -async def test_terminate_instance( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - clusters_keeper_ec2: ClustersKeeperEC2, - app_settings: ApplicationSettings, - faker: Faker, - mocker: MockerFixture, -): - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - # create some instance - instance_type = faker.pystr() - tags = faker.pydict(allowed_types=(str,)) - startup_script = faker.pystr() - created_instances = await clusters_keeper_ec2.start_aws_instance( - app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES, - instance_type, - tags=tags, - startup_script=startup_script, - number_of_instances=1, - ) - assert len(created_instances) == 1 - - # terminate the instance - await clusters_keeper_ec2.terminate_instances(created_instances) - # calling it several times is ok, the instance stays a while - await clusters_keeper_ec2.terminate_instances(created_instances) - - -async def test_terminate_instance_not_existing_raises( - mocked_aws_server_envs: None, - aws_vpc_id: str, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - ec2_client: EC2Client, - clusters_keeper_ec2: ClustersKeeperEC2, - app_settings: ApplicationSettings, - fake_ec2_instance_data: Callable[..., EC2InstanceData], -): - assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES - # we have nothing running now in ec2 - all_instances = await ec2_client.describe_instances() - assert not all_instances["Reservations"] - with pytest.raises(Ec2InstanceNotFoundError): - await clusters_keeper_ec2.terminate_instances([fake_ec2_instance_data()]) diff --git a/services/clusters-keeper/tests/unit/test_rpc_clusters.py b/services/clusters-keeper/tests/unit/test_rpc_clusters.py index 34d4dad6fec..7d84e6425c8 100644 --- a/services/clusters-keeper/tests/unit/test_rpc_clusters.py +++ b/services/clusters-keeper/tests/unit/test_rpc_clusters.py @@ -15,6 +15,7 @@ from models_library.users import UserID from models_library.wallets import WalletID from pytest_mock.plugin import MockerFixture +from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.clusters_keeper.clusters import ( get_or_create_cluster, @@ -43,11 +44,9 @@ def wallet_id(faker: Faker) -> WalletID: def _base_configuration( docker_swarm: None, enabled_rabbitmq: None, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], mocked_redis_server: None, + mocked_ec2_server_envs: EnvVarsDict, + mocked_primary_ec2_instances_envs: EnvVarsDict, initialized_app: FastAPI, ) -> None: ... diff --git a/services/clusters-keeper/tests/unit/test_rpc_ec2_instances.py b/services/clusters-keeper/tests/unit/test_rpc_ec2_instances.py index ff8a506309f..d03b6b74502 100644 --- a/services/clusters-keeper/tests/unit/test_rpc_ec2_instances.py +++ b/services/clusters-keeper/tests/unit/test_rpc_ec2_instances.py @@ -4,7 +4,8 @@ import pytest from fastapi import FastAPI -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType +from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet +from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient, RPCServerError from servicelib.rabbitmq.rpc_interfaces.clusters_keeper.ec2_instances import ( get_instance_type_details, @@ -21,11 +22,8 @@ def _base_configuration( docker_swarm: None, enabled_rabbitmq: None, - aws_subnet_id: str, - aws_security_group_id: str, - aws_ami_id: str, - aws_allowed_ec2_instance_type_names_env: list[str], mocked_redis_server: None, + mocked_ec2_server_envs: EnvVarsDict, initialized_app: FastAPI, ) -> None: ... @@ -42,7 +40,7 @@ async def test_get_instance_type_details_all_options( ) assert rpc_response assert isinstance(rpc_response, list) - assert isinstance(rpc_response[0], EC2InstanceType) + assert isinstance(rpc_response[0], EC2InstanceTypeGet) async def test_get_instance_type_details_specific_type_names( diff --git a/services/clusters-keeper/tests/unit/test_utils_clusters.py b/services/clusters-keeper/tests/unit/test_utils_clusters.py index 495cdbb3d5f..3e772896da0 100644 --- a/services/clusters-keeper/tests/unit/test_utils_clusters.py +++ b/services/clusters-keeper/tests/unit/test_utils_clusters.py @@ -7,11 +7,11 @@ from typing import Any import pytest +from aws_library.ec2.models import EC2InstanceData from faker import Faker from models_library.api_schemas_clusters_keeper.clusters import ClusterState from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_clusters_keeper.core.settings import ApplicationSettings -from simcore_service_clusters_keeper.models import EC2InstanceData from simcore_service_clusters_keeper.utils.clusters import ( create_cluster_from_ec2_instance, create_startup_script, @@ -26,7 +26,7 @@ def cluster_machines_name_prefix(faker: Faker) -> str: def test_create_startup_script( disabled_rabbitmq: None, - mocked_aws_server_envs: EnvVarsDict, + mocked_ec2_server_envs: EnvVarsDict, mocked_redis_server: None, app_settings: ApplicationSettings, cluster_machines_name_prefix: str, diff --git a/services/clusters-keeper/tests/unit/test_utils_ec2.py b/services/clusters-keeper/tests/unit/test_utils_ec2.py index e278d3dc890..51ccbe9a7da 100644 --- a/services/clusters-keeper/tests/unit/test_utils_ec2.py +++ b/services/clusters-keeper/tests/unit/test_utils_ec2.py @@ -6,6 +6,7 @@ from faker import Faker from models_library.users import UserID from models_library.wallets import WalletID +from pytest_simcore.helpers.utils_envs import EnvVarsDict from simcore_service_clusters_keeper.core.settings import ApplicationSettings from simcore_service_clusters_keeper.utils.ec2 import ( _APPLICATION_TAG_KEY, @@ -52,7 +53,7 @@ def test_get_cluster_name( def test_creation_ec2_tags( - mocked_aws_server_envs: None, + mocked_ec2_server_envs: EnvVarsDict, disabled_rabbitmq: None, mocked_redis_server: None, app_settings: ApplicationSettings, @@ -79,7 +80,7 @@ def test_creation_ec2_tags( def test_all_created_ec2_instances_filter( - mocked_aws_server_envs: None, + mocked_ec2_server_envs: EnvVarsDict, disabled_rabbitmq: None, mocked_redis_server: None, app_settings: ApplicationSettings, diff --git a/services/dask-sidecar/tests/unit/conftest.py b/services/dask-sidecar/tests/unit/conftest.py index 97c916cf5b1..40fa1466bc3 100644 --- a/services/dask-sidecar/tests/unit/conftest.py +++ b/services/dask-sidecar/tests/unit/conftest.py @@ -25,7 +25,7 @@ from yarl import URL pytest_plugins = [ - "pytest_simcore.aws_services", + "pytest_simcore.aws_server", "pytest_simcore.cli_runner", "pytest_simcore.docker_compose", "pytest_simcore.docker_registry", diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py index 57ede61ebd3..214b46d2c8c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py @@ -5,7 +5,7 @@ import aiopg.sa import arrow from dask_task_models_library.container_tasks.protocol import ContainerEnvsDict -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType +from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from models_library.api_schemas_directorv2.services import ( NodeRequirements, ServiceExtras, @@ -252,7 +252,7 @@ async def _update_project_node_resources_from_hardware_info( return try: unordered_list_ec2_instance_types: list[ - EC2InstanceType + EC2InstanceTypeGet ] = await get_instance_type_details( rabbitmq_rpc_client, instance_type_names=set(hardware_info.aws_ec2_instances), @@ -261,7 +261,7 @@ async def _update_project_node_resources_from_hardware_info( assert unordered_list_ec2_instance_types # nosec # NOTE: with the current implementation, there is no use to get the instance past the first one - def _by_type_name(ec2: EC2InstanceType) -> bool: + def _by_type_name(ec2: EC2InstanceTypeGet) -> bool: return bool(ec2.name == hardware_info.aws_ec2_instances[0]) selected_ec2_instance_type = next( diff --git a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py index 8b52cbeae8f..32cbcbf5fc5 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py +++ b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py @@ -25,8 +25,8 @@ from models_library.users import UserID from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from servicelib.common_headers import ( X_DYNAMIC_SIDECAR_REQUEST_DNS, X_DYNAMIC_SIDECAR_REQUEST_SCHEME, diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 66f94472efa..ea71f7fbb44 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -46,8 +46,8 @@ from models_library.projects_state import RunningState from models_library.users import UserID from pydantic import AnyHttpUrl, parse_obj_as -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from servicelib.fastapi.long_running_tasks.client import ( Client, ProgressMessage, diff --git a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py index fae73a6e62e..ca528632c9d 100644 --- a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py +++ b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py @@ -20,8 +20,8 @@ from models_library.users import UserID from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict -from pytest_simcore.helpers.utils_docker import get_localhost_ip from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_host import get_localhost_ip from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from utils import ( diff --git a/services/director-v2/tests/integration/02/utils.py b/services/director-v2/tests/integration/02/utils.py index 209d98f3911..1dc81ec3c2f 100644 --- a/services/director-v2/tests/integration/02/utils.py +++ b/services/director-v2/tests/integration/02/utils.py @@ -21,7 +21,7 @@ ) from models_library.users import UserID from pydantic import PositiveInt, parse_obj_as -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from servicelib.common_headers import ( X_DYNAMIC_SIDECAR_REQUEST_DNS, X_DYNAMIC_SIDECAR_REQUEST_SCHEME, diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py index f3a83d487db..1cc63734366 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_computations.py @@ -21,7 +21,7 @@ import respx from faker import Faker from fastapi import FastAPI -from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceType +from models_library.api_schemas_clusters_keeper.ec2_instances import EC2InstanceTypeGet from models_library.api_schemas_directorv2.comp_tasks import ( ComputationCreate, ComputationGet, @@ -407,7 +407,7 @@ def mocked_clusters_keeper_service_get_instance_type_details( return mocker.patch( "simcore_service_director_v2.modules.db.repositories.comp_tasks._utils.get_instance_type_details", return_value=[ - EC2InstanceType( + EC2InstanceTypeGet( name=default_pricing_plan_aws_ec2_type, cpus=fake_ec2_cpus, ram=fake_ec2_ram, @@ -426,7 +426,7 @@ def mocked_clusters_keeper_service_get_instance_type_details_with_invalid_name( return mocker.patch( "simcore_service_director_v2.modules.db.repositories.comp_tasks._utils.get_instance_type_details", return_value=[ - EC2InstanceType( + EC2InstanceTypeGet( name=faker.pystr(), cpus=fake_ec2_cpus, ram=fake_ec2_ram, diff --git a/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py b/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py index 0d5e6f2ef17..d42108bbdc2 100644 --- a/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py +++ b/services/dynamic-sidecar/tests/unit/test_modules_outputs_watcher.py @@ -337,6 +337,7 @@ async def test_does_not_trigger_on_attribute_change( assert mock_event_filter_upload_trigger.call_count == 1 +@pytest.mark.flaky(max_runs=3) async def test_port_key_sequential_event_generation( mock_long_running_upload_outputs: AsyncMock, mounted_volumes: MountedVolumes, diff --git a/services/osparc-gateway-server/tests/integration/conftest.py b/services/osparc-gateway-server/tests/integration/conftest.py index 1e1161670ad..7ac15707662 100644 --- a/services/osparc-gateway-server/tests/integration/conftest.py +++ b/services/osparc-gateway-server/tests/integration/conftest.py @@ -4,7 +4,7 @@ import asyncio import json -from typing import Any, AsyncIterator, Awaitable, Callable, Union +from typing import Any, AsyncIterator, Awaitable, Callable import aiodocker import dask_gateway @@ -19,7 +19,7 @@ OSPARC_SCHEDULER_API_PORT, OSPARC_SCHEDULER_DASHBOARD_PORT, ) -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from tenacity._asyncio import AsyncRetrying from tenacity.wait import wait_fixed @@ -56,7 +56,7 @@ def gateway_password(faker: Faker) -> str: return faker.password() -def _convert_to_dict(c: Union[traitlets.config.Config, dict]) -> dict[str, Any]: +def _convert_to_dict(c: traitlets.config.Config | dict) -> dict[str, Any]: converted_dict = {} for x, y in c.items(): if isinstance(y, (dict, traitlets.config.Config)): diff --git a/services/osparc-gateway-server/tests/integration/test_clusters.py b/services/osparc-gateway-server/tests/integration/test_clusters.py index 1ba132b9ef9..87f776bd5c3 100644 --- a/services/osparc-gateway-server/tests/integration/test_clusters.py +++ b/services/osparc-gateway-server/tests/integration/test_clusters.py @@ -11,7 +11,7 @@ from aiodocker import Docker from dask_gateway import Gateway from faker import Faker -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from tenacity._asyncio import AsyncRetrying from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed diff --git a/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py b/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py index 4b46e5e226e..fa7e3554870 100644 --- a/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py +++ b/services/osparc-gateway-server/tests/integration/test_dask_sidecar.py @@ -7,7 +7,7 @@ import aiodocker import pytest from faker import Faker -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from tenacity._asyncio import AsyncRetrying from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed diff --git a/services/osparc-gateway-server/tests/integration/test_gateway.py b/services/osparc-gateway-server/tests/integration/test_gateway.py index 205772a0cee..44ae68ea3e3 100644 --- a/services/osparc-gateway-server/tests/integration/test_gateway.py +++ b/services/osparc-gateway-server/tests/integration/test_gateway.py @@ -9,7 +9,7 @@ from dask_gateway_server.app import DaskGateway from faker import Faker from osparc_gateway_server.backend.osparc import OsparcBackend -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip @pytest.fixture( diff --git a/services/osparc-gateway-server/tests/system/test_deploy.py b/services/osparc-gateway-server/tests/system/test_deploy.py index 4dd4e114ec3..79cc1221884 100644 --- a/services/osparc-gateway-server/tests/system/test_deploy.py +++ b/services/osparc-gateway-server/tests/system/test_deploy.py @@ -12,7 +12,7 @@ import dask_gateway import pytest from faker import Faker -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from tenacity._asyncio import AsyncRetrying from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 531d4f67927..8864901f427 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -42,7 +42,7 @@ from moto.server import ThreadedMotoServer from pydantic import ByteSize, parse_obj_as from pytest_simcore.helpers.utils_assert import assert_status -from pytest_simcore.helpers.utils_docker import get_localhost_ip +from pytest_simcore.helpers.utils_host import get_localhost_ip from simcore_postgres_database.storage_models import file_meta_data, projects, users from simcore_service_storage.application import create from simcore_service_storage.dsm import get_dsm_provider From 125ae0b3f6c6af919af501ed78ed024c8defe365 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 14 Nov 2023 15:54:46 +0100 Subject: [PATCH 07/24] =?UTF-8?q?=F0=9F=90=9BDynamic=20autoscaling:=20not?= =?UTF-8?q?=20scaling=20beside=201=20machine=20(#5026)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/simcore_service_autoscaling/models.py | 15 +- .../modules/auto_scaling_core.py | 162 ++++++++++++------ .../modules/auto_scaling_mode_base.py | 14 +- .../auto_scaling_mode_computational.py | 15 +- .../modules/auto_scaling_mode_dynamic.py | 18 +- .../utils/auto_scaling_core.py | 79 +++++---- .../utils/computational_scaling.py | 41 +++-- .../utils/dynamic_scaling.py | 48 +++--- services/autoscaling/tests/manual/.env-devel | 5 + services/autoscaling/tests/manual/Makefile | 4 + services/autoscaling/tests/manual/README.md | 47 ++++- services/autoscaling/tests/unit/conftest.py | 2 +- ...test_modules_auto_scaling_computational.py | 153 +++++++++-------- .../unit/test_modules_auto_scaling_dynamic.py | 34 ++-- .../unit/test_utils_computational_scaling.py | 32 ++-- .../tests/unit/test_utils_dynamic_scaling.py | 37 ++-- 16 files changed, 438 insertions(+), 268 deletions(-) diff --git a/services/autoscaling/src/simcore_service_autoscaling/models.py b/services/autoscaling/src/simcore_service_autoscaling/models.py index 9e3ffec2df9..d9c3bf3978d 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/models.py +++ b/services/autoscaling/src/simcore_service_autoscaling/models.py @@ -1,10 +1,23 @@ from dataclasses import dataclass, field from typing import Any, TypeAlias -from aws_library.ec2.models import EC2InstanceData +from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources from models_library.generated_models.docker_rest_api import Node +@dataclass(frozen=True, kw_only=True) +class AssignedTasksToInstance: + instance: EC2InstanceData + available_resources: Resources + assigned_tasks: list + + +@dataclass(frozen=True, kw_only=True) +class AssignedTasksToInstanceType: + instance_type: EC2InstanceType + assigned_tasks: list + + @dataclass(frozen=True) class AssociatedInstance: node: Node diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py index 382b7f85c6c..08cd815c5ad 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py @@ -30,7 +30,12 @@ Ec2TooManyInstancesError, ) from ..core.settings import ApplicationSettings, get_application_settings -from ..models import AssociatedInstance, Cluster +from ..models import ( + AssignedTasksToInstance, + AssignedTasksToInstanceType, + AssociatedInstance, + Cluster, +) from ..utils import utils_docker, utils_ec2 from ..utils.auto_scaling_core import ( associate_ec2_instances_with_nodes, @@ -268,6 +273,56 @@ async def _activate_drained_nodes( ) +async def _try_assign_tasks_to_instances( + app: FastAPI, + task, + auto_scaling_mode: BaseAutoscaling, + task_defined_ec2_type: InstanceTypeType | None, + active_instances_to_tasks: list[AssignedTasksToInstance], + pending_instances_to_tasks: list[AssignedTasksToInstance], + drained_instances_to_tasks: list[AssignedTasksToInstance], + needed_new_instance_types_for_tasks: list[AssignedTasksToInstanceType], +) -> bool: + ( + filtered_active_instance_to_task, + filtered_pending_instance_to_task, + filtered_drained_instances_to_task, + filtered_needed_new_instance_types_to_task, + ) = filter_by_task_defined_instance( + task_defined_ec2_type, + active_instances_to_tasks, + pending_instances_to_tasks, + drained_instances_to_tasks, + needed_new_instance_types_for_tasks, + ) + # try to assign the task to one of the active, pending or net created instances + if ( + await auto_scaling_mode.try_assigning_task_to_instances( + app, + task, + filtered_active_instance_to_task, + notify_progress=False, + ) + or await auto_scaling_mode.try_assigning_task_to_instances( + app, + task, + filtered_pending_instance_to_task, + notify_progress=True, + ) + or await auto_scaling_mode.try_assigning_task_to_instances( + app, + task, + filtered_drained_instances_to_task, + notify_progress=False, + ) + or auto_scaling_mode.try_assigning_task_to_instance_types( + task, filtered_needed_new_instance_types_to_task + ) + ): + return True + return False + + async def _find_needed_instances( app: FastAPI, pending_tasks: list, @@ -275,67 +330,51 @@ async def _find_needed_instances( cluster: Cluster, auto_scaling_mode: BaseAutoscaling, ) -> dict[EC2InstanceType, int]: - type_to_instance_map = {t.name: t for t in available_ec2_types} - # 1. check first the pending task needs - active_instances_to_tasks: list[tuple[EC2InstanceData, list]] = [ - (i.ec2_instance, []) for i in cluster.active_nodes + active_instances_to_tasks: list[AssignedTasksToInstance] = [ + AssignedTasksToInstance( + instance=i.ec2_instance, + assigned_tasks=[], + available_resources=i.ec2_instance.resources + - await auto_scaling_mode.compute_node_used_resources(app, i), + ) + for i in cluster.active_nodes ] - pending_instances_to_tasks: list[tuple[EC2InstanceData, list]] = [ - (i, []) for i in cluster.pending_ec2s + pending_instances_to_tasks: list[AssignedTasksToInstance] = [ + AssignedTasksToInstance( + instance=i, assigned_tasks=[], available_resources=i.resources + ) + for i in cluster.pending_ec2s ] - drained_instances_to_tasks: list[tuple[EC2InstanceData, list]] = [ - (i.ec2_instance, []) for i in cluster.drained_nodes + drained_instances_to_tasks: list[AssignedTasksToInstance] = [ + AssignedTasksToInstance( + instance=i.ec2_instance, + assigned_tasks=[], + available_resources=i.ec2_instance.resources, + ) + for i in cluster.drained_nodes ] - needed_new_instance_types_for_tasks: list[tuple[EC2InstanceType, list]] = [] + needed_new_instance_types_for_tasks: list[AssignedTasksToInstanceType] = [] for task in pending_tasks: task_defined_ec2_type = await auto_scaling_mode.get_task_defined_instance( app, task ) - ( - filtered_active_instance_to_task, - filtered_pending_instance_to_task, - filtered_drained_instances_to_task, - filtered_needed_new_instance_types_to_task, - ) = filter_by_task_defined_instance( + _logger.info( + "task %s %s", + task, + f"defines ec2 type as {task_defined_ec2_type}" + if task_defined_ec2_type + else "does NOT define ec2 type", + ) + if await _try_assign_tasks_to_instances( + app, + task, + auto_scaling_mode, task_defined_ec2_type, active_instances_to_tasks, pending_instances_to_tasks, drained_instances_to_tasks, needed_new_instance_types_for_tasks, - ) - - # try to assign the task to one of the active, pending or net created instances - _logger.debug( - "Try to assign %s to any active/pending/created instance in the %s", - f"{task}", - f"{cluster=}", - ) - if ( - await auto_scaling_mode.try_assigning_task_to_instances( - app, - task, - filtered_active_instance_to_task, - type_to_instance_map, - notify_progress=False, - ) - or await auto_scaling_mode.try_assigning_task_to_instances( - app, - task, - filtered_pending_instance_to_task, - type_to_instance_map, - notify_progress=True, - ) - or await auto_scaling_mode.try_assigning_task_to_instances( - app, - task, - filtered_drained_instances_to_task, - type_to_instance_map, - notify_progress=False, - ) - or auto_scaling_mode.try_assigning_task_to_instance_types( - task, filtered_needed_new_instance_types_to_task - ) ): continue @@ -346,7 +385,11 @@ async def _find_needed_instances( defined_ec2 = find_selected_instance_type_for_task( task_defined_ec2_type, available_ec2_types, auto_scaling_mode, task ) - needed_new_instance_types_for_tasks.append((defined_ec2, [task])) + needed_new_instance_types_for_tasks.append( + AssignedTasksToInstanceType( + instance_type=defined_ec2, assigned_tasks=[task] + ) + ) else: # we go for best fitting type best_ec2_instance = utils_ec2.find_best_fitting_ec2_instance( @@ -354,7 +397,11 @@ async def _find_needed_instances( auto_scaling_mode.get_max_resources_from_task(task), score_type=utils_ec2.closest_instance_policy, ) - needed_new_instance_types_for_tasks.append((best_ec2_instance, [task])) + needed_new_instance_types_for_tasks.append( + AssignedTasksToInstanceType( + instance_type=best_ec2_instance, assigned_tasks=[task] + ) + ) except Ec2InstanceNotFoundError: _logger.exception( "Task %s needs more resources than any EC2 instance " @@ -365,7 +412,10 @@ async def _find_needed_instances( _logger.exception("Unexpected error:") num_instances_per_type = collections.defaultdict( - int, collections.Counter(t for t, _ in needed_new_instance_types_for_tasks) + int, + collections.Counter( + t.instance_type for t in needed_new_instance_types_for_tasks + ), ) # 2. check the buffer needs @@ -379,9 +429,7 @@ async def _find_needed_instances( ) > 0: # check if some are already pending remaining_pending_instances = [ - instance - for instance, assigned_tasks in pending_instances_to_tasks - if not assigned_tasks + i.instance for i in pending_instances_to_tasks if not i.assigned_tasks ] if len(remaining_pending_instances) < ( app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MACHINES_BUFFER @@ -622,10 +670,14 @@ async def _autoscale_cluster( ) -> Cluster: # 1. check if we have pending tasks and resolve them by activating some drained nodes unrunnable_tasks = await auto_scaling_mode.list_unrunnable_tasks(app) + _logger.info("found %s unrunnable tasks", len(unrunnable_tasks)) # 2. try to activate drained nodes to cover some of the tasks still_unrunnable_tasks, cluster = await _activate_drained_nodes( app, cluster, unrunnable_tasks, auto_scaling_mode ) + _logger.info( + "still %s unrunnable tasks after node activation", len(still_unrunnable_tasks) + ) # let's check if there are still pending tasks or if the reserve was used app_settings = get_application_settings(app) diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py index 08ae67d435f..3c9e2742075 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py @@ -1,15 +1,18 @@ from abc import ABC, abstractmethod -from collections.abc import Iterable from dataclasses import dataclass -from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources +from aws_library.ec2.models import EC2InstanceData, Resources from fastapi import FastAPI from models_library.docker import DockerLabelKey from models_library.generated_models.docker_rest_api import Node as DockerNode from servicelib.logging_utils import LogLevelInt from types_aiobotocore_ec2.literals import InstanceTypeType -from ..models import AssociatedInstance +from ..models import ( + AssignedTasksToInstance, + AssignedTasksToInstanceType, + AssociatedInstance, +) @dataclass @@ -48,8 +51,7 @@ def try_assigning_task_to_node( async def try_assigning_task_to_instances( app: FastAPI, pending_task, - instances_to_tasks: Iterable[tuple[EC2InstanceData, list]], - type_to_instance_map: dict[str, EC2InstanceType], + instances_to_tasks: list[AssignedTasksToInstance], *, notify_progress: bool ) -> bool: @@ -59,7 +61,7 @@ async def try_assigning_task_to_instances( @abstractmethod def try_assigning_task_to_instance_types( pending_task, - instance_types_to_tasks: Iterable[tuple[EC2InstanceType, list]], + instance_types_to_tasks: list[AssignedTasksToInstanceType], ) -> bool: ... diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py index f627e0398a1..9ddb7df5c14 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py @@ -2,7 +2,7 @@ import logging from collections.abc import Iterable -from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources +from aws_library.ec2.models import EC2InstanceData, Resources from fastapi import FastAPI from models_library.docker import ( DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY, @@ -15,7 +15,12 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import AssociatedInstance, DaskTask +from ..models import ( + AssignedTasksToInstance, + AssignedTasksToInstanceType, + AssociatedInstance, + DaskTask, +) from ..utils import computational_scaling as utils from ..utils import utils_docker, utils_ec2 from . import dask @@ -65,12 +70,10 @@ def try_assigning_task_to_node( async def try_assigning_task_to_instances( app: FastAPI, pending_task, - instances_to_tasks: Iterable[tuple[EC2InstanceData, list]], - type_to_instance_map: dict[str, EC2InstanceType], + instances_to_tasks: list[AssignedTasksToInstance], *, notify_progress: bool ) -> bool: - assert type_to_instance_map # nosec return await utils.try_assigning_task_to_instances( app, pending_task, @@ -81,7 +84,7 @@ async def try_assigning_task_to_instances( @staticmethod def try_assigning_task_to_instance_types( pending_task, - instance_types_to_tasks: Iterable[tuple[EC2InstanceType, list]], + instance_types_to_tasks: list[AssignedTasksToInstanceType], ) -> bool: return utils.try_assigning_task_to_instance_types( pending_task, instance_types_to_tasks diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py index b2107f2a043..ec3870eb422 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py @@ -1,6 +1,6 @@ from collections.abc import Iterable -from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources +from aws_library.ec2.models import EC2InstanceData, Resources from fastapi import FastAPI from models_library.docker import DockerLabelKey from models_library.generated_models.docker_rest_api import Node, Task @@ -8,7 +8,11 @@ from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import AssociatedInstance +from ..models import ( + AssignedTasksToInstance, + AssignedTasksToInstanceType, + AssociatedInstance, +) from ..utils import dynamic_scaling as utils from ..utils import utils_docker, utils_ec2 from ..utils.rabbitmq import log_tasks_message, progress_tasks_message @@ -57,25 +61,23 @@ def try_assigning_task_to_node( async def try_assigning_task_to_instances( app: FastAPI, pending_task, - instances_to_tasks: Iterable[tuple[EC2InstanceData, list]], - type_to_instance_map: dict[str, EC2InstanceType], + instances_to_tasks: list[AssignedTasksToInstance], *, notify_progress: bool ) -> bool: - return await utils.try_assigning_task_to_pending_instances( + return await utils.try_assigning_task_to_instances( app, pending_task, instances_to_tasks, - type_to_instance_map, notify_progress=notify_progress, ) @staticmethod def try_assigning_task_to_instance_types( pending_task, - instance_types_to_tasks: Iterable[tuple[EC2InstanceType, list]], + instance_types_to_tasks: list[AssignedTasksToInstanceType], ) -> bool: - return utils.try_assigning_task_to_instances( + return utils.try_assigning_task_to_instance_types( pending_task, instance_types_to_tasks ) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py index 701f9d1ae39..106b68ea352 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py @@ -9,7 +9,11 @@ from ..core.errors import Ec2InstanceInvalidError, Ec2InvalidDnsNameError from ..core.settings import ApplicationSettings -from ..models import AssociatedInstance +from ..models import ( + AssignedTasksToInstance, + AssignedTasksToInstanceType, + AssociatedInstance, +) from ..modules.auto_scaling_mode_base import BaseAutoscaling from . import utils_docker @@ -108,52 +112,63 @@ def _instance_type_by_type_name( def _instance_type_map_by_type_name( - mapping: tuple[EC2InstanceType, list], *, type_name: InstanceTypeType | None + mapping: AssignedTasksToInstanceType, *, type_name: InstanceTypeType | None ) -> bool: - ec2_type, _ = mapping - return _instance_type_by_type_name(ec2_type, type_name=type_name) + return _instance_type_by_type_name(mapping.instance_type, type_name=type_name) def _instance_data_map_by_type_name( - mapping: tuple[EC2InstanceData, list], *, type_name: InstanceTypeType | None + mapping: AssignedTasksToInstance, *, type_name: InstanceTypeType | None ) -> bool: if type_name is None: return True - ec2_data, _ = mapping - return bool(ec2_data.type == type_name) + return bool(mapping.instance.type == type_name) def filter_by_task_defined_instance( instance_type_name: InstanceTypeType | None, - active_instances_to_tasks, - pending_instances_to_tasks, - drained_instances_to_tasks, - needed_new_instance_types_for_tasks, -) -> tuple: + active_instances_to_tasks: list[AssignedTasksToInstance], + pending_instances_to_tasks: list[AssignedTasksToInstance], + drained_instances_to_tasks: list[AssignedTasksToInstance], + needed_new_instance_types_for_tasks: list[AssignedTasksToInstanceType], +) -> tuple[ + list[AssignedTasksToInstance], + list[AssignedTasksToInstance], + list[AssignedTasksToInstance], + list[AssignedTasksToInstanceType], +]: return ( - filter( - functools.partial( - _instance_data_map_by_type_name, type_name=instance_type_name - ), - active_instances_to_tasks, + list( + filter( + functools.partial( + _instance_data_map_by_type_name, type_name=instance_type_name + ), + active_instances_to_tasks, + ) ), - filter( - functools.partial( - _instance_data_map_by_type_name, type_name=instance_type_name - ), - pending_instances_to_tasks, + list( + filter( + functools.partial( + _instance_data_map_by_type_name, type_name=instance_type_name + ), + pending_instances_to_tasks, + ) ), - filter( - functools.partial( - _instance_data_map_by_type_name, type_name=instance_type_name - ), - drained_instances_to_tasks, + list( + filter( + functools.partial( + _instance_data_map_by_type_name, type_name=instance_type_name + ), + drained_instances_to_tasks, + ) ), - filter( - functools.partial( - _instance_type_map_by_type_name, type_name=instance_type_name - ), - needed_new_instance_types_for_tasks, + list( + filter( + functools.partial( + _instance_type_map_by_type_name, type_name=instance_type_name + ), + needed_new_instance_types_for_tasks, + ) ), ) diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py b/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py index 8fcefb5c8de..f7ff8253716 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/computational_scaling.py @@ -3,14 +3,19 @@ from collections.abc import Iterable from typing import Final -from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources +from aws_library.ec2.models import Resources from dask_task_models_library.constants import DASK_TASK_EC2_RESOURCE_RESTRICTION_KEY from fastapi import FastAPI from servicelib.utils_formatting import timedelta_as_minute_second from types_aiobotocore_ec2.literals import InstanceTypeType from ..core.settings import get_application_settings -from ..models import AssociatedInstance, DaskTask +from ..models import ( + AssignedTasksToInstance, + AssignedTasksToInstanceType, + AssociatedInstance, + DaskTask, +) _logger = logging.getLogger(__name__) @@ -54,7 +59,7 @@ def try_assigning_task_to_node( async def try_assigning_task_to_instances( app: FastAPI, pending_task: DaskTask, - instances_to_tasks: Iterable[tuple[EC2InstanceData, list[DaskTask]]], + instances_to_tasks: list[AssignedTasksToInstance], *, notify_progress: bool, ) -> bool: @@ -63,20 +68,23 @@ async def try_assigning_task_to_instances( instance_max_time_to_start = ( app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_START_TIME ) - for instance, instance_assigned_tasks in instances_to_tasks: - instance_total_resources = instance.resources + for assigned_tasks_to_instance in instances_to_tasks: tasks_needed_resources = _compute_tasks_needed_resources( - instance_assigned_tasks + assigned_tasks_to_instance.assigned_tasks ) if ( - instance_total_resources - tasks_needed_resources + assigned_tasks_to_instance.available_resources - tasks_needed_resources ) >= get_max_resources_from_dask_task(pending_task): - instance_assigned_tasks.append(pending_task) + assigned_tasks_to_instance.assigned_tasks.append(pending_task) if notify_progress: now = datetime.datetime.now(datetime.timezone.utc) - time_since_launch = now - instance.launch_time + time_since_launch = ( + now - assigned_tasks_to_instance.instance.launch_time + ) estimated_time_to_completion = ( - instance.launch_time + instance_max_time_to_start - now + assigned_tasks_to_instance.instance.launch_time + + instance_max_time_to_start + - now ) _logger.info( "LOG: %s", @@ -94,16 +102,19 @@ async def try_assigning_task_to_instances( def try_assigning_task_to_instance_types( pending_task: DaskTask, - instance_types_to_tasks: Iterable[tuple[EC2InstanceType, list[DaskTask]]], + instance_types_to_tasks: list[AssignedTasksToInstanceType], ) -> bool: - for instance, instance_assigned_tasks in instance_types_to_tasks: - instance_total_resource = Resources(cpus=instance.cpus, ram=instance.ram) + for assigned_tasks_to_instance_type in instance_types_to_tasks: + instance_total_resource = Resources( + cpus=assigned_tasks_to_instance_type.instance_type.cpus, + ram=assigned_tasks_to_instance_type.instance_type.ram, + ) tasks_needed_resources = _compute_tasks_needed_resources( - instance_assigned_tasks + assigned_tasks_to_instance_type.assigned_tasks ) if ( instance_total_resource - tasks_needed_resources ) >= get_max_resources_from_dask_task(pending_task): - instance_assigned_tasks.append(pending_task) + assigned_tasks_to_instance_type.assigned_tasks.append(pending_task) return True return False diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py b/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py index 93af22f7f4e..ab6f6756d43 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/dynamic_scaling.py @@ -2,13 +2,17 @@ import logging from collections.abc import Iterable -from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources +from aws_library.ec2.models import Resources from fastapi import FastAPI from models_library.generated_models.docker_rest_api import Task from servicelib.utils_formatting import timedelta_as_minute_second from ..core.settings import get_application_settings -from ..models import AssociatedInstance +from ..models import ( + AssignedTasksToInstance, + AssignedTasksToInstanceType, + AssociatedInstance, +) from . import utils_docker from .rabbitmq import log_tasks_message, progress_tasks_message @@ -32,28 +36,30 @@ def try_assigning_task_to_node( return False -def try_assigning_task_to_instances( +def try_assigning_task_to_instance_types( pending_task: Task, - instance_types_to_tasks: Iterable[tuple[EC2InstanceType, list[Task]]], + instance_types_to_tasks: list[AssignedTasksToInstanceType], ) -> bool: - for instance, instance_assigned_tasks in instance_types_to_tasks: - instance_total_resource = Resources(cpus=instance.cpus, ram=instance.ram) + for assigned_tasks_to_instance_type in instance_types_to_tasks: + instance_total_resource = Resources( + cpus=assigned_tasks_to_instance_type.instance_type.cpus, + ram=assigned_tasks_to_instance_type.instance_type.ram, + ) tasks_needed_resources = utils_docker.compute_tasks_needed_resources( - instance_assigned_tasks + assigned_tasks_to_instance_type.assigned_tasks ) if ( instance_total_resource - tasks_needed_resources ) >= utils_docker.get_max_resources_from_docker_task(pending_task): - instance_assigned_tasks.append(pending_task) + assigned_tasks_to_instance_type.assigned_tasks.append(pending_task) return True return False -async def try_assigning_task_to_pending_instances( +async def try_assigning_task_to_instances( app: FastAPI, pending_task: Task, - instances_to_tasks: Iterable[tuple[EC2InstanceData, list[Task]]], - type_to_instance_map: dict[str, EC2InstanceType], + instances_to_tasks: list[AssignedTasksToInstance], *, notify_progress: bool, ) -> bool: @@ -62,23 +68,23 @@ async def try_assigning_task_to_pending_instances( instance_max_time_to_start = ( app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_START_TIME ) - for instance, instance_assigned_tasks in instances_to_tasks: - instance_type = type_to_instance_map[instance.type] - instance_total_resources = Resources( - cpus=instance_type.cpus, ram=instance_type.ram - ) + for assigned_tasks_to_instance in instances_to_tasks: tasks_needed_resources = utils_docker.compute_tasks_needed_resources( - instance_assigned_tasks + assigned_tasks_to_instance.assigned_tasks ) if ( - instance_total_resources - tasks_needed_resources + assigned_tasks_to_instance.available_resources - tasks_needed_resources ) >= utils_docker.get_max_resources_from_docker_task(pending_task): - instance_assigned_tasks.append(pending_task) + assigned_tasks_to_instance.assigned_tasks.append(pending_task) if notify_progress: now = datetime.datetime.now(datetime.timezone.utc) - time_since_launch = now - instance.launch_time + time_since_launch = ( + now - assigned_tasks_to_instance.instance.launch_time + ) estimated_time_to_completion = ( - instance.launch_time + instance_max_time_to_start - now + assigned_tasks_to_instance.instance.launch_time + + instance_max_time_to_start + - now ) await log_tasks_message( diff --git a/services/autoscaling/tests/manual/.env-devel b/services/autoscaling/tests/manual/.env-devel index 63bbdbb9fb2..80e54412ce4 100644 --- a/services/autoscaling/tests/manual/.env-devel +++ b/services/autoscaling/tests/manual/.env-devel @@ -10,6 +10,11 @@ EC2_INSTANCES_SUBNET_ID=XXXXXXXXXX EC2_SECRET_ACCESS_KEY=XXXXXXXXXX EC2_INSTANCES_NAME_PREFIX=testing-osparc-computational-cluster LOG_FORMAT_LOCAL_DEV_ENABLED=True +# define the following to activate dynamic autoscaling +# NODES_MONITORING_NEW_NODES_LABELS="[\"testing.autoscaled-node\"]" +# NODES_MONITORING_NODE_LABELS="[\"testing.monitored-node\"]" +# NODES_MONITORING_SERVICE_LABELS="[\"testing.monitored-service\"]" + # may be activated or not # RABBIT_HOST=rabbit # RABBIT_PASSWORD=test diff --git a/services/autoscaling/tests/manual/Makefile b/services/autoscaling/tests/manual/Makefile index 48ab85d9068..a5f6be51ad4 100644 --- a/services/autoscaling/tests/manual/Makefile +++ b/services/autoscaling/tests/manual/Makefile @@ -18,10 +18,14 @@ include ../../../../scripts/common-service.Makefile up-devel: .init-swarm .stack-devel.yml ## starts local test application @docker stack deploy --with-registry-auth --compose-file=.stack-devel.yml autoscaling + # to follow logs of autoscaling, run + # docker service logs --follow autoscaling_autoscaling up-computational-devel: .init-swarm .stack-computational-devel.yml ## starts local test application in computational mode # DASK_MONITORING_URL set to $(DASK_MONITORING_URL) @docker stack deploy --with-registry-auth --compose-file=.stack-computational-devel.yml comp-autoscaling + # to follow logs of autoscaling, run + # docker service logs --follow comp-autoscaling_autoscaling down: .env ## stops local test app dependencies (running bare metal against AWS) # remove stacks diff --git a/services/autoscaling/tests/manual/README.md b/services/autoscaling/tests/manual/README.md index a8be43853f5..e44e739d182 100644 --- a/services/autoscaling/tests/manual/README.md +++ b/services/autoscaling/tests/manual/README.md @@ -5,15 +5,17 @@ The autoscaling service may be started either in computational mode or in dynami The computational mode is used in conjunction with a dask-scheduler/dask-worker subsystem. The dynamic mode is used directly with docker swarm facilities. -## computational mode - -When ```DASK_MONITORING_URL``` is set the computational mode is enabled. - ### requirements 1. AWS EC2 access 2. a machine running in EC2 with docker installed and access to osparc-simcore repository + +## computational mode + +When ```DASK_MONITORING_URL``` is set the computational mode is enabled. + + ### instructions 1. prepare autoscaling @@ -67,3 +69,40 @@ future.done() # shall return True once done # remove the future from the dask-scheduler memory, shall trigger the autoscaling service to remove the created machine del future ``` + + +## dynamic mode + +When ```NODES_MONITORING_NEW_NODES_LABELS```, ```NODES_MONITORING_NODE_LABELS``` and ```NODES_MONITORING_SERVICE_LABELS``` are set the dynamic mode is enabled. + +### instructions + +1. prepare autoscaling + +```bash +# run on EC2 instance +git clone https://github.com/ITISFoundation/osparc-simcore.git +cd osparc-simcore/services/autoscaling +make build-devel # this will build the autoscaling devel image +``` + +2. setup environment variables +```bash +# run on EC2 instance +cd osparc-simcore/services/autoscaling/tests/manual +make .env # generate an initial .env file +nano .env # edit .env and set the variables as needed +``` + +3. start autoscaling stack +```bash +# run on EC2 instance +cd osparc-simcore/services/autoscaling/tests/manual +make up-devel # this will deploy the autoscaling stack +``` + +4. start some docker services to trigger autoscaling +```bash +# run on EC2 instance +docker service create --name=test-service --reserve-cpu=4 --reserve-memory=1GiB --constraint=node.labels.testing.monitored-node==true --label=testing.monitored-service=true redis # will create a redis service reserving 4 CPUs and 1GiB of RAM +``` diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index fb6dca9f814..14f1132c531 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -597,7 +597,7 @@ def _creator(required_resources: DaskTaskResources) -> distributed.Future: @pytest.fixture -def mock_set_node_availability(mocker: MockerFixture) -> mock.Mock: +def mock_docker_set_node_availability(mocker: MockerFixture) -> mock.Mock: async def _fake_set_node_availability( docker_client: AutoscalingDocker, node: Node, *, available: bool ) -> Node: diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py index 7d4f1755ff3..9446bdfb2d2 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py @@ -163,7 +163,7 @@ def _assert_rabbit_autoscaling_message_sent( @pytest.fixture -def mock_tag_node(mocker: MockerFixture) -> mock.Mock: +def mock_docker_tag_node(mocker: MockerFixture) -> mock.Mock: async def fake_tag_node(*args, **kwargs) -> DockerNode: return args[1] @@ -175,7 +175,7 @@ async def fake_tag_node(*args, **kwargs) -> DockerNode: @pytest.fixture -def mock_find_node_with_name( +def mock_docker_find_node_with_name( mocker: MockerFixture, fake_node: DockerNode ) -> Iterator[mock.Mock]: return mocker.patch( @@ -186,7 +186,7 @@ def mock_find_node_with_name( @pytest.fixture -def mock_compute_node_used_resources(mocker: MockerFixture) -> mock.Mock: +def mock_docker_compute_node_used_resources(mocker: MockerFixture) -> mock.Mock: return mocker.patch( "simcore_service_autoscaling.modules.auto_scaling_core.utils_docker.compute_node_used_resources", autospec=True, @@ -288,6 +288,24 @@ def _do( return _do +@pytest.fixture +def mock_dask_get_worker_has_results_in_memory(mocker: MockerFixture) -> mock.Mock: + return mocker.patch( + "simcore_service_autoscaling.modules.dask.get_worker_still_has_results_in_memory", + return_value=0, + autospec=True, + ) + + +@pytest.fixture +def mock_dask_get_worker_used_resources(mocker: MockerFixture) -> mock.Mock: + return mocker.patch( + "simcore_service_autoscaling.modules.dask.get_worker_used_resources", + return_value=Resources.create_as_empty(), + autospec=True, + ) + + @pytest.mark.acceptance_test() @pytest.mark.parametrize( "dask_task_imposed_ec2_type, dask_ram, expected_ec2_type", @@ -324,12 +342,14 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 initialized_app: FastAPI, create_dask_task: Callable[[DaskTaskResources], distributed.Future], ec2_client: EC2Client, - mock_tag_node: mock.Mock, + mock_docker_tag_node: mock.Mock, fake_node: DockerNode, mock_rabbitmq_post_message: mock.Mock, - mock_find_node_with_name: mock.Mock, - mock_set_node_availability: mock.Mock, - mock_compute_node_used_resources: mock.Mock, + mock_docker_find_node_with_name: mock.Mock, + mock_docker_set_node_availability: mock.Mock, + mock_docker_compute_node_used_resources: mock.Mock, + mock_dask_get_worker_has_results_in_memory: mock.Mock, + mock_dask_get_worker_used_resources: mock.Mock, mocker: MockerFixture, dask_spec_local_cluster: distributed.SpecCluster, create_dask_task_resources: Callable[..., DaskTaskResources], @@ -376,10 +396,12 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 ) # as the new node is already running, but is not yet connected, hence not tagged and drained - mock_find_node_with_name.assert_not_called() - mock_tag_node.assert_not_called() - mock_set_node_availability.assert_not_called() - mock_compute_node_used_resources.assert_not_called() + mock_docker_find_node_with_name.assert_not_called() + mock_docker_tag_node.assert_not_called() + mock_docker_set_node_availability.assert_not_called() + mock_docker_compute_node_used_resources.assert_not_called() + mock_dask_get_worker_has_results_in_memory.assert_not_called() + mock_dask_get_worker_used_resources.assert_not_called() # check rabbit messages were sent _assert_rabbit_autoscaling_message_sent( mock_rabbitmq_post_message, @@ -395,7 +417,10 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 await auto_scale_cluster( app=initialized_app, auto_scaling_mode=ComputationalAutoscaling() ) - mock_compute_node_used_resources.assert_not_called() + mock_dask_get_worker_has_results_in_memory.assert_called_once() + mock_dask_get_worker_has_results_in_memory.reset_mock() + mock_dask_get_worker_used_resources.assert_called_once() + mock_dask_get_worker_used_resources.reset_mock() internal_dns_names = await _assert_ec2_instances( ec2_client, num_reservations=1, @@ -407,25 +432,24 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 internal_dns_name = internal_dns_names[0].removesuffix(".ec2.internal") # the node is tagged and made active right away since we still have the pending task - mock_find_node_with_name.assert_called_once() - mock_find_node_with_name.reset_mock() + mock_docker_find_node_with_name.assert_called_once() + mock_docker_find_node_with_name.reset_mock() expected_docker_node_tags = { DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY: expected_ec2_type } - mock_tag_node.assert_called_once_with( + mock_docker_tag_node.assert_called_once_with( get_docker_client(initialized_app), fake_node, tags=expected_docker_node_tags, available=False, ) - mock_tag_node.reset_mock() - mock_set_node_availability.assert_called_once_with( + mock_docker_tag_node.reset_mock() + mock_docker_set_node_availability.assert_called_once_with( get_docker_client(initialized_app), fake_node, available=True ) - mock_set_node_availability.reset_mock() - - # in this case there is no message sent since the worker was not started yet - mock_rabbitmq_post_message.assert_not_called() + mock_docker_set_node_availability.reset_mock() + mock_rabbitmq_post_message.assert_called_once() + mock_rabbitmq_post_message.reset_mock() # now we have 1 monitored node that needs to be mocked auto_scaling_mode = ComputationalAutoscaling() @@ -442,14 +466,22 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 fake_node.Description.Hostname = internal_dns_name # 3. calling this multiple times should do nothing - for _ in range(10): + num_useless_calls = 10 + for _ in range(num_useless_calls): await auto_scale_cluster( app=initialized_app, auto_scaling_mode=auto_scaling_mode ) - mock_compute_node_used_resources.assert_not_called() - mock_find_node_with_name.assert_not_called() - mock_tag_node.assert_not_called() - mock_set_node_availability.assert_not_called() + mock_dask_get_worker_has_results_in_memory.assert_called() + assert ( + mock_dask_get_worker_has_results_in_memory.call_count == 2 * num_useless_calls + ) + mock_dask_get_worker_has_results_in_memory.reset_mock() + mock_dask_get_worker_used_resources.assert_called() + assert mock_dask_get_worker_used_resources.call_count == 2 * num_useless_calls + mock_dask_get_worker_used_resources.reset_mock() + mock_docker_find_node_with_name.assert_not_called() + mock_docker_tag_node.assert_not_called() + mock_docker_set_node_availability.assert_not_called() # check the number of instances did not change and is still running await _assert_ec2_instances( ec2_client, @@ -459,48 +491,28 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 instance_state="running", ) - # check no rabbit messages were sent - # NOTE: we currently have no real dask-worker here - mock_rabbitmq_post_message.assert_not_called() + # check rabbit messages were sent + mock_rabbitmq_post_message.assert_called() + assert mock_rabbitmq_post_message.call_count == num_useless_calls + mock_rabbitmq_post_message.reset_mock() # # 4. now scaling down, as we deleted all the tasks # del dask_future - # the worker will not be found since there is none in this testing environment, but it should not raise - await auto_scale_cluster(app=initialized_app, auto_scaling_mode=auto_scaling_mode) - # check the number of instances did not change and is still running - await _assert_ec2_instances( - ec2_client, - num_reservations=1, - num_instances=1, - instance_type=expected_ec2_type, - instance_state="running", - ) - # now mock the call so that it triggers deactivation - mocked_dask_get_worker_still_has_results_in_memory = mocker.patch( - "simcore_service_autoscaling.modules.dask.get_worker_still_has_results_in_memory", - return_value=0, - autospec=True, - ) - mocked_dask_get_worker_used_resources = mocker.patch( - "simcore_service_autoscaling.modules.dask.get_worker_used_resources", - return_value=Resources.create_as_empty(), - autospec=True, - ) await auto_scale_cluster(app=initialized_app, auto_scaling_mode=auto_scaling_mode) - mocked_dask_get_worker_still_has_results_in_memory.assert_called() - assert mocked_dask_get_worker_still_has_results_in_memory.call_count == 2 - mocked_dask_get_worker_still_has_results_in_memory.reset_mock() - mocked_dask_get_worker_used_resources.assert_called() - assert mocked_dask_get_worker_used_resources.call_count == 2 - mocked_dask_get_worker_used_resources.reset_mock() + mock_dask_get_worker_has_results_in_memory.assert_called() + assert mock_dask_get_worker_has_results_in_memory.call_count == 2 + mock_dask_get_worker_has_results_in_memory.reset_mock() + mock_dask_get_worker_used_resources.assert_called() + assert mock_dask_get_worker_used_resources.call_count == 2 + mock_dask_get_worker_used_resources.reset_mock() # the node shall be set to drain, but not yet terminated - mock_set_node_availability.assert_called_once_with( + mock_docker_set_node_availability.assert_called_once_with( get_docker_client(initialized_app), fake_node, available=False ) - mock_set_node_availability.reset_mock() + mock_docker_set_node_availability.reset_mock() await _assert_ec2_instances( ec2_client, num_reservations=1, @@ -551,6 +563,9 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 instance_state="terminated", ) + # this call should never be used in computational mode + mock_docker_compute_node_used_resources.assert_not_called() + async def test_cluster_does_not_scale_up_if_defined_instance_is_not_allowed( minimal_configuration: None, @@ -658,11 +673,11 @@ async def test_cluster_scaling_up_starts_multiple_instances( initialized_app: FastAPI, create_dask_task: Callable[[DaskTaskResources], distributed.Future], ec2_client: EC2Client, - mock_tag_node: mock.Mock, + mock_docker_tag_node: mock.Mock, scale_up_params: _ScaleUpParams, mock_rabbitmq_post_message: mock.Mock, - mock_find_node_with_name: mock.Mock, - mock_set_node_availability: mock.Mock, + mock_docker_find_node_with_name: mock.Mock, + mock_docker_set_node_availability: mock.Mock, dask_spec_local_cluster: distributed.SpecCluster, ): # we have nothing running now @@ -697,9 +712,9 @@ async def test_cluster_scaling_up_starts_multiple_instances( ) # as the new node is already running, but is not yet connected, hence not tagged and drained - mock_find_node_with_name.assert_not_called() - mock_tag_node.assert_not_called() - mock_set_node_availability.assert_not_called() + mock_docker_find_node_with_name.assert_not_called() + mock_docker_tag_node.assert_not_called() + mock_docker_set_node_availability.assert_not_called() # check rabbit messages were sent _assert_rabbit_autoscaling_message_sent( mock_rabbitmq_post_message, @@ -728,7 +743,7 @@ async def test__deactivate_empty_nodes( cluster: Callable[..., Cluster], host_node: DockerNode, fake_associated_host_instance: AssociatedInstance, - mock_set_node_availability: mock.Mock, + mock_docker_set_node_availability: mock.Mock, ): # since we have no service running, we expect the passed node to be set to drain active_cluster = cluster(active_nodes=[fake_associated_host_instance]) @@ -737,7 +752,7 @@ async def test__deactivate_empty_nodes( ) assert not updated_cluster.active_nodes assert len(updated_cluster.drained_nodes) == len(active_cluster.active_nodes) - mock_set_node_availability.assert_called_once_with( + mock_docker_set_node_availability.assert_called_once_with( mock.ANY, host_node, available=False ) @@ -748,7 +763,7 @@ async def test__deactivate_empty_nodes_with_finished_tasks_should_not_deactivate cluster: Callable[..., Cluster], host_node: DockerNode, fake_associated_host_instance: AssociatedInstance, - mock_set_node_availability: mock.Mock, + mock_docker_set_node_availability: mock.Mock, create_dask_task: Callable[[DaskTaskResources], distributed.Future], ): dask_future = create_dask_task({}) @@ -762,7 +777,7 @@ async def test__deactivate_empty_nodes_with_finished_tasks_should_not_deactivate initialized_app, deepcopy(active_cluster), ComputationalAutoscaling() ) assert updated_cluster.active_nodes - mock_set_node_availability.assert_not_called() + mock_docker_set_node_availability.assert_not_called() # now removing the dask_future shall remove the result from the memory del dask_future @@ -772,6 +787,6 @@ async def test__deactivate_empty_nodes_with_finished_tasks_should_not_deactivate ) assert not updated_cluster.active_nodes assert len(updated_cluster.drained_nodes) == len(active_cluster.active_nodes) - mock_set_node_availability.assert_called_once_with( + mock_docker_set_node_availability.assert_called_once_with( mock.ANY, host_node, available=False ) diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py index 3f4caf35d1d..c81c4968bb7 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py @@ -446,7 +446,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 fake_node: Node, mock_rabbitmq_post_message: mock.Mock, mock_find_node_with_name: mock.Mock, - mock_set_node_availability: mock.Mock, + mock_docker_set_node_availability: mock.Mock, mock_compute_node_used_resources: mock.Mock, mocker: MockerFixture, docker_service_imposed_ec2_type: InstanceTypeType | None, @@ -487,7 +487,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 # as the new node is already running, but is not yet connected, hence not tagged and drained mock_find_node_with_name.assert_not_called() mock_tag_node.assert_not_called() - mock_set_node_availability.assert_not_called() + mock_docker_set_node_availability.assert_not_called() mock_compute_node_used_resources.assert_not_called() # check rabbit messages were sent _assert_rabbit_autoscaling_message_sent( @@ -537,10 +537,10 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 available=False, ) mock_tag_node.reset_mock() - mock_set_node_availability.assert_called_once_with( + mock_docker_set_node_availability.assert_called_once_with( get_docker_client(initialized_app), fake_node, available=True ) - mock_set_node_availability.reset_mock() + mock_docker_set_node_availability.reset_mock() # check rabbit messages were sent, we do have worker assert fake_node.Description @@ -586,7 +586,7 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 mock_compute_node_used_resources.assert_called() mock_find_node_with_name.assert_not_called() mock_tag_node.assert_not_called() - mock_set_node_availability.assert_not_called() + mock_docker_set_node_availability.assert_not_called() # check the number of instances did not change and is still running await _assert_ec2_instances( ec2_client, @@ -615,17 +615,17 @@ async def test_cluster_scaling_up_and_down( # noqa: PLR0915 instance_type=expected_ec2_type, instance_state="running", ) - mock_set_node_availability.assert_called_once_with( + mock_docker_set_node_availability.assert_called_once_with( get_docker_client(initialized_app), fake_node, available=False ) - mock_set_node_availability.reset_mock() + mock_docker_set_node_availability.reset_mock() # calling again does the exact same await auto_scale_cluster(app=initialized_app, auto_scaling_mode=auto_scaling_mode) - mock_set_node_availability.assert_called_once_with( + mock_docker_set_node_availability.assert_called_once_with( get_docker_client(initialized_app), fake_node, available=False ) - mock_set_node_availability.reset_mock() + mock_docker_set_node_availability.reset_mock() await _assert_ec2_instances( ec2_client, num_reservations=1, @@ -730,7 +730,7 @@ async def test_cluster_scaling_up_starts_multiple_instances( scale_up_params: _ScaleUpParams, mock_rabbitmq_post_message: mock.Mock, mock_find_node_with_name: mock.Mock, - mock_set_node_availability: mock.Mock, + mock_docker_set_node_availability: mock.Mock, ): # we have nothing running now all_instances = await ec2_client.describe_instances() @@ -774,7 +774,7 @@ async def test_cluster_scaling_up_starts_multiple_instances( # as the new node is already running, but is not yet connected, hence not tagged and drained mock_find_node_with_name.assert_not_called() mock_tag_node.assert_not_called() - mock_set_node_availability.assert_not_called() + mock_docker_set_node_availability.assert_not_called() # check rabbit messages were sent _assert_rabbit_autoscaling_message_sent( mock_rabbitmq_post_message, @@ -791,7 +791,7 @@ async def test__deactivate_empty_nodes( cluster: Callable[..., Cluster], host_node: Node, fake_ec2_instance_data: Callable[..., EC2InstanceData], - mock_set_node_availability: mock.Mock, + mock_docker_set_node_availability: mock.Mock, ): # since we have no service running, we expect the passed node to be set to drain active_cluster = cluster( @@ -802,7 +802,7 @@ async def test__deactivate_empty_nodes( ) assert not updated_cluster.active_nodes assert len(updated_cluster.drained_nodes) == len(active_cluster.active_nodes) - mock_set_node_availability.assert_called_once_with( + mock_docker_set_node_availability.assert_called_once_with( mock.ANY, host_node, available=False ) @@ -813,7 +813,7 @@ async def test__deactivate_empty_nodes_to_drain_when_services_running_are_missin cluster: Callable[..., Cluster], host_node: Node, fake_ec2_instance_data: Callable[..., EC2InstanceData], - mock_set_node_availability: mock.Mock, + mock_docker_set_node_availability: mock.Mock, create_service: Callable[ [dict[str, Any], dict[DockerLabelKey, str], str], Awaitable[Service] ], @@ -838,7 +838,7 @@ async def test__deactivate_empty_nodes_to_drain_when_services_running_are_missin ) assert not updated_cluster.active_nodes assert len(updated_cluster.drained_nodes) == len(active_cluster.active_nodes) - mock_set_node_availability.assert_called_once_with( + mock_docker_set_node_availability.assert_called_once_with( mock.ANY, host_node, available=False ) @@ -850,7 +850,7 @@ async def test__deactivate_empty_nodes_does_not_drain_if_service_is_running_with cluster: Callable[..., Cluster], host_node: Node, fake_ec2_instance_data: Callable[..., EC2InstanceData], - mock_set_node_availability: mock.Mock, + mock_docker_set_node_availability: mock.Mock, create_service: Callable[ [dict[str, Any], dict[DockerLabelKey, str], str], Awaitable[Service] ], @@ -878,7 +878,7 @@ async def test__deactivate_empty_nodes_does_not_drain_if_service_is_running_with initialized_app, active_cluster, DynamicAutoscaling() ) assert updated_cluster == active_cluster - mock_set_node_availability.assert_not_called() + mock_docker_set_node_availability.assert_not_called() async def test__find_terminateable_nodes_with_no_hosts( diff --git a/services/autoscaling/tests/unit/test_utils_computational_scaling.py b/services/autoscaling/tests/unit/test_utils_computational_scaling.py index 945b883187e..fa06a5a18b8 100644 --- a/services/autoscaling/tests/unit/test_utils_computational_scaling.py +++ b/services/autoscaling/tests/unit/test_utils_computational_scaling.py @@ -14,6 +14,8 @@ from pydantic import ByteSize, parse_obj_as from pytest_mock import MockerFixture from simcore_service_autoscaling.models import ( + AssignedTasksToInstance, + AssignedTasksToInstanceType, AssociatedInstance, DaskTask, DaskTaskResources, @@ -133,7 +135,7 @@ async def test_try_assigning_task_to_node( assert instance_to_tasks[0][1] == [task, task] -async def test_try_assigning_task_to_pending_instances_with_no_instances( +async def test_try_assigning_task_to_instances_with_no_instances( fake_app: mock.Mock, fake_task: Callable[..., DaskTask], ): @@ -144,15 +146,19 @@ async def test_try_assigning_task_to_pending_instances_with_no_instances( ) -async def test_try_assigning_task_to_pending_instances( +async def test_try_assigning_task_to_instances( fake_app: mock.Mock, fake_task: Callable[..., DaskTask], fake_ec2_instance_data: Callable[..., EC2InstanceData], ): task = fake_task(required_resources={"CPU": 2}) ec2_instance = fake_ec2_instance_data() - pending_instance_to_tasks: list[tuple[EC2InstanceData, list[DaskTask]]] = [ - (ec2_instance, []) + pending_instance_to_tasks: list[AssignedTasksToInstance] = [ + AssignedTasksToInstance( + instance=ec2_instance, + assigned_tasks=[], + available_resources=Resources(cpus=4, ram=ByteSize(1024**2)), + ) ] # calling once should allow to add that task to the instance @@ -165,7 +171,7 @@ async def test_try_assigning_task_to_pending_instances( ) is True ) - assert pending_instance_to_tasks[0][1] == [task] + assert pending_instance_to_tasks[0].assigned_tasks == [task] # calling a second time as well should allow to add that task to the instance assert ( await try_assigning_task_to_instances( @@ -176,7 +182,7 @@ async def test_try_assigning_task_to_pending_instances( ) is True ) - assert pending_instance_to_tasks[0][1] == [task, task] + assert pending_instance_to_tasks[0].assigned_tasks == [task, task] # calling a third time should fail assert ( await try_assigning_task_to_instances( @@ -187,7 +193,7 @@ async def test_try_assigning_task_to_pending_instances( ) is False ) - assert pending_instance_to_tasks[0][1] == [task, task] + assert pending_instance_to_tasks[0].assigned_tasks == [task, task] def test_try_assigning_task_to_instance_types_with_empty_types( @@ -205,16 +211,16 @@ def test_try_assigning_task_to_instance_types( fake_instance_type = EC2InstanceType( name=faker.name(), cpus=6, ram=parse_obj_as(ByteSize, "2GiB") ) - instance_type_to_tasks: list[tuple[EC2InstanceType, list[DaskTask]]] = [ - (fake_instance_type, []) + instance_type_to_tasks: list[AssignedTasksToInstanceType] = [ + AssignedTasksToInstanceType(instance_type=fake_instance_type, assigned_tasks=[]) ] # now this should work 3 times assert try_assigning_task_to_instance_types(task, instance_type_to_tasks) is True - assert instance_type_to_tasks[0][1] == [task] + assert instance_type_to_tasks[0].assigned_tasks == [task] assert try_assigning_task_to_instance_types(task, instance_type_to_tasks) is True - assert instance_type_to_tasks[0][1] == [task, task] + assert instance_type_to_tasks[0].assigned_tasks == [task, task] assert try_assigning_task_to_instance_types(task, instance_type_to_tasks) is True - assert instance_type_to_tasks[0][1] == [task, task, task] + assert instance_type_to_tasks[0].assigned_tasks == [task, task, task] # now it should fail assert try_assigning_task_to_instance_types(task, instance_type_to_tasks) is False - assert instance_type_to_tasks[0][1] == [task, task, task] + assert instance_type_to_tasks[0].assigned_tasks == [task, task, task] diff --git a/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py b/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py index d27bc0dd28f..ab3b3c55e3f 100644 --- a/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py +++ b/services/autoscaling/tests/unit/test_utils_dynamic_scaling.py @@ -8,13 +8,14 @@ from datetime import timedelta import pytest -from aws_library.ec2.models import EC2InstanceData, EC2InstanceType +from aws_library.ec2.models import EC2InstanceData, Resources from faker import Faker from models_library.generated_models.docker_rest_api import Task from pydantic import ByteSize from pytest_mock import MockerFixture +from simcore_service_autoscaling.models import AssignedTasksToInstance from simcore_service_autoscaling.utils.dynamic_scaling import ( - try_assigning_task_to_pending_instances, + try_assigning_task_to_instances, ) @@ -28,22 +29,21 @@ def _creator(**overrides) -> Task: return _creator -async def test_try_assigning_task_to_pending_instances_with_no_instances( +async def test_try_assigning_task_to_instances_with_no_instances( mocker: MockerFixture, fake_task: Callable[..., Task], - fake_ec2_instance_data: Callable[..., EC2InstanceData], ): fake_app = mocker.Mock() pending_task = fake_task() assert ( - await try_assigning_task_to_pending_instances( - fake_app, pending_task, [], {}, notify_progress=True + await try_assigning_task_to_instances( + fake_app, pending_task, [], notify_progress=True ) is False ) -async def test_try_assigning_task_to_pending_instances( +async def test_try_assigning_task_to_instances( mocker: MockerFixture, fake_task: Callable[..., Task], fake_ec2_instance_data: Callable[..., EC2InstanceData], @@ -56,43 +56,40 @@ async def test_try_assigning_task_to_pending_instances( Spec={"Resources": {"Reservations": {"NanoCPUs": 2 * 1e9}}} ) fake_instance = fake_ec2_instance_data() - pending_instance_to_tasks: list[tuple[EC2InstanceData, list[Task]]] = [ - (fake_instance, []) - ] - type_to_instance_map = { - fake_instance.type: EC2InstanceType( - name=fake_instance.type, cpus=4, ram=ByteSize(1024 * 1024) + pending_instance_to_tasks: list[AssignedTasksToInstance] = [ + AssignedTasksToInstance( + instance=fake_instance, + assigned_tasks=[], + available_resources=Resources(cpus=4, ram=ByteSize(1024**3)), ) - } + ] + # calling once should allow to add that task to the instance assert ( - await try_assigning_task_to_pending_instances( + await try_assigning_task_to_instances( fake_app, pending_task, pending_instance_to_tasks, - type_to_instance_map, notify_progress=True, ) is True ) # calling a second time as well should allow to add that task to the instance assert ( - await try_assigning_task_to_pending_instances( + await try_assigning_task_to_instances( fake_app, pending_task, pending_instance_to_tasks, - type_to_instance_map, notify_progress=True, ) is True ) # calling a third time should fail assert ( - await try_assigning_task_to_pending_instances( + await try_assigning_task_to_instances( fake_app, pending_task, pending_instance_to_tasks, - type_to_instance_map, notify_progress=True, ) is False From a4d9fecdbc53dc354a5c44fc4a2cb1d82ceab3a9 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:05:48 +0100 Subject: [PATCH 08/24] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Clusters-keeper:=20r?= =?UTF-8?q?enamed=20ENV=20variables=20(#5032)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data/docker-compose.yml | 8 ++++---- .../src/simcore_service_clusters_keeper/utils/clusters.py | 8 ++++---- services/clusters-keeper/tests/manual/README.md | 8 ++++---- services/docker-compose.yml | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml b/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml index 87f4a6cf641..213aac41821 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml @@ -47,8 +47,8 @@ services: hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}" environment: DASK_MONITORING_URL: tcp://dask-scheduler:8786 - EC2_ACCESS_KEY_ID: ${EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID} - EC2_ENDPOINT: ${EC2_CLUSTERS_KEEPER_ENDPOINT} + EC2_ACCESS_KEY_ID: ${CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID} + EC2_ENDPOINT: ${CLUSTERS_KEEPER_EC2_ENDPOINT} EC2_INSTANCES_ALLOWED_TYPES: ${WORKERS_EC2_INSTANCES_ALLOWED_TYPES} EC2_INSTANCES_AMI_ID: ${WORKERS_EC2_INSTANCES_AMI_ID} EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS: ${WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS} @@ -59,8 +59,8 @@ services: EC2_INSTANCES_SECURITY_GROUP_IDS: ${WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS} EC2_INSTANCES_SUBNET_ID: ${WORKERS_EC2_INSTANCES_SUBNET_ID} EC2_INSTANCES_TIME_BEFORE_TERMINATION: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION} - EC2_REGION_NAME: ${EC2_CLUSTERS_KEEPER_REGION_NAME} - EC2_SECRET_ACCESS_KEY: ${EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY} + EC2_REGION_NAME: ${CLUSTERS_KEEPER_EC2_REGION_NAME} + EC2_SECRET_ACCESS_KEY: ${CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY} LOG_FORMAT_LOCAL_DEV_ENABLED: 1 LOG_LEVEL: ${LOG_LEVEL:-WARNING} REDIS_HOST: redis diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py index 64cdf4fcb10..805d793a919 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py @@ -40,8 +40,8 @@ def _convert_to_env_list(entries: list[Any]) -> str: environment_variables = [ f"DOCKER_IMAGE_TAG={app_settings.CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG}", - f"EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ACCESS_KEY_ID}", - f"EC2_CLUSTERS_KEEPER_ENDPOINT={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ENDPOINT}", + f"CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ACCESS_KEY_ID}", + f"CLUSTERS_KEEPER_EC2_ENDPOINT={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ENDPOINT}", f"WORKERS_EC2_INSTANCES_ALLOWED_TYPES={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_ALLOWED_TYPES)}", f"WORKERS_EC2_INSTANCES_AMI_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_AMI_ID}", f"WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS)}", @@ -52,8 +52,8 @@ def _convert_to_env_list(entries: list[Any]) -> str: f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}", f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}", f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}", - f"EC2_CLUSTERS_KEEPER_REGION_NAME={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_REGION_NAME}", - f"EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_SECRET_ACCESS_KEY}", + f"CLUSTERS_KEEPER_EC2_REGION_NAME={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_REGION_NAME}", + f"CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_SECRET_ACCESS_KEY}", f"LOG_LEVEL={app_settings.LOG_LEVEL}", ] diff --git a/services/clusters-keeper/tests/manual/README.md b/services/clusters-keeper/tests/manual/README.md index 4d508b28226..a2ecd57c8a0 100644 --- a/services/clusters-keeper/tests/manual/README.md +++ b/services/clusters-keeper/tests/manual/README.md @@ -60,10 +60,10 @@ S3_SECURE=1 4. prepare clusters-keeper: ```bash CLUSTERS_KEEPER_EC2_ACCESS={} -EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID=XXXXXXX -EC2_CLUSTERS_KEEPER_ENDPOINT=https://ec2.amazonaws.com -EC2_CLUSTERS_KEEPER_REGION_NAME=us-east-1 -EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY=XXXXXXX +CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID=XXXXXXX +CLUSTERS_KEEPER_EC2_ENDPOINT=https://ec2.amazonaws.com +CLUSTERS_KEEPER_EC2_REGION_NAME=us-east-1 +CLUSTERS_KEEPER_SECRET_EC2_ACCESS_KEY=XXXXXXX CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES={} PRIMARY_EC2_INSTANCES_ALLOWED_TYPES="[\"t2.micro\"]" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 0d37c5d002d..e8f63bbaaf7 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -86,10 +86,10 @@ services: - CLUSTERS_KEEPER_TASK_INTERVAL=${CLUSTERS_KEEPER_TASK_INTERVAL} - CLUSTERS_KEEPER_LOGLEVEL=${LOG_LEVEL:-WARNING} - CLUSTERS_KEEPER_EC2_ACCESS=${CLUSTERS_KEEPER_EC2_ACCESS} - - EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID=${EC2_CLUSTERS_KEEPER_ACCESS_KEY_ID} - - EC2_CLUSTERS_KEEPER_ENDPOINT=${EC2_CLUSTERS_KEEPER_ENDPOINT} - - EC2_CLUSTERS_KEEPER_REGION_NAME=${EC2_CLUSTERS_KEEPER_REGION_NAME} - - EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY=${EC2_CLUSTERS_KEEPER_SECRET_ACCESS_KEY} + - CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID=${CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID} + - CLUSTERS_KEEPER_EC2_ENDPOINT=${CLUSTERS_KEEPER_EC2_ENDPOINT} + - CLUSTERS_KEEPER_EC2_REGION_NAME=${CLUSTERS_KEEPER_EC2_REGION_NAME} + - CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY=${CLUSTERS_KEEPER_} - LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED} - CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=${CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES} - PRIMARY_EC2_INSTANCES_ALLOWED_TYPES=${PRIMARY_EC2_INSTANCES_ALLOWED_TYPES} From cd84f2fbedc618776ffc73e39e30c1cbaeeb7fde Mon Sep 17 00:00:00 2001 From: Julian Querido Date: Wed, 15 Nov 2023 10:27:49 +0100 Subject: [PATCH 09/24] =?UTF-8?q?=E2=9A=97=EF=B8=8F=20Perform=20experiment?= =?UTF-8?q?s=20-=20MarkerIO=20(#5030)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/static-webserver/client/source/boot/index.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/static-webserver/client/source/boot/index.html b/services/static-webserver/client/source/boot/index.html index 014f74b544d..0452a911a4b 100644 --- a/services/static-webserver/client/source/boot/index.html +++ b/services/static-webserver/client/source/boot/index.html @@ -56,4 +56,12 @@ ${preBootJs} + From 633ec552d67c15fa8ffae24eaf16fe7b1c654f73 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Wed, 15 Nov 2023 12:27:49 +0000 Subject: [PATCH 10/24] =?UTF-8?q?=E2=9C=A8oSPARC=20API=20keys=20are=20crea?= =?UTF-8?q?ted=20and=20removed=20if=20a=20service=20requires=20them=20(#50?= =?UTF-8?q?04)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- api/specs/web-server/_auth.py | 2 +- .../api_schemas_api_server/__init__.py | 0 .../api_schemas_api_server}/api_keys.py | 5 +- services/api-server/openapi.json | 8 + .../tests/unit/_with_db/conftest.py | 6 +- .../tests/unit/_with_db/test_product.py | 6 +- .../core/application.py | 2 + .../modules/api_keys_manager.py | 106 +++++++++++ .../dynamic_sidecar/docker_compose_specs.py | 13 +- .../scheduler/_core/_events_utils.py | 62 ++++--- .../modules/osparc_variables_substitutions.py | 88 ++++++++-- .../utils/distributed_identifer.py | 69 ++++++++ .../utils/osparc_variables.py | 20 ++- .../unit/test_modules_api_keys_manager.py | 100 +++++++++++ .../unit/test_utils_distributed_identifier.py | 137 +++++++++++++++ .../tests/unit/test_utils_osparc_variables.py | 4 +- .../api/v0/openapi.yaml | 7 - .../api_keys/_api.py | 12 +- .../simcore_service_webserver/api_keys/_db.py | 24 ++- .../api_keys/_rpc.py | 15 +- .../tests/unit/with_dbs/01/test_api_keys.py | 9 +- .../unit/with_dbs/01/test_api_keys_rpc.py | 164 ++++++++++++++++++ 22 files changed, 786 insertions(+), 73 deletions(-) create mode 100644 packages/models-library/src/models_library/api_schemas_api_server/__init__.py rename {services/api-server/src/simcore_service_api_server/models/domain => packages/models-library/src/models_library/api_schemas_api_server}/api_keys.py (79%) create mode 100644 services/director-v2/src/simcore_service_director_v2/modules/api_keys_manager.py create mode 100644 services/director-v2/src/simcore_service_director_v2/utils/distributed_identifer.py create mode 100644 services/director-v2/tests/unit/test_modules_api_keys_manager.py create mode 100644 services/director-v2/tests/unit/test_utils_distributed_identifier.py create mode 100644 services/web/server/tests/unit/with_dbs/01/test_api_keys_rpc.py diff --git a/api/specs/web-server/_auth.py b/api/specs/web-server/_auth.py index 6d2b714f393..d922ce41228 100644 --- a/api/specs/web-server/_auth.py +++ b/api/specs/web-server/_auth.py @@ -266,7 +266,7 @@ async def email_confirmation(code: str): }, }, ) -async def list_api_keys(code: str): +async def list_api_keys(): """lists display names of API keys by this user""" diff --git a/packages/models-library/src/models_library/api_schemas_api_server/__init__.py b/packages/models-library/src/models_library/api_schemas_api_server/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/api-server/src/simcore_service_api_server/models/domain/api_keys.py b/packages/models-library/src/models_library/api_schemas_api_server/api_keys.py similarity index 79% rename from services/api-server/src/simcore_service_api_server/models/domain/api_keys.py rename to packages/models-library/src/models_library/api_schemas_api_server/api_keys.py index 467b885cb82..d828fc6507d 100644 --- a/services/api-server/src/simcore_service_api_server/models/domain/api_keys.py +++ b/packages/models-library/src/models_library/api_schemas_api_server/api_keys.py @@ -6,7 +6,10 @@ class ApiKey(BaseModel): api_secret: SecretStr -class ApiKeyInDB(ApiKey): +class ApiKeyInDB(BaseModel): + api_key: str + api_secret: str + id_: int = Field(0, alias="id") display_name: str user_id: int diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 369e0a8bf63..b9ebbb549c3 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -1580,6 +1580,7 @@ "Profile": { "title": "Profile", "required": [ + "id", "login", "role" ], @@ -1595,6 +1596,12 @@ "type": "string", "example": "Maxwell" }, + "id": { + "title": "Id", + "exclusiveMinimum": true, + "type": "integer", + "minimum": 0 + }, "login": { "title": "Login", "type": "string", @@ -1614,6 +1621,7 @@ } }, "example": { + "id": "20", "first_name": "James", "last_name": "Maxwell", "login": "james-maxwell@itis.swiss", diff --git a/services/api-server/tests/unit/_with_db/conftest.py b/services/api-server/tests/unit/_with_db/conftest.py index ee13aabcc72..f9a8815bff6 100644 --- a/services/api-server/tests/unit/_with_db/conftest.py +++ b/services/api-server/tests/unit/_with_db/conftest.py @@ -22,6 +22,7 @@ import yaml from aiopg.sa.connection import SAConnection from fastapi import FastAPI +from models_library.api_schemas_api_server.api_keys import ApiKeyInDB from pydantic import PositiveInt from pytest_simcore.helpers.rawdata_fakers import ( random_api_key, @@ -36,7 +37,6 @@ from simcore_postgres_database.models.users import users from simcore_service_api_server.core.application import init_app from simcore_service_api_server.core.settings import PostgresSettings -from simcore_service_api_server.models.domain.api_keys import ApiKeyInDB ## POSTGRES ----- @@ -269,5 +269,5 @@ async def auth( ) -> httpx.BasicAuth: """overrides auth and uses access to real repositories instead of mocks""" async for key in create_fake_api_keys(1): - return httpx.BasicAuth(key.api_key, key.api_secret.get_secret_value()) - assert False, "Did not generate authentication" + return httpx.BasicAuth(key.api_key, key.api_secret) + pytest.fail("Did not generate authentication") diff --git a/services/api-server/tests/unit/_with_db/test_product.py b/services/api-server/tests/unit/_with_db/test_product.py index a5b721be3f7..af0fb995ffd 100644 --- a/services/api-server/tests/unit/_with_db/test_product.py +++ b/services/api-server/tests/unit/_with_db/test_product.py @@ -13,13 +13,13 @@ from faker import Faker from fastapi import status from fastapi.encoders import jsonable_encoder +from models_library.api_schemas_api_server.api_keys import ApiKeyInDB from models_library.api_schemas_webserver.wallets import WalletGetWithAvailableCredits from models_library.generics import Envelope from models_library.users import UserID from models_library.wallets import WalletStatus from pydantic import PositiveInt from simcore_service_api_server._meta import API_VTAG -from simcore_service_api_server.models.domain.api_keys import ApiKeyInDB async def test_product_webserver( @@ -70,7 +70,7 @@ def _check_key_product_compatibility(request: httpx.Request, **kwargs): key = keys[wallet_id] response = await client.get( f"{API_VTAG}/wallets/{wallet_id}", - auth=httpx.BasicAuth(key.api_key, key.api_secret.get_secret_value()), + auth=httpx.BasicAuth(key.api_key, key.api_secret), ) assert response.status_code == status.HTTP_200_OK assert wallet_get_mock.call_count == len(keys) @@ -104,7 +104,7 @@ def _get_service_side_effect(request: httpx.Request, **kwargs): for key in keys: response = await client.get( f"{API_VTAG}/solvers/simcore/services/comp/isolve/releases/2.0.24", - auth=httpx.BasicAuth(key.api_key, key.api_secret.get_secret_value()), + auth=httpx.BasicAuth(key.api_key, key.api_secret), ) assert respx_mock.call_count == len(keys) diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 9ad93646615..55a02280053 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -17,6 +17,7 @@ from ..api.errors.validation_error import http422_error_handler from ..meta import API_VERSION, API_VTAG, PROJECT_NAME, SUMMARY from ..modules import ( + api_keys_manager, catalog, comp_scheduler, dask_clients_pool, @@ -166,6 +167,7 @@ def init_app(settings: AppSettings | None = None) -> FastAPI: if dynamic_scheduler_enabled: dynamic_sidecar.setup(app) + api_keys_manager.setup(app) if ( settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND.COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED diff --git a/services/director-v2/src/simcore_service_director_v2/modules/api_keys_manager.py b/services/director-v2/src/simcore_service_director_v2/modules/api_keys_manager.py new file mode 100644 index 00000000000..0fa6f457fe8 --- /dev/null +++ b/services/director-v2/src/simcore_service_director_v2/modules/api_keys_manager.py @@ -0,0 +1,106 @@ +from uuid import uuid5 + +from fastapi import FastAPI +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.api_schemas_webserver.auth import ApiKeyCreate, ApiKeyGet +from models_library.products import ProductName +from models_library.projects_nodes_io import NodeID +from models_library.rabbitmq_basic_types import RPCMethodName +from models_library.users import UserID +from pydantic import parse_obj_as +from servicelib.rabbitmq import RabbitMQRPCClient + +from ..utils.distributed_identifer import BaseDistributedIdentifierManager +from .rabbitmq import get_rabbitmq_rpc_client + + +class APIKeysManager(BaseDistributedIdentifierManager[str, ApiKeyGet]): + def __init__(self, app: FastAPI) -> None: + self.GET_OR_CREATE_INJECTS_IDENTIFIER = True + self.app = app + + @property + def rpc_client(self) -> RabbitMQRPCClient: + return get_rabbitmq_rpc_client(self.app) + + # pylint:disable=arguments-differ + + async def get( # type:ignore [override] + self, identifier: str, product_name: ProductName, user_id: UserID + ) -> ApiKeyGet | None: + result = await self.rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "api_key_get"), + product_name=product_name, + user_id=user_id, + name=identifier, + ) + return parse_obj_as(ApiKeyGet | None, result) + + async def create( # type:ignore [override] + self, identifier: str, product_name: ProductName, user_id: UserID + ) -> tuple[str, ApiKeyGet]: + result = await self.rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "create_api_keys"), + product_name=product_name, + user_id=user_id, + new=ApiKeyCreate(display_name=identifier, expiration=None), + ) + return identifier, ApiKeyGet.parse_obj(result) + + async def destroy( # type:ignore [override] + self, identifier: str, product_name: ProductName, user_id: UserID + ) -> None: + await self.rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "delete_api_keys"), + product_name=product_name, + user_id=user_id, + name=identifier, + ) + + +async def get_or_create_api_key( + app: FastAPI, *, product_name: ProductName, user_id: UserID, node_id: NodeID +) -> ApiKeyGet: + api_keys_manager = _get_api_keys_manager(app) + display_name = _get_api_key_name(node_id) + + key_data: ApiKeyGet | None = await api_keys_manager.get( + identifier=display_name, product_name=product_name, user_id=user_id + ) + if key_data is None: + _, key_data = await api_keys_manager.create( + identifier=display_name, product_name=product_name, user_id=user_id + ) + + return key_data + + +async def safe_remove( + app: FastAPI, *, node_id: NodeID, product_name: ProductName, user_id: UserID +) -> None: + api_keys_manager = _get_api_keys_manager(app) + display_name = _get_api_key_name(node_id) + + await api_keys_manager.remove( + identifier=display_name, product_name=product_name, user_id=user_id + ) + + +def _get_api_key_name(node_id: NodeID) -> str: + obfuscated_node_id = uuid5(node_id, f"{node_id}") + return f"_auto_{obfuscated_node_id}" + + +def _get_api_keys_manager(app: FastAPI) -> APIKeysManager: + api_keys_manager: APIKeysManager = app.state.api_keys_manager + return api_keys_manager + + +def setup(app: FastAPI) -> None: + async def on_startup() -> None: + app.state.api_keys_manager = APIKeysManager(app) + + app.add_event_handler("startup", on_startup) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py index 54e837b8f5d..e67c3c45f64 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py @@ -28,6 +28,7 @@ from ...core.dynamic_services_settings.egress_proxy import EgressProxySettings from ...modules.osparc_variables_substitutions import ( + resolve_and_substitute_service_lifetime_variables_in_specs, resolve_and_substitute_session_variables_in_model, resolve_and_substitute_session_variables_in_specs, substitute_vendor_secrets_in_model, @@ -376,11 +377,19 @@ async def assemble_spec( # pylint: disable=too-many-arguments # noqa: PLR0913 app=app, specs=service_spec, user_id=user_id, + safe=True, + product_name=product_name, + project_id=project_id, + node_id=node_id, + ) + service_spec = await resolve_and_substitute_service_lifetime_variables_in_specs( + app=app, + specs=service_spec, # NOTE: at this point all OsparcIdentifiers have to be replaced # an error will be raised otherwise - safe=False, + safe=True, product_name=product_name, - project_id=project_id, + user_id=user_id, node_id=node_id, ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index d46811a3aab..e5c83523f6c 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -2,8 +2,7 @@ import json import logging -from collections import deque -from typing import Any, Deque, Final +from typing import Any, Final from fastapi import FastAPI from models_library.projects_networks import ProjectsNetworks @@ -39,6 +38,7 @@ SchedulerData, ) from .....utils.db import get_repository +from ....api_keys_manager import safe_remove from ....db.repositories.projects import ProjectsRepository from ....db.repositories.projects_networks import ProjectsNetworksRepository from ....director_v0 import DirectorV0Client @@ -64,7 +64,7 @@ from ...errors import EntrypointContainerNotFoundError from ...volumes import DY_SIDECAR_SHARED_STORE_PATH, DynamicSidecarVolumesPathsResolver -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) # Used to ensure no more that X services per node pull or push data @@ -75,22 +75,19 @@ def get_director_v0_client(app: FastAPI) -> DirectorV0Client: - client = DirectorV0Client.instance(app) - return client + return DirectorV0Client.instance(app) def parse_containers_inspect( containers_inspect: dict[str, Any] | None ) -> list[DockerContainerInspect]: - results: Deque[DockerContainerInspect] = deque() - if containers_inspect is None: return [] - for container_id in containers_inspect: - container_inspect_data = containers_inspect[container_id] - results.append(DockerContainerInspect.from_container(container_inspect_data)) - return list(results) + return [ + DockerContainerInspect.from_container(containers_inspect[container_id]) + for container_id in containers_inspect + ] def are_all_user_services_containers_running( @@ -107,7 +104,9 @@ def _get_scheduler_data(app: FastAPI, node_uuid: NodeID) -> SchedulerData: ) # pylint: disable=protected-access scheduler_data: SchedulerData = ( - dynamic_sidecars_scheduler._scheduler.get_scheduler_data(node_uuid) + dynamic_sidecars_scheduler._scheduler.get_scheduler_data( # noqa: SLF001 + node_uuid + ) ) return scheduler_data @@ -125,7 +124,7 @@ async def service_remove_containers( scheduler_data.endpoint, progress_callback=progress_callback ) except (BaseClientHTTPError, TaskClientResultError) as e: - logger.warning( + _logger.warning( ( "Could not remove service containers for " "%s\n%s. Will continue to save the data from the service!" @@ -193,7 +192,7 @@ async def service_remove_sidecar_proxy_docker_networks_and_volumes( if scheduler_data.dynamic_sidecar.were_state_and_outputs_saved: if scheduler_data.dynamic_sidecar.docker_node_id is None: - logger.warning( + _logger.warning( "Skipped volume removal for %s, since a docker_node_id was not found.", scheduler_data.node_uuid, ) @@ -210,11 +209,11 @@ async def service_remove_sidecar_proxy_docker_networks_and_volumes( DY_SIDECAR_SHARED_STORE_PATH, scheduler_data.paths_mapping.inputs_path, scheduler_data.paths_mapping.outputs_path, + *scheduler_data.paths_mapping.state_paths, ] - + scheduler_data.paths_mapping.state_paths ] with log_context( - logger, logging.DEBUG, f"removing volumes via service for {node_uuid}" + _logger, logging.DEBUG, f"removing volumes via service for {node_uuid}" ): await remove_volumes_from_node( swarm_stack_name=swarm_stack_name, @@ -225,7 +224,7 @@ async def service_remove_sidecar_proxy_docker_networks_and_volumes( node_uuid=scheduler_data.node_uuid, ) - logger.debug( + _logger.debug( "Removed dynamic-sidecar services and crated container for '%s'", scheduler_data.service_name, ) @@ -245,7 +244,7 @@ async def service_remove_sidecar_proxy_docker_networks_and_volumes( # pylint: disable=protected-access scheduler_data.dynamic_sidecar.service_removal_state.mark_removed() await ( - app.state.dynamic_sidecar_scheduler._scheduler.remove_service_from_observation( + app.state.dynamic_sidecar_scheduler._scheduler.remove_service_from_observation( # noqa: SLF001 scheduler_data.node_uuid ) ) @@ -287,7 +286,7 @@ async def _remove_containers_save_state_and_outputs() -> None: ) if can_really_save and scheduler_data.dynamic_sidecar.were_containers_created: - logger.info("Calling into dynamic-sidecar to save: state and output ports") + _logger.info("Calling into dynamic-sidecar to save: state and output ports") try: tasks = [ service_push_outputs(app, scheduler_data.node_uuid, sidecars_client) @@ -305,9 +304,9 @@ async def _remove_containers_save_state_and_outputs() -> None: await logged_gather(*tasks, max_concurrency=2) scheduler_data.dynamic_sidecar.were_state_and_outputs_saved = True - logger.info("dynamic-sidecar saved: state and output ports") + _logger.info("dynamic-sidecar saved: state and output ports") except (BaseClientHTTPError, TaskClientResultError) as e: - logger.error( + _logger.error( # noqa: TRY400 ( "Could not contact dynamic-sidecar to save service " "state or output ports %s\n%s" @@ -322,7 +321,7 @@ async def _remove_containers_save_state_and_outputs() -> None: scheduler_data.dynamic_sidecar.wait_for_manual_intervention_after_error = ( True ) - raise e + raise if node_resource_limits_enabled(app): node_rights_manager = await NodeRightsManager.instance(app) @@ -335,7 +334,7 @@ async def _remove_containers_save_state_and_outputs() -> None: await _remove_containers_save_state_and_outputs() except NodeRightsAcquireError: # Next observation cycle, the service will try again - logger.debug( + _logger.debug( "Skip saving service state for %s. Docker node %s is busy. Will try later.", scheduler_data.node_uuid, scheduler_data.dynamic_sidecar.docker_node_id, @@ -348,6 +347,13 @@ async def _remove_containers_save_state_and_outputs() -> None: TaskProgress.create(), app, scheduler_data.node_uuid, settings.SWARM_STACK_NAME ) + await safe_remove( + app, + node_id=scheduler_data.node_uuid, + product_name=scheduler_data.product_name, + user_id=scheduler_data.user_id, + ) + # remove sidecar's api client remove_sidecars_client(app, scheduler_data.node_uuid) @@ -368,7 +374,7 @@ async def _remove_containers_save_state_and_outputs() -> None: async def attach_project_networks(app: FastAPI, scheduler_data: SchedulerData) -> None: - logger.debug("Attaching project networks for %s", scheduler_data.service_name) + _logger.debug("Attaching project networks for %s", scheduler_data.service_name) sidecars_client = get_sidecars_client(app, scheduler_data.node_uuid) dynamic_sidecar_endpoint = scheduler_data.endpoint @@ -410,13 +416,13 @@ async def wait_for_sidecar_api(app: FastAPI, scheduler_data: SchedulerData) -> N ), wait=wait_fixed(1), retry_error_cls=EntrypointContainerNotFoundError, - before_sleep=before_sleep_log(logger, logging.DEBUG), + before_sleep=before_sleep_log(_logger, logging.DEBUG), ): with attempt: if not await get_dynamic_sidecar_service_health( app, scheduler_data, with_retry=False ): - raise TryAgain() + raise TryAgain scheduler_data.dynamic_sidecar.is_healthy = True @@ -473,7 +479,7 @@ async def _pull_outputs_and_state(): service_outputs_labels = json.loads( simcore_service_labels.dict().get("io.simcore.outputs", "{}") ).get("outputs", {}) - logger.debug( + _logger.debug( "Creating dirs from service outputs labels: %s", service_outputs_labels, ) @@ -494,7 +500,7 @@ async def _pull_outputs_and_state(): await _pull_outputs_and_state() except NodeRightsAcquireError: # Next observation cycle, the service will try again - logger.debug( + _logger.debug( "Skip saving service state for %s. Docker node %s is busy. Will try later.", scheduler_data.node_uuid, scheduler_data.dynamic_sidecar.docker_node_id, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py index 8a4dfcd390e..078c6f9eae2 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py @@ -2,7 +2,6 @@ """ import logging -from collections.abc import Callable, Mapping from copy import deepcopy from typing import Any @@ -27,6 +26,7 @@ OsparcVariablesTable, resolve_variables_from_context, ) +from .api_keys_manager import get_or_create_api_key from .db.repositories.services_environments import ServicesEnvironmentsRepository _logger = logging.getLogger(__name__) @@ -156,10 +156,14 @@ async def resolve_and_substitute_session_variables_in_specs( if requested := set(resolver.get_identifiers()): available = set(table.variables_names()) - - if identifiers := available.intersection(requested): + identifiers_to_replace = available.intersection(requested) + _logger.debug( + "resolve_and_substitute_session_variables_in_specs identifiers_to_replace=%s", + identifiers_to_replace, + ) + if identifiers_to_replace: environs = await resolve_variables_from_context( - table.copy(include=identifiers), + table.copy(include=identifiers_to_replace), context=ContextDict( app=app, user_id=user_id, @@ -176,14 +180,76 @@ async def resolve_and_substitute_session_variables_in_specs( return deepcopy(specs) -async def resolve_and_substitute_lifespan_variables_in_specs( - _app: FastAPI, - _specs: dict[str, Any], +async def resolve_and_substitute_service_lifetime_variables_in_specs( + app: FastAPI, + specs: dict[str, Any], *, - _callbacks_registry: Mapping[str, Callable], + product_name: ProductName, + user_id: UserID, + node_id: NodeID, safe: bool = True, -): - raise NotImplementedError +) -> dict[str, Any]: + registry: OsparcVariablesTable = app.state.lifespan_osparc_variables_table + + resolver = SpecsSubstitutionsResolver(specs, upgrade=False) + + if requested := set(resolver.get_identifiers()): + available = set(registry.variables_names()) + + if identifiers := available.intersection(requested): + environs = await resolve_variables_from_context( + registry.copy(include=identifiers), + context=ContextDict( + app=app, + product_name=product_name, + user_id=user_id, + node_id=node_id, + ), + # NOTE: the api key and secret cannot be resolved in parallel + # due to race conditions + resolve_in_parallel=False, + ) + + resolver.set_substitutions(mappings=environs) + new_specs: dict[str, Any] = resolver.run(safe=safe) + return new_specs + + return deepcopy(specs) + + +async def _get_or_create_api_key( + app: FastAPI, product_name: ProductName, user_id: UserID, node_id: NodeID +) -> str: + key_data = await get_or_create_api_key( + app, + product_name=product_name, + user_id=user_id, + node_id=node_id, + ) + return key_data.api_key # type:ignore [no-any-return] + + +async def _get_or_create_api_secret( + app: FastAPI, product_name: ProductName, user_id: UserID, node_id: NodeID +) -> str: + key_data = await get_or_create_api_key( + app, + product_name=product_name, + user_id=user_id, + node_id=node_id, + ) + return key_data.api_secret # type:ignore [no-any-return] + + +def _setup_lifespan_osparc_variables_table(app: FastAPI): + app.state.lifespan_osparc_variables_table = table = OsparcVariablesTable() + + table.register_from_handler("OSPARC_VARIABLE_API_KEY")(_get_or_create_api_key) + table.register_from_handler("OSPARC_VARIABLE_API_SECRET")(_get_or_create_api_secret) + + _logger.debug( + "Registered lifespan_osparc_variables_table=%s", sorted(table.variables_names()) + ) async def _request_user_email(app: FastAPI, user_id: UserID) -> EmailStr: @@ -205,6 +271,7 @@ def _setup_session_osparc_variables(app: FastAPI): ("OSPARC_VARIABLE_PRODUCT_NAME", "product_name"), ("OSPARC_VARIABLE_STUDY_UUID", "project_id"), ("OSPARC_VARIABLE_NODE_ID", "node_id"), + # ANE -> PC: why not register the user_id as well at this point? ]: table.register_from_context(name, context_name) @@ -226,5 +293,6 @@ def setup(app: FastAPI): def on_startup() -> None: _setup_session_osparc_variables(app) + _setup_lifespan_osparc_variables_table(app) app.add_event_handler("startup", on_startup) diff --git a/services/director-v2/src/simcore_service_director_v2/utils/distributed_identifer.py b/services/director-v2/src/simcore_service_director_v2/utils/distributed_identifer.py new file mode 100644 index 00000000000..05d84a72f08 --- /dev/null +++ b/services/director-v2/src/simcore_service_director_v2/utils/distributed_identifer.py @@ -0,0 +1,69 @@ +import logging +from abc import ABC, abstractmethod +from typing import Generic, TypeVar + +from servicelib.logging_utils import log_catch, log_context + +_logger = logging.getLogger(__name__) + +Ident = TypeVar("Ident") +Res = TypeVar("Res") + + +class BaseDistributedIdentifierManager(ABC, Generic[Ident, Res]): + """Common interface used to manage the lifecycle of osparc resources. + + An osparc resource can be anything that needs to be created and then removed + during the runtime of the platform. + + Safe remove the resource. + + For usage check ``packages/service-library/tests/test_osparc_generic_resource.py`` + """ + + @abstractmethod + async def get(self, identifier: Ident, **extra_kwargs) -> Res | None: + """Returns a resource if exists. + + Arguments: + identifier -- user chosen identifier for the resource + **extra_kwargs -- can be overloaded by the user + + Returns: + None if the resource does not exit + """ + + @abstractmethod + async def create(self, **extra_kwargs) -> tuple[Ident, Res]: + """Used for creating the resources + + Arguments: + **extra_kwargs -- can be overloaded by the user + + Returns: + tuple[identifier for the resource, resource object] + """ + + @abstractmethod + async def destroy(self, identifier: Ident, **extra_kwargs) -> None: + """Used to destroy an existing resource + + Arguments: + identifier -- user chosen identifier for the resource + **extra_kwargs -- can be overloaded by the user + """ + + async def remove( + self, identifier: Ident, *, reraise: bool = False, **extra_kwargs + ) -> None: + """Attempts to remove the resource, if an error occurs it is logged. + + Arguments: + identifier -- user chosen identifier for the resource + reraise -- when True raises any exception raised by ``destroy`` (default: {False}) + **extra_kwargs -- can be overloaded by the user + """ + with log_context( + _logger, logging.DEBUG, f"{self.__class__}: removing {identifier}" + ), log_catch(_logger, reraise=reraise): + await self.destroy(identifier, **extra_kwargs) diff --git a/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py b/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py index 5e783a807a9..94eadd9fb47 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/osparc_variables.py @@ -5,6 +5,7 @@ from models_library.utils.specs_substitution import SubstitutionValue from pydantic import NonNegativeInt, parse_obj_as +from servicelib.utils import logged_gather ContextDict: TypeAlias = dict[str, Any] ContextGetter: TypeAlias = Callable[[ContextDict], Any] @@ -94,7 +95,21 @@ def copy( async def resolve_variables_from_context( variables_getters: dict[str, ContextGetter], context: ContextDict, + *, + resolve_in_parallel: bool = True, ) -> dict[str, SubstitutionValue]: + """Resolves variables given a list of handlers and a context + containing vars which can be used by the handlers. + + Arguments: + variables_getters -- mapping of awaitables which resolve the value + context -- variables which can be passed to the awaitables + + Keyword Arguments: + resolve_in_parallel -- sometimes the variable_getters cannot be ran in parallel, + for example due to race conditions, + for those situations set to False (default: {True}) + """ # evaluate getters from context values pre_environs: dict[str, SubstitutionValue | RequestTuple] = { key: fun(context) for key, fun in variables_getters.items() @@ -113,7 +128,10 @@ async def resolve_variables_from_context( environs[key] = value # evaluates handlers - values = await asyncio.gather(*coros.values()) + values = await logged_gather( + *coros.values(), + max_concurrency=0 if resolve_in_parallel else 1, + ) for key, value in zip(coros.keys(), values, strict=True): environs[key] = value diff --git a/services/director-v2/tests/unit/test_modules_api_keys_manager.py b/services/director-v2/tests/unit/test_modules_api_keys_manager.py new file mode 100644 index 00000000000..3c56a05936c --- /dev/null +++ b/services/director-v2/tests/unit/test_modules_api_keys_manager.py @@ -0,0 +1,100 @@ +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +from collections.abc import Awaitable, Callable + +import pytest +from faker import Faker +from fastapi import FastAPI +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.api_schemas_webserver.auth import ApiKeyCreate, ApiKeyGet +from models_library.products import ProductName +from models_library.projects_nodes_io import NodeID +from models_library.users import UserID +from pytest_mock import MockerFixture +from servicelib.rabbitmq import RabbitMQRPCClient, RPCRouter +from simcore_service_director_v2.modules.api_keys_manager import ( + APIKeysManager, + _get_api_key_name, +) + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +@pytest.fixture +def node_id(faker: Faker) -> NodeID: + return faker.uuid4(cast_to=None) + + +def test_get_api_key_name_is_not_randomly_generated(node_id: NodeID): + api_key_names = {_get_api_key_name(node_id) for x in range(1000)} + assert len(api_key_names) == 1 + + +@pytest.fixture +async def mock_rpc_server( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], + mocker: MockerFixture, +) -> RabbitMQRPCClient: + rpc_client = await rabbitmq_rpc_client("client") + rpc_server = await rabbitmq_rpc_client("mock_server") + + router = RPCRouter() + + # mocks the interface defined in the webserver + + @router.expose() + async def api_key_get( + product_name: ProductName, user_id: UserID, name: str + ) -> ApiKeyGet: + return ApiKeyGet.parse_obj(ApiKeyGet.Config.schema_extra["examples"][0]) + + @router.expose() + async def create_api_keys( + product_name: ProductName, user_id: UserID, new: ApiKeyCreate + ) -> ApiKeyGet: + return ApiKeyGet.parse_obj(ApiKeyGet.Config.schema_extra["examples"][0]) + + @router.expose() + async def delete_api_keys( + product_name: ProductName, user_id: UserID, name: str + ) -> None: + ... + + await rpc_server.register_router(router, namespace=WEBSERVER_RPC_NAMESPACE) + + # mock returned client + mocker.patch( + "simcore_service_director_v2.modules.api_keys_manager.get_rabbitmq_rpc_client", + return_value=rpc_client, + ) + + return rpc_client + + +async def test_rpc_endpoints( + mock_rpc_server: RabbitMQRPCClient, + faker: Faker, +): + manager = APIKeysManager(FastAPI()) + + identifier = faker.pystr() + product_name = faker.pystr() + user_id = faker.pyint() + + api_key = await manager.get( + identifier=identifier, product_name=product_name, user_id=user_id + ) + assert isinstance(api_key, ApiKeyGet) + + identifier, api_key = await manager.create( + identifier=identifier, product_name=product_name, user_id=user_id + ) + assert isinstance(identifier, str) + assert isinstance(api_key, ApiKeyGet) + + await manager.destroy( + identifier=identifier, product_name=product_name, user_id=user_id + ) diff --git a/services/director-v2/tests/unit/test_utils_distributed_identifier.py b/services/director-v2/tests/unit/test_utils_distributed_identifier.py new file mode 100644 index 00000000000..9d645e7b1da --- /dev/null +++ b/services/director-v2/tests/unit/test_utils_distributed_identifier.py @@ -0,0 +1,137 @@ +# pylint:disable=redefined-outer-name + +import random +import string +from typing import Any +from uuid import UUID, uuid4 + +import pytest +from pytest_mock import MockerFixture +from simcore_service_director_v2.utils.distributed_identifer import ( + BaseDistributedIdentifierManager, +) + + +# define a custom type of ID for the API +class UserDefinedID: + def __init__(self, uuid: UUID | None = None) -> None: + self._id = uuid if uuid else uuid4() + + def __eq__(self, other: "UserDefinedID") -> bool: + return self._id == other._id + + def __hash__(self): + return hash(str(self._id)) + + +# mocked api interface +class RandomTextAPI: + def __init__(self) -> None: + self._created: dict[UserDefinedID, Any] = {} + + @staticmethod + def _random_string(length: int) -> str: + letters_and_digits = string.ascii_letters + string.digits + return "".join( + random.choice(letters_and_digits) for _ in range(length) # noqa: S311 + ) + + def create(self, length: int) -> tuple[UserDefinedID, Any]: + identifier = UserDefinedID(uuid4()) + self._created[identifier] = self._random_string(length) + return identifier, self._created[identifier] + + def delete(self, identifier: UserDefinedID) -> None: + del self._created[identifier] + + def get(self, identifier: UserDefinedID) -> Any | None: + return self._created.get(identifier, None) + + +# define a custom manager using the custom user defined identifiers +# NOTE: note that the generic uses `[UserDefinedID, Any]` +# which enforces typing constraints on the overloaded abstract methods +class RandomTextResourcesManager(BaseDistributedIdentifierManager[UserDefinedID, Any]): + # pylint:disable=arguments-differ + + def __init__(self) -> None: + self.api = RandomTextAPI() + + async def get(self, identifier: UserDefinedID, **_) -> Any | None: + return self.api.get(identifier) + + async def create(self, length: int) -> tuple[UserDefinedID, Any]: + return self.api.create(length) + + async def destroy(self, identifier: UserDefinedID) -> None: + self.api.delete(identifier) + + +@pytest.fixture +def manager() -> RandomTextResourcesManager: + return RandomTextResourcesManager() + + +async def test_resource_is_missing(manager: RandomTextResourcesManager): + missing_identifier = UserDefinedID() + assert await manager.get(missing_identifier) is None + + +async def test_manual_workflow(manager: RandomTextResourcesManager): + # creation + identifier, _ = await manager.create(length=1) + assert await manager.get(identifier) is not None + + # removal + await manager.destroy(identifier) + + # resource no longer exists + assert await manager.get(identifier) is None + + +@pytest.mark.parametrize("delete_before_removal", [True, False]) +async def test_automatic_cleanup_workflow( + manager: RandomTextResourcesManager, delete_before_removal: bool +): + # creation + identifier, _ = await manager.create(length=1) + assert await manager.get(identifier) is not None + + # optional removal + if delete_before_removal: + await manager.destroy(identifier) + + is_still_present = not delete_before_removal + assert (await manager.get(identifier) is not None) is is_still_present + + # safe remove the resource + await manager.remove(identifier) + + # resource no longer exists + assert await manager.get(identifier) is None + + +@pytest.mark.parametrize("reraise", [True, False]) +async def test_remove_raises_error( + mocker: MockerFixture, + manager: RandomTextResourcesManager, + caplog: pytest.LogCaptureFixture, + reraise: bool, +): + caplog.clear() + + error_message = "mock error during resource destroy" + mocker.patch.object(manager, "destroy", side_effect=RuntimeError(error_message)) + + # after creation object is present + identifier, _ = await manager.create(length=1) + assert await manager.get(identifier) is not None + + if reraise: + with pytest.raises(RuntimeError): + await manager.remove(identifier, reraise=reraise) + else: + await manager.remove(identifier, reraise=reraise) + # check logs in case of error + assert "Unhandled exception:" in caplog.text + assert error_message in caplog.text diff --git a/services/director-v2/tests/unit/test_utils_osparc_variables.py b/services/director-v2/tests/unit/test_utils_osparc_variables.py index 84bf7ce2e40..17c3b4d9899 100644 --- a/services/director-v2/tests/unit/test_utils_osparc_variables.py +++ b/services/director-v2/tests/unit/test_utils_osparc_variables.py @@ -18,7 +18,7 @@ from pytest_simcore.helpers.faker_compose_specs import generate_fake_docker_compose from simcore_postgres_database.models.users import UserRole from simcore_service_director_v2.modules.osparc_variables_substitutions import ( - resolve_and_substitute_lifespan_variables_in_specs, + resolve_and_substitute_service_lifetime_variables_in_specs, resolve_and_substitute_session_variables_in_specs, substitute_vendor_secrets_in_specs, ) @@ -49,7 +49,7 @@ def session_context(faker: Faker) -> ContextDict: async def test_resolve_session_environs(faker: Faker, session_context: ContextDict): assert resolve_and_substitute_session_variables_in_specs assert substitute_vendor_secrets_in_specs - assert resolve_and_substitute_lifespan_variables_in_specs + assert resolve_and_substitute_service_lifetime_variables_in_specs async def _request_user_role(app: FastAPI, user_id: UserID) -> SubstitutionValue: print(app, user_id) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 223c4402b0d..d84d9c55f10 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -445,13 +445,6 @@ paths: summary: List Api Keys description: lists display names of API keys by this user operationId: list_api_keys - parameters: - - required: true - schema: - title: Code - type: string - name: code - in: query responses: '200': description: returns the display names of API keys diff --git a/services/web/server/src/simcore_service_webserver/api_keys/_api.py b/services/web/server/src/simcore_service_webserver/api_keys/_api.py index 9ce15603ee4..6768ce40fcb 100644 --- a/services/web/server/src/simcore_service_webserver/api_keys/_api.py +++ b/services/web/server/src/simcore_service_webserver/api_keys/_api.py @@ -1,4 +1,3 @@ -import logging import re import string from typing import Final @@ -11,9 +10,6 @@ from ..login.utils import get_random_string from ._db import ApiKeyRepo -_logger = logging.getLogger(__name__) - - _PUNCTUATION_REGEX = re.compile( pattern="[" + re.escape(string.punctuation.replace("_", "")) + "]" ) @@ -62,6 +58,14 @@ async def create_api_key( ) +async def get( + app: web.Application, *, name: str, user_id: UserID, product_name: ProductName +) -> ApiKeyGet | None: + repo = ApiKeyRepo.create_from_app(app) + row = await repo.get(display_name=name, user_id=user_id, product_name=product_name) + return ApiKeyGet.parse_obj(row) if row else None + + async def delete_api_key( app: web.Application, *, diff --git a/services/web/server/src/simcore_service_webserver/api_keys/_db.py b/services/web/server/src/simcore_service_webserver/api_keys/_db.py index 8cb12f07172..abbbf05e52b 100644 --- a/services/web/server/src/simcore_service_webserver/api_keys/_db.py +++ b/services/web/server/src/simcore_service_webserver/api_keys/_db.py @@ -5,7 +5,8 @@ import sqlalchemy as sa from aiohttp import web from aiopg.sa.engine import Engine -from aiopg.sa.result import ResultProxy +from aiopg.sa.result import ResultProxy, RowProxy +from models_library.api_schemas_api_server.api_keys import ApiKeyInDB from models_library.basic_types import IdInt from models_library.products import ProductName from models_library.users import UserID @@ -27,7 +28,7 @@ async def list_names( self, *, user_id: UserID, product_name: ProductName ) -> list[str]: async with self.engine.acquire() as conn: - stmt = sa.select(api_keys.c.display_name,).where( + stmt = sa.select(api_keys.c.display_name).where( (api_keys.c.user_id == user_id) & (api_keys.c.product_name == product_name) ) @@ -64,9 +65,24 @@ async def create( rows = await result.fetchall() or [] return [r.id for r in rows] + async def get( + self, *, display_name: str, user_id: UserID, product_name: ProductName + ) -> ApiKeyInDB | None: + async with self.engine.acquire() as conn: + stmt = sa.select(api_keys).where( + (api_keys.c.user_id == user_id) + & (api_keys.c.display_name == display_name) + & (api_keys.c.product_name == product_name) + ) + + result: ResultProxy = await conn.execute(stmt) + row: RowProxy | None = await result.fetchone() + _logger.error("debug %s", row) + return ApiKeyInDB.from_orm(row) if row else None + async def delete_by_name( self, *, display_name: str, user_id: UserID, product_name: ProductName - ): + ) -> None: async with self.engine.acquire() as conn: stmt = api_keys.delete().where( (api_keys.c.user_id == user_id) @@ -77,7 +93,7 @@ async def delete_by_name( async def delete_by_key( self, *, api_key: str, user_id: UserID, product_name: ProductName - ): + ) -> None: async with self.engine.acquire() as conn: stmt = api_keys.delete().where( (api_keys.c.user_id == user_id) diff --git a/services/web/server/src/simcore_service_webserver/api_keys/_rpc.py b/services/web/server/src/simcore_service_webserver/api_keys/_rpc.py index 8c94ed271e8..ef7fb9b303c 100644 --- a/services/web/server/src/simcore_service_webserver/api_keys/_rpc.py +++ b/services/web/server/src/simcore_service_webserver/api_keys/_rpc.py @@ -31,12 +31,23 @@ async def delete_api_keys( product_name: ProductName, user_id: UserID, name: str, -): - return await _api.delete_api_key( +) -> None: + await _api.delete_api_key( app, name=name, user_id=user_id, product_name=product_name ) +@router.expose() +async def api_key_get( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + name: str, +) -> ApiKeyGet | None: + return await _api.get(app, name=name, user_id=user_id, product_name=product_name) + + async def register_rpc_routes_on_startup(app: web.Application): rpc_server = get_rabbitmq_rpc_server(app) await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/tests/unit/with_dbs/01/test_api_keys.py b/services/web/server/tests/unit/with_dbs/01/test_api_keys.py index ef218513b36..1b4eeb7c257 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_api_keys.py +++ b/services/web/server/tests/unit/with_dbs/01/test_api_keys.py @@ -80,20 +80,19 @@ async def test_create_api_keys( expected: type[web.HTTPException], disable_gc_manual_guest_users: None, ): - resp = await client.post("/v0/auth/api-keys", json={"display_name": "foo"}) + display_name = "foo" + resp = await client.post("/v0/auth/api-keys", json={"display_name": display_name}) data, errors = await assert_status(resp, expected) if not errors: - assert data["display_name"] == "foo" + assert data["display_name"] == display_name assert "api_key" in data assert "api_secret" in data resp = await client.get("/v0/auth/api-keys") data, _ = await assert_status(resp, expected) - assert sorted(data) == [ - "foo", - ] + assert sorted(data) == [display_name] @pytest.mark.parametrize( diff --git a/services/web/server/tests/unit/with_dbs/01/test_api_keys_rpc.py b/services/web/server/tests/unit/with_dbs/01/test_api_keys_rpc.py new file mode 100644 index 00000000000..b70812f09ae --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/01/test_api_keys_rpc.py @@ -0,0 +1,164 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +from collections.abc import AsyncIterable, Awaitable, Callable + +import pytest +from aiohttp.test_utils import TestServer +from faker import Faker +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.api_schemas_webserver.auth import ApiKeyCreate +from models_library.products import ProductName +from models_library.rabbitmq_basic_types import RPCMethodName +from pydantic import parse_obj_as +from pytest_mock import MockerFixture +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_login import UserInfoDict +from servicelib.rabbitmq import RabbitMQRPCClient +from settings_library.rabbit import RabbitSettings +from simcore_postgres_database.models.users import UserRole +from simcore_service_webserver.api_keys._db import ApiKeyRepo +from simcore_service_webserver.application_settings import ApplicationSettings + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +@pytest.fixture +def app_environment( + rabbit_service: RabbitSettings, + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, +): + new_envs = setenvs_from_dict( + monkeypatch, + { + **app_environment, + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + }, + ) + + settings = ApplicationSettings.create_from_envs() + assert settings.WEBSERVER_RABBITMQ + + return new_envs + + +@pytest.fixture +def user_role() -> UserRole: + return UserRole.USER + + +@pytest.fixture +async def fake_user_api_keys( + user_role: UserRole, + web_server: TestServer, + logged_user: UserInfoDict, + osparc_product_name: ProductName, +) -> AsyncIterable[list[str]]: + names = ["foo", "bar", "beta", "alpha"] + repo = ApiKeyRepo.create_from_app(app=web_server.app) + + for name in names: + await repo.create( + user_id=logged_user["id"], + product_name=osparc_product_name, + display_name=name, + expiration=None, + api_key=f"{name}-key", + api_secret=f"{name}-secret", + ) + + yield names + + for name in names: + await repo.delete_by_name( + display_name=name, + user_id=logged_user["id"], + product_name=osparc_product_name, + ) + + +@pytest.fixture +async def rpc_client( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], + mocker: MockerFixture, +) -> RabbitMQRPCClient: + return await rabbitmq_rpc_client("client") + + +async def test_api_key_get( + fake_user_api_keys: list[str], + rpc_client: RabbitMQRPCClient, + osparc_product_name: ProductName, + logged_user: UserInfoDict, +): + for api_key_name in fake_user_api_keys: + result = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "api_key_get"), + product_name=osparc_product_name, + user_id=logged_user["id"], + name=api_key_name, + ) + assert result.display_name == api_key_name + + +async def test_api_keys_workflow( + web_server: TestServer, + rpc_client: RabbitMQRPCClient, + osparc_product_name: ProductName, + logged_user: UserInfoDict, + faker: Faker, +): + key_name = faker.pystr() + + # creating a key + created_api_key = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "create_api_keys"), + product_name=osparc_product_name, + user_id=logged_user["id"], + new=ApiKeyCreate(display_name=key_name, expiration=None), + ) + assert created_api_key.display_name == key_name + + # query the key is still present + queried_api_key = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "api_key_get"), + product_name=osparc_product_name, + user_id=logged_user["id"], + name=key_name, + ) + assert queried_api_key.display_name == key_name + + assert created_api_key == queried_api_key + + # remove the key + delete_key_result = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "delete_api_keys"), + product_name=osparc_product_name, + user_id=logged_user["id"], + name=key_name, + ) + assert delete_key_result is None + + # key no longer present + query_missing_query = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "api_key_get"), + product_name=osparc_product_name, + user_id=logged_user["id"], + name=key_name, + ) + assert query_missing_query is None From 5477e3d417bf69017cd153fd595c004761798969 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:59:25 +0100 Subject: [PATCH 11/24] =?UTF-8?q?=20=E2=9C=A8=20New=20`pay=20with=20paymen?= =?UTF-8?q?t-method`=20implementation=20(#5017)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + .vscode/extensions.json | 2 + api/specs/web-server/_wallets.py | 4 +- .../api_schemas_webserver/wallets.py | 26 ++-- .../pytest_simcore/helpers/rawdata_fakers.py | 6 +- services/payments/Makefile | 4 +- services/payments/VERSION | 2 +- services/payments/openapi.json | 14 +- services/payments/scripts/Makefile | 2 +- .../payments/scripts/fake_payment_gateway.py | 13 +- services/payments/setup.cfg | 4 +- .../api/rest/_dependencies.py | 3 +- .../api/rpc/_payments_methods.py | 7 +- .../db/payments_transactions_repo.py | 2 +- .../src/simcore_service_payments/models/db.py | 28 +++- .../models/payments_gateway.py | 15 +- .../models/schemas/acknowledgements.py | 51 ++++-- .../simcore_service_payments/models/utils.py | 3 - .../services/payments.py | 67 +++++--- .../services/payments_gateway.py | 63 +++++++- .../services/postgres.py | 6 + services/payments/tests/unit/conftest.py | 46 ++++-- .../tests/unit/test_rpc_payments_methods.py | 18 +-- .../unit/test_services_payments_gateway.py | 23 ++- .../paymentMethods/PaymentMethodDetails.js | 5 +- .../api/v0/openapi.yaml | 27 +--- .../payments/_methods_api.py | 5 +- .../payments/_onetime_api.py | 145 ++++++++++++++---- .../payments/_rpc.py | 11 +- .../simcore_service_webserver/payments/api.py | 8 +- .../wallets/_payments_handlers.py | 80 ++++++---- .../with_dbs/03/wallets/payments/conftest.py | 31 ++-- .../wallets/payments/test_payments_methods.py | 22 +-- 33 files changed, 489 insertions(+), 255 deletions(-) diff --git a/.gitignore b/.gitignore index ec92564a31c..9836532c4c9 100644 --- a/.gitignore +++ b/.gitignore @@ -178,5 +178,6 @@ Untitled* # service settings.schemas.json services/**/settings-schema.json +services/payments/scripts/openapi.json tests/public-api/osparc_python_wheels/* diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 65f20197c67..02968d94c72 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,5 +1,6 @@ { "recommendations": [ + "42Crunch.vscode-openapi", "charliermarsh.ruff", "eamodio.gitlens", "exiasr.hadolint", @@ -7,6 +8,7 @@ "ms-python.black-formatter", "ms-python.pylint", "ms-python.python", + "ms-vscode.makefile-tools", "njpwerner.autodocstring", "samuelcolvin.jinjahtml", "timonwong.shellcheck", diff --git a/api/specs/web-server/_wallets.py b/api/specs/web-server/_wallets.py index c93054a2cec..9a0643073d2 100644 --- a/api/specs/web-server/_wallets.py +++ b/api/specs/web-server/_wallets.py @@ -168,10 +168,10 @@ async def delete_payment_method( @router.post( "/wallets/{wallet_id}/payments-methods/{payment_method_id}:pay", response_model=Envelope[WalletPaymentInitiated], - response_description="Payment initialized", + response_description="Pay with payment-method", status_code=status.HTTP_202_ACCEPTED, ) -async def init_payment_with_payment_method( +async def pay_with_payment_method( wallet_id: WalletID, payment_method_id: PaymentMethodID, _body: CreateWalletPayment ): ... diff --git a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py index 47d328b1df2..85a96b73e50 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py @@ -4,7 +4,7 @@ from pydantic import Field, HttpUrl -from ..basic_types import IDStr, NonNegativeDecimal +from ..basic_types import AmountDecimal, IDStr, NonNegativeDecimal from ..users import GroupID from ..utils.pydantic_tools_extension import FieldNotRequired from ..wallets import WalletID, WalletStatus @@ -55,7 +55,7 @@ class PutWalletBodyParams(OutputSchema): class CreateWalletPayment(InputSchema): - price_dollars: Decimal + price_dollars: AmountDecimal comment: str = FieldNotRequired(max_length=100) @@ -124,14 +124,11 @@ class Config(OutputSchema.Config): class PaymentMethodGet(OutputSchema): idr: PaymentMethodID wallet_id: WalletID - card_holder_name: str - card_number_masked: str - card_type: str - expiration_month: int - expiration_year: int - street_address: str - zipcode: str - country: str + card_holder_name: str | None = None + card_number_masked: str | None = None + card_type: str | None = None + expiration_month: int | None = None + expiration_year: int | None = None created: datetime auto_recharge: bool = Field( default=False, @@ -149,12 +146,15 @@ class Config(OutputSchema.Config): "cardType": "Visa", "expirationMonth": 10, "expirationYear": 2025, - "streetAddress": "123 Main St", - "zipcode": "12345", - "country": "United States", "created": "2023-09-13T15:30:00Z", "autoRecharge": "False", }, + { + "idr": "pm_1234567890", + "walletId": 3, + "created": "2024-09-13T15:30:00Z", + "autoRecharge": "False", + }, ], } diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py index 0b5564f9d20..036d0a9c126 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py @@ -251,16 +251,14 @@ def random_api_key(product_name: str, user_id: int, **overrides) -> dict[str, An def random_payment_method_data(**overrides) -> dict[str, Any]: # Produces data for GetPaymentMethod data = { - "idr": FAKE.uuid4(), + "id": FAKE.uuid4(), "card_holder_name": FAKE.name(), "card_number_masked": f"**** **** **** {FAKE.credit_card_number()[:4]}", "card_type": FAKE.credit_card_provider(), "expiration_month": FAKE.random_int(min=1, max=12), "expiration_year": FAKE.future_date().year, - "street_address": FAKE.street_address(), - "zipcode": FAKE.zipcode(), - "country": FAKE.country(), "created": utcnow(), } + assert set(overrides.keys()).issubset(data.keys()) data.update(**overrides) return data diff --git a/services/payments/Makefile b/services/payments/Makefile index db3152765a6..0af63b77c0c 100644 --- a/services/payments/Makefile +++ b/services/payments/Makefile @@ -23,8 +23,8 @@ external ?= .env-secret test-dev-unit-external: ## runs test-dev against external service defined in $(external) envfile # Running tests using external environ '$(external)' - $(MAKE) test-dev-unit pytest-parameters="--external-envfile=$(external)" + $(MAKE) test-dev-unit pytest-parameters="--external-envfile=$(external) -m can_run_against_external" test-ci-unit-external: ## runs test-ci against external service defined in $(external) envfile # Running tests using external environ '$(external)' - $(MAKE) test-ci-unit pytest-parameters="--external-envfile=$(external)" + $(MAKE) test-ci-unit pytest-parameters="--external-envfile=$(external) -m can_run_against_external" diff --git a/services/payments/VERSION b/services/payments/VERSION index 3a3cd8cc8b0..88c5fb891dc 100644 --- a/services/payments/VERSION +++ b/services/payments/VERSION @@ -1 +1 @@ -1.3.1 +1.4.0 diff --git a/services/payments/openapi.json b/services/payments/openapi.json index 692374075ef..0a0590edf31 100644 --- a/services/payments/openapi.json +++ b/services/payments/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "simcore-service-payments web API", "description": " Service that manages creation and validation of registration payments", - "version": "1.3.1" + "version": "1.4.0" }, "paths": { "/": { @@ -220,6 +220,13 @@ "type": "string", "title": "Message" }, + "provider_payment_id": { + "type": "string", + "maxLength": 50, + "minLength": 1, + "title": "Provider Payment Id", + "description": "Payment ID from the provider (e.g. stripe payment ID)" + }, "invoice_url": { "type": "string", "maxLength": 2083, @@ -235,7 +242,7 @@ } ], "title": "Saved", - "description": "If the user decided to save the payment methodafter payment it returns the payment-method acknoledgement response.Otherwise it defaults to None." + "description": "Gets the payment-method if user opted to save it during payment.If used did not opt to save of payment-method was already saved, then it defaults to None" } }, "type": "object", @@ -245,10 +252,11 @@ "title": "AckPayment", "example": { "success": true, + "provider_payment_id": "pi_123ABC", "invoice_url": "https://invoices.com/id=12345", "saved": { "success": true, - "payment_method_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + "payment_method_id": "3FA85F64-5717-4562-B3FC-2C963F66AFA6" } } }, diff --git a/services/payments/scripts/Makefile b/services/payments/scripts/Makefile index 78a4a19b81f..166c0763f9d 100644 --- a/services/payments/scripts/Makefile +++ b/services/payments/scripts/Makefile @@ -9,6 +9,6 @@ run-devel: ## runs fake_payment_gateway server uvicorn fake_payment_gateway:the_app --reload -openapi.json: +openapi.json: ## creates OAS @set -o allexport; source .env-secret; set +o allexport; \ python fake_payment_gateway.py openapi > $@ diff --git a/services/payments/scripts/fake_payment_gateway.py b/services/payments/scripts/fake_payment_gateway.py index 883ed6d8845..0df5e7b76f2 100644 --- a/services/payments/scripts/fake_payment_gateway.py +++ b/services/payments/scripts/fake_payment_gateway.py @@ -39,7 +39,10 @@ PaymentMethodInitiated, PaymentMethodsBatch, ) -from simcore_service_payments.models.schemas.acknowledgements import AckPayment +from simcore_service_payments.models.schemas.acknowledgements import ( + AckPayment, + AckPaymentWithPaymentMethod, +) from simcore_service_payments.models.schemas.auth import Token logging.basicConfig(level=logging.INFO) @@ -291,7 +294,7 @@ def batch_get_payment_methods( @router.get( "/{id}", - response_class=GetPaymentMethod, + response_model=GetPaymentMethod, responses=ERROR_RESPONSES, ) def get_payment_method( @@ -315,10 +318,10 @@ def delete_payment_method( @router.post( "/{id}:pay", - response_model=PaymentInitiated, + response_model=AckPaymentWithPaymentMethod, responses=ERROR_RESPONSES, ) - def init_payment_with_payment_method( + def pay_with_payment_method( id: PaymentMethodID, payment: InitPayment, auth: Annotated[int, Depends(auth_session)], @@ -344,7 +347,7 @@ async def _app_lifespan(app: FastAPI): def create_app(): app = FastAPI( title="fake-payment-gateway", - version="0.2.1", + version="0.3.0", lifespan=_app_lifespan, debug=True, ) diff --git a/services/payments/setup.cfg b/services/payments/setup.cfg index 7a52ccb4934..cb32927a81b 100644 --- a/services/payments/setup.cfg +++ b/services/payments/setup.cfg @@ -1,14 +1,16 @@ [bumpversion] -current_version = 1.3.1 +current_version = 1.4.0 commit = True message = services/payments version: {current_version} → {new_version} tag = False commit_args = --no-verify [bumpversion:file:VERSION] +[bumpversion:file:openapi.json] [tool:pytest] asyncio_mode = auto markers = testit: "marks test to run during development" acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." + can_run_against_external: "marks tests that *can* be run against an external configuration passed by --external-envfile" diff --git a/services/payments/src/simcore_service_payments/api/rest/_dependencies.py b/services/payments/src/simcore_service_payments/api/rest/_dependencies.py index 8388242d820..773f215cbba 100644 --- a/services/payments/src/simcore_service_payments/api/rest/_dependencies.py +++ b/services/payments/src/simcore_service_payments/api/rest/_dependencies.py @@ -12,6 +12,7 @@ from ...db.base import BaseRepository from ...models.auth import SessionData from ...services.auth import get_session_data +from ...services.postgres import get_engine from ...services.resource_usage_tracker import ResourceUsageTrackerApi _logger = logging.getLogger(__name__) @@ -43,7 +44,7 @@ def get_rut_api(request: Request) -> ResourceUsageTrackerApi: def get_db_engine(request: Request) -> AsyncEngine: - engine: AsyncEngine = request.app.state.engine + engine: AsyncEngine = get_engine(request.app) assert engine # nosec return engine diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py index 5b02867ef57..ef531a5b585 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py @@ -6,7 +6,6 @@ PaymentMethodGet, PaymentMethodID, PaymentMethodInitiated, - WalletPaymentInitiated, ) from models_library.basic_types import IDStr from models_library.users import UserID @@ -113,7 +112,7 @@ async def delete_payment_method( @router.expose() -async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-arguments +async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-arguments app: FastAPI, *, payment_method_id: PaymentMethodID, @@ -126,8 +125,8 @@ async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=t user_name: str, user_email: EmailStr, comment: str | None = None, -) -> WalletPaymentInitiated: - return await payments.init_payment_with_payment_method( +): + return await payments.pay_with_payment_method( gateway=PaymentsGatewayApi.get_from_app_state(app), repo_transactions=PaymentsTransactionsRepo(db_engine=app.state.engine), repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine), diff --git a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py index 0b7e972be76..8cfde93980d 100644 --- a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py @@ -36,10 +36,10 @@ async def insert_init_payment_transaction( initiated_at: datetime.datetime, ) -> PaymentID: """Annotates init-payment transaction + Raises: PaymentAlreadyExistsError """ - try: async with self.db_engine.begin() as conn: await conn.execute( diff --git a/services/payments/src/simcore_service_payments/models/db.py b/services/payments/src/simcore_service_payments/models/db.py index be7f858bfe2..85fbea0c160 100644 --- a/services/payments/src/simcore_service_payments/models/db.py +++ b/services/payments/src/simcore_service_payments/models/db.py @@ -2,7 +2,11 @@ from decimal import Decimal from typing import Any, ClassVar -from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID +from models_library.api_schemas_webserver.wallets import ( + PaymentID, + PaymentMethodID, + PaymentTransaction, +) from models_library.emails import LowerCaseEmailStr from models_library.products import ProductName from models_library.users import UserID @@ -59,6 +63,28 @@ class Config: ] } + def to_api_model(self) -> PaymentTransaction: + data: dict[str, Any] = { + "payment_id": self.payment_id, + "price_dollars": self.price_dollars, + "osparc_credits": self.osparc_credits, + "wallet_id": self.wallet_id, + "created_at": self.initiated_at, + "state": self.state, + "completed_at": self.completed_at, + } + + if self.comment: + data["comment"] = self.comment + + if self.state_message: + data["state_message"] = self.state_message + + if self.invoice_url: + data["invoice_url"] = self.invoice_url + + return PaymentTransaction.parse_obj(data) + _EXAMPLE_AFTER_INIT_PAYMENT_METHOD = { "payment_method_id": "12345", diff --git a/services/payments/src/simcore_service_payments/models/payments_gateway.py b/services/payments/src/simcore_service_payments/models/payments_gateway.py index b21c118d0f4..6614f67d303 100644 --- a/services/payments/src/simcore_service_payments/models/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/models/payments_gateway.py @@ -51,15 +51,12 @@ class PaymentMethodInitiated(BaseModel): class GetPaymentMethod(BaseModel): - idr: PaymentMethodID - card_holder_name: str - card_number_masked: str - card_type: str - expiration_month: int - expiration_year: int - street_address: str - zipcode: str - country: str + id: PaymentMethodID + card_holder_name: str | None = None + card_number_masked: str | None = None + card_type: str | None = None + expiration_month: int | None = None + expiration_year: int | None = None created: datetime diff --git a/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py b/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py index ac08098a2cd..59ef23ed51b 100644 --- a/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py +++ b/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py @@ -1,6 +1,7 @@ from typing import Any, ClassVar from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID +from models_library.basic_types import IDStr from pydantic import BaseModel, Field, HttpUrl, validator @@ -9,6 +10,17 @@ class _BaseAck(BaseModel): message: str = Field(default=None) +class _BaseAckPayment(_BaseAck): + provider_payment_id: IDStr = Field( + default=None, + description="Payment ID from the provider (e.g. stripe payment ID)", + ) + + invoice_url: HttpUrl | None = Field( + default=None, description="Link to invoice is required when success=true" + ) + + # # ACK payment-methods # @@ -22,8 +34,13 @@ class SavedPaymentMethod(AckPaymentMethod): payment_method_id: PaymentMethodID +# +# ACK payments +# + _ONE_TIME_SUCCESS: dict[str, Any] = { "success": True, + "provider_payment_id": "pi_123ABC", "invoice_url": "https://invoices.com/id=12345", } _EXAMPLES: list[dict[str, Any]] = [ @@ -34,7 +51,7 @@ class SavedPaymentMethod(AckPaymentMethod): **_ONE_TIME_SUCCESS, "saved": { "success": True, - "payment_method_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "payment_method_id": "3FA85F64-5717-4562-B3FC-2C963F66AFA6", }, }, # 2. one-time-payment successful but payment-method-saved failed @@ -53,20 +70,12 @@ class SavedPaymentMethod(AckPaymentMethod): ] -# -# ACK one-time payments -# +class AckPayment(_BaseAckPayment): - -class AckPayment(_BaseAck): - invoice_url: HttpUrl | None = Field( - default=None, description="Link to invoice is required when success=true" - ) saved: SavedPaymentMethod | None = Field( default=None, - description="If the user decided to save the payment method" - "after payment it returns the payment-method acknoledgement response." - "Otherwise it defaults to None.", + description="Gets the payment-method if user opted to save it during payment." + "If used did not opt to save of payment-method was already saved, then it defaults to None", ) class Config: @@ -85,6 +94,24 @@ def success_requires_invoice(cls, v, values): return v +class AckPaymentWithPaymentMethod(_BaseAckPayment): + # NOTE: This model is equivalent to `AckPayment`, nonetheless + # I decided to separate it for clarity in the OAS since in payments + # w/ payment-method the field `saved` will never be provided, + + payment_id: PaymentID = Field( + default=None, description="Payment ID from the gateway" + ) + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "example": { + **_ONE_TIME_SUCCESS, + "payment_id": "D19EE68B-B007-4B61-A8BC-32B7115FB244", + }, # shown in openapi.json + } + + assert PaymentID # nosec assert PaymentMethodID # nosec diff --git a/services/payments/src/simcore_service_payments/models/utils.py b/services/payments/src/simcore_service_payments/models/utils.py index 3326ca1be08..af5cbfe537e 100644 --- a/services/payments/src/simcore_service_payments/models/utils.py +++ b/services/payments/src/simcore_service_payments/models/utils.py @@ -15,9 +15,6 @@ def merge_models(got: GetPaymentMethod, acked: PaymentsMethodsDB) -> PaymentMeth card_type=got.card_type, expiration_month=got.expiration_month, expiration_year=got.expiration_year, - street_address=got.street_address, - zipcode=got.zipcode, - country=got.country, created=acked.completed_at, auto_recharge=False, # this will be fileld in the web/server ) diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index dcb0dceac17..5fa4f374c33 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -6,12 +6,14 @@ # pylint: disable=too-many-arguments import logging +import uuid from decimal import Decimal import arrow from models_library.api_schemas_webserver.wallets import ( PaymentID, PaymentMethodID, + PaymentTransaction, WalletPaymentInitiated, ) from models_library.users import UserID @@ -22,13 +24,20 @@ PaymentTransactionState, ) from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo +from tenacity import AsyncRetrying +from tenacity.retry import retry_if_exception_type +from tenacity.stop import stop_after_attempt from .._constants import RUT -from ..core.errors import PaymentAlreadyAckedError, PaymentNotFoundError +from ..core.errors import ( + PaymentAlreadyAckedError, + PaymentAlreadyExistsError, + PaymentNotFoundError, +) from ..db.payments_transactions_repo import PaymentsTransactionsRepo from ..models.db import PaymentsTransactionsDB from ..models.payments_gateway import InitPayment, PaymentInitiated -from ..models.schemas.acknowledgements import AckPayment +from ..models.schemas.acknowledgements import AckPayment, AckPaymentWithPaymentMethod from ..services.resource_usage_tracker import ResourceUsageTrackerApi from .payments_gateway import PaymentsGatewayApi @@ -91,7 +100,6 @@ async def cancel_one_time_payment( user_id: UserID, wallet_id: WalletID, ) -> None: - payment = await repo.get_payment_transaction( payment_id=payment_id, user_id=user_id, wallet_id=wallet_id ) @@ -123,7 +131,6 @@ async def acknowledge_one_time_payment( payment_id: PaymentID, ack: AckPayment, ) -> PaymentsTransactionsDB: - return await repo_transactions.update_ack_payment_transaction( payment_id=payment_id, completion_state=( @@ -173,7 +180,7 @@ async def on_payment_completed( ) -async def init_payment_with_payment_method( +async def pay_with_payment_method( # noqa: PLR0913 gateway: PaymentsGatewayApi, repo_transactions: PaymentsTransactionsRepo, repo_methods: PaymentsMethodsRepo, @@ -188,14 +195,14 @@ async def init_payment_with_payment_method( user_name: str, user_email: EmailStr, comment: str | None = None, -) -> WalletPaymentInitiated: +) -> PaymentTransaction: initiated_at = arrow.utcnow().datetime acked = await repo_methods.get_payment_method( payment_method_id, user_id=user_id, wallet_id=wallet_id ) - payment_inited = await gateway.init_payment_with_payment_method( + ack: AckPaymentWithPaymentMethod = await gateway.pay_with_payment_method( acked.payment_method_id, payment=InitPayment( amount_dollars=amount_dollars, @@ -206,23 +213,41 @@ async def init_payment_with_payment_method( ), ) - payment_id = await repo_transactions.insert_init_payment_transaction( - payment_id=payment_inited.payment_id, - price_dollars=amount_dollars, - osparc_credits=target_credits, - product_name=product_name, - user_id=user_id, - user_email=user_email, - wallet_id=wallet_id, - comment=comment, - initiated_at=initiated_at, - ) + payment_id = ack.payment_id - return WalletPaymentInitiated( - payment_id=f"{payment_id}", - payment_form_url=None, + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + retry=retry_if_exception_type(PaymentAlreadyExistsError), + reraise=True, + ): + with attempt: + payment_id = await repo_transactions.insert_init_payment_transaction( + ack.payment_id or f"{uuid.uuid4()}", + price_dollars=amount_dollars, + osparc_credits=target_credits, + product_name=product_name, + user_id=user_id, + user_email=user_email, + wallet_id=wallet_id, + comment=comment, + initiated_at=initiated_at, + ) + + assert payment_id is not None # nosec + + transaction = await repo_transactions.update_ack_payment_transaction( + payment_id=payment_id, + completion_state=( + PaymentTransactionState.SUCCESS + if ack.success + else PaymentTransactionState.FAILED + ), + state_message=ack.message, + invoice_url=ack.invoice_url, ) + return transaction.to_api_model() + async def get_payments_page( repo: PaymentsTransactionsRepo, diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index 90d60dfe25d..c9d4174e054 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -5,19 +5,28 @@ """ - +import contextlib +import functools import logging +from collections.abc import Callable +from contextlib import suppress import httpx from fastapi import FastAPI from fastapi.encoders import jsonable_encoder -from httpx import URL +from httpx import URL, HTTPStatusError from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID +from pydantic import ValidationError +from pydantic.errors import PydanticErrorMixin from servicelib.fastapi.http_client import AppStateMixin, BaseHttpApi +from simcore_service_payments.models.schemas.acknowledgements import ( + AckPaymentWithPaymentMethod, +) from ..core.settings import ApplicationSettings from ..models.payments_gateway import ( BatchGetPaymentMethods, + ErrorModel, GetPaymentMethod, InitPayment, InitPaymentMethod, @@ -30,6 +39,43 @@ _logger = logging.getLogger(__name__) +class PaymentsGatewayError(PydanticErrorMixin, ValueError): + msg_template = "Payment-gateway got {status_code} for {operation_id}: {reason}" + + +def _parse_raw_or_none(text: str | None): + if text: + with suppress(ValidationError): + return ErrorModel.parse_raw(text) + return None + + +@contextlib.contextmanager +def _raise_if_error(operation_id: str): + try: + + yield + + except HTTPStatusError as err: + model = _parse_raw_or_none(err.response.text) + + raise PaymentsGatewayError( + operation_id=f"PaymentsGatewayApi.{operation_id}", + reason=model.message if model else f"{err}", + http_status_error=err, + model=model, + ) from err + + +def _handle_status_errors(coro: Callable): + @functools.wraps(coro) + async def _wrapper(self, *args, **kwargs): + with _raise_if_error(operation_id=coro.__name__): + return await coro(self, *args, **kwargs) + + return _wrapper + + class _GatewayApiAuth(httpx.Auth): def __init__(self, secret): self.token = secret @@ -46,6 +92,7 @@ class PaymentsGatewayApi(BaseHttpApi, AppStateMixin): # api: one-time-payment workflow # + @_handle_status_errors async def init_payment(self, payment: InitPayment) -> PaymentInitiated: response = await self.client.post( "/init", @@ -57,6 +104,7 @@ async def init_payment(self, payment: InitPayment) -> PaymentInitiated: def get_form_payment_url(self, id_: PaymentID) -> URL: return self.client.base_url.copy_with(path="/pay", params={"id": f"{id_}"}) + @_handle_status_errors async def cancel_payment( self, payment_initiated: PaymentInitiated ) -> PaymentCancelled: @@ -71,6 +119,7 @@ async def cancel_payment( # api: payment method workflows # + @_handle_status_errors async def init_payment_method( self, payment_method: InitPaymentMethod, @@ -89,6 +138,7 @@ def get_form_payment_method_url(self, id_: PaymentMethodID) -> URL: # CRUD + @_handle_status_errors async def get_many_payment_methods( self, ids_: list[PaymentMethodID] ) -> list[GetPaymentMethod]: @@ -99,24 +149,27 @@ async def get_many_payment_methods( response.raise_for_status() return PaymentMethodsBatch.parse_obj(response.json()).items + @_handle_status_errors async def get_payment_method(self, id_: PaymentMethodID) -> GetPaymentMethod: response = await self.client.get(f"/payment-methods/{id_}") response.raise_for_status() return GetPaymentMethod.parse_obj(response.json()) + @_handle_status_errors async def delete_payment_method(self, id_: PaymentMethodID) -> None: response = await self.client.delete(f"/payment-methods/{id_}") response.raise_for_status() - async def init_payment_with_payment_method( + @_handle_status_errors + async def pay_with_payment_method( self, id_: PaymentMethodID, payment: InitPayment - ) -> PaymentInitiated: + ) -> AckPaymentWithPaymentMethod: response = await self.client.post( f"/payment-methods/{id_}:pay", json=jsonable_encoder(payment), ) response.raise_for_status() - return PaymentInitiated.parse_obj(response.json()) + return AckPaymentWithPaymentMethod.parse_obj(response.json()) def setup_payments_gateway(app: FastAPI): diff --git a/services/payments/src/simcore_service_payments/services/postgres.py b/services/payments/src/simcore_service_payments/services/postgres.py index 59c7d0f950e..59d1c6f18c4 100644 --- a/services/payments/src/simcore_service_payments/services/postgres.py +++ b/services/payments/src/simcore_service_payments/services/postgres.py @@ -5,6 +5,12 @@ from ..core.settings import ApplicationSettings +def get_engine(app: FastAPI) -> AsyncEngine: + assert app.state.engine # nosec + engine: AsyncEngine = app.state.engine + return engine + + def setup_postgres(app: FastAPI): app.state.engine = None diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index b46bc3a4673..086f6052d98 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -37,9 +37,12 @@ PaymentMethodInitiated, PaymentMethodsBatch, ) +from simcore_service_payments.models.schemas.acknowledgements import ( + AckPaymentWithPaymentMethod, +) -@pytest.fixture +@pytest.fixture(scope="session") def is_pdb_enabled(request: pytest.FixtureRequest): """Returns true if tests are set to use interactive debugger, i.e. --pdb""" options = request.config.option @@ -126,15 +129,22 @@ def payments_clean_db(postgres_db: sa.engine.Engine) -> Iterator[None]: con.execute(payments_transactions.delete()) +# +MAX_TIME_FOR_APP_TO_STARTUP = 10 +MAX_TIME_FOR_APP_TO_SHUTDOWN = 10 + + @pytest.fixture -async def app(app_environment: EnvVarsDict) -> AsyncIterator[FastAPI]: - test_app = create_app() +async def app( + app_environment: EnvVarsDict, is_pdb_enabled: bool +) -> AsyncIterator[FastAPI]: + the_test_app = create_app() async with LifespanManager( - test_app, - startup_timeout=None, # for debugging - shutdown_timeout=10, + the_test_app, + startup_timeout=None if is_pdb_enabled else MAX_TIME_FOR_APP_TO_STARTUP, + shutdown_timeout=None if is_pdb_enabled else MAX_TIME_FOR_APP_TO_SHUTDOWN, ): - yield test_app + yield the_test_app # @@ -205,7 +215,7 @@ def _init(request: httpx.Request): pm_id = faker.uuid4() _payment_methods[pm_id] = PaymentMethodInfoTuple( init=InitPaymentMethod.parse_raw(request.content), - get=GetPaymentMethod(**random_payment_method_data(idr=pm_id)), + get=GetPaymentMethod(**random_payment_method_data(id=pm_id)), ) return httpx.Response( @@ -247,16 +257,25 @@ def _batch_get(request: httpx.Request): json=jsonable_encoder(PaymentMethodsBatch(items=items)), ) - def _init_payment(request: httpx.Request, pm_id: PaymentMethodID): + def _pay(request: httpx.Request, pm_id: PaymentMethodID): assert "*" not in request.headers["X-Init-Api-Secret"] assert InitPayment.parse_raw(request.content) is not None # checks _get(request, pm_id) + payment_id = faker.uuid4() return httpx.Response( status.HTTP_200_OK, - json=jsonable_encoder(PaymentInitiated(payment_id=faker.uuid4())), + json=jsonable_encoder( + AckPaymentWithPaymentMethod( + success=True, + message=f"Payment '{payment_id}' with payment-method '{pm_id}'", + invoice_url=faker.url(), + provider_payment_id="pi_123456ABCDEFG123456ABCDE", + payment_id=payment_id, + ) + ), ) # ------ @@ -283,8 +302,8 @@ def _init_payment(request: httpx.Request, pm_id: PaymentMethodID): mock_router.post( path__regex=r"/payment-methods/(?P[\w-]+):pay$", - name="init_payment_with_payment_method", - ).mock(side_effect=_init_payment) + name="pay_with_payment_method", + ).mock(side_effect=_pay) yield _mock @@ -305,7 +324,7 @@ def pytest_addoption(parser: pytest.Parser): ) -@pytest.fixture +@pytest.fixture(scope="session") def external_environment(request: pytest.FixtureRequest) -> EnvVarsDict: """ If a file under test folder prefixed with `.env-secret` is present, @@ -332,7 +351,6 @@ def mock_payments_gateway_service_or_none( mock_payments_methods_routes: Callable, external_environment: EnvVarsDict, ) -> MockRouter | None: - # EITHER tests against external payments-gateway if payments_gateway_url := external_environment.get("PAYMENTS_GATEWAY_URL"): print("🚨 EXTERNAL: these tests are running against", f"{payments_gateway_url=}") diff --git a/services/payments/tests/unit/test_rpc_payments_methods.py b/services/payments/tests/unit/test_rpc_payments_methods.py index 15ce2f05724..7cce4ae6f95 100644 --- a/services/payments/tests/unit/test_rpc_payments_methods.py +++ b/services/payments/tests/unit/test_rpc_payments_methods.py @@ -10,7 +10,7 @@ from fastapi import FastAPI from models_library.api_schemas_webserver.wallets import ( PaymentMethodInitiated, - WalletPaymentInitiated, + PaymentTransaction, ) from models_library.basic_types import IDStr from models_library.products import ProductName @@ -222,9 +222,9 @@ async def test_webserver_pay_with_payment_method_workflow( ack=AckPaymentMethod(success=True, message="Faked ACK"), ) - payment_inited = await rpc_client.request( + transaction = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_payment_with_payment_method"), + parse_obj_as(RPCMethodName, "pay_with_payment_method"), payment_method_id=created.payment_method_id, amount_dollars=faker.pyint(), target_credits=faker.pyint(), @@ -237,14 +237,14 @@ async def test_webserver_pay_with_payment_method_workflow( comment="Payment with stored credit-card", ) - assert isinstance(payment_inited, WalletPaymentInitiated) - assert payment_inited.payment_id - assert payment_inited.payment_form_url is None + assert isinstance(transaction, PaymentTransaction) + assert transaction.payment_id + assert transaction.state == "SUCCESS" payment = await PaymentsTransactionsRepo(app.state.engine).get_payment_transaction( - payment_inited.payment_id, user_id=user_id, wallet_id=wallet_id + transaction.payment_id, user_id=user_id, wallet_id=wallet_id ) assert payment is not None - assert payment.payment_id == payment_inited.payment_id - assert payment.state == PaymentTransactionState.PENDING + assert payment.payment_id == transaction.payment_id + assert payment.state == PaymentTransactionState.SUCCESS assert payment.comment == "Payment with stored credit-card" diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index f29008257f9..8a0b34f9350 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -4,7 +4,6 @@ # pylint: disable=too-many-arguments -import httpx import pytest from faker import Faker from fastapi import FastAPI, status @@ -17,6 +16,7 @@ ) from simcore_service_payments.services.payments_gateway import ( PaymentsGatewayApi, + PaymentsGatewayError, setup_payments_gateway, ) @@ -92,7 +92,6 @@ async def test_one_time_payment_workflow( mock_payments_gateway_service_or_none: MockRouter | None, amount_dollars: float, ): - payment_gateway_api = PaymentsGatewayApi.get_from_app_state(app) assert payment_gateway_api @@ -125,13 +124,13 @@ async def test_one_time_payment_workflow( assert mock_payments_gateway_service_or_none.routes["cancel_payment"].called +@pytest.mark.can_run_against_external() async def test_payment_methods_workflow( app: FastAPI, faker: Faker, mock_payments_gateway_service_or_none: MockRouter | None, amount_dollars: float, ): - payments_gateway_api: PaymentsGatewayApi = PaymentsGatewayApi.get_from_app_state( app ) @@ -161,7 +160,7 @@ async def test_payment_methods_workflow( got_payment_method = await payments_gateway_api.get_payment_method( payment_method_id ) - assert got_payment_method.idr == payment_method_id + assert got_payment_method.id == payment_method_id print(got_payment_method.json(indent=2)) # list payment-methods @@ -171,8 +170,7 @@ async def test_payment_methods_workflow( assert len(items) == 1 assert items[0] == got_payment_method - # init payments with payment-method (needs to be ACK to complete) - payment_initiated = await payments_gateway_api.init_payment_with_payment_method( + payment_with_payment_method = await payments_gateway_api.pay_with_payment_method( id_=payment_method_id, payment=InitPayment( amount_dollars=amount_dollars, @@ -182,23 +180,20 @@ async def test_payment_methods_workflow( wallet_name=faker.word(), ), ) - - # cancel payment - payment_canceled = await payments_gateway_api.cancel_payment(payment_initiated) - assert payment_canceled is not None + assert payment_with_payment_method.success # delete payment-method await payments_gateway_api.delete_payment_method(payment_method_id) - with pytest.raises(httpx.HTTPStatusError) as err_info: + with pytest.raises(PaymentsGatewayError) as err_info: await payments_gateway_api.get_payment_method(payment_method_id) - http_status_error = err_info.value + assert err_info.value.operation_id == "PaymentsGatewayApi.get_payment_method" + + http_status_error = err_info.value.http_status_error assert http_status_error.response.status_code == status.HTTP_404_NOT_FOUND if mock_payments_gateway_service_or_none: - assert mock_payments_gateway_service_or_none.routes["cancel_payment"].called - # all defined payment-methods for route in mock_payments_gateway_service_or_none.routes: if route.name and "payment_method" in route.name: diff --git a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js index 80fbd659e38..49dbab989ed 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js +++ b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js @@ -50,10 +50,7 @@ qx.Class.define("osparc.desktop.paymentMethods.PaymentMethodDetails", { [this.tr("Holder name"), paymentMethodData["cardHolderName"]], [this.tr("Type"), paymentMethodData["cardType"]], [this.tr("Number"), paymentMethodData["cardNumberMasked"]], - [this.tr("Expiration date"), paymentMethodData["expirationMonth"] + "/" + paymentMethodData["expirationYear"]], - [this.tr("Address"), paymentMethodData["streetAddress"]], - [this.tr("ZIP code"), paymentMethodData["zipcode"]], - [this.tr("Country"), paymentMethodData["country"]] + [this.tr("Expiration date"), paymentMethodData["expirationMonth"] + "/" + paymentMethodData["expirationYear"]] ].forEach((pair, idx) => { this._add(new qx.ui.basic.Label(pair[0]).set({ font: "text-14" diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index d84d9c55f10..15d76e2571f 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -4312,8 +4312,8 @@ paths: post: tags: - wallets - summary: Init Payment With Payment Method - operationId: init_payment_with_payment_method + summary: Pay With Payment Method + operationId: pay_with_payment_method parameters: - required: true schema: @@ -4339,7 +4339,7 @@ paths: required: true responses: '202': - description: Payment initialized + description: Pay with payment-method content: application/json: schema: @@ -5256,7 +5256,11 @@ components: properties: priceDollars: title: Pricedollars + exclusiveMaximum: true + exclusiveMinimum: true type: number + maximum: 1000000.0 + minimum: 0.0 comment: title: Comment maxLength: 100 @@ -7545,14 +7549,6 @@ components: required: - idr - walletId - - cardHolderName - - cardNumberMasked - - cardType - - expirationMonth - - expirationYear - - streetAddress - - zipcode - - country - created type: object properties: @@ -7581,15 +7577,6 @@ components: expirationYear: title: Expirationyear type: integer - streetAddress: - title: Streetaddress - type: string - zipcode: - title: Zipcode - type: string - country: - title: Country - type: string created: title: Created type: string diff --git a/services/web/server/src/simcore_service_webserver/payments/_methods_api.py b/services/web/server/src/simcore_service_webserver/payments/_methods_api.py index 354f87ae67b..c4b2b56309c 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_methods_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_methods_api.py @@ -40,15 +40,12 @@ def _generate_fake_data(fake: Faker): return { - "idr": fake.uuid4(), + "id": fake.uuid4(), "card_holder_name": fake.name(), "card_number_masked": f"**** **** **** {fake.credit_card_number()[:4]}", "card_type": fake.credit_card_provider(), "expiration_month": fake.random_int(min=1, max=12), "expiration_year": fake.future_date().year, - "street_address": fake.street_address(), - "zipcode": fake.zipcode(), - "country": fake.country(), } diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index 9a49028cbe8..00ce812f7fa 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -135,7 +135,6 @@ async def init_creation_of_wallet_payment( user_id: UserID, wallet_id: WalletID, comment: str | None, - payment_method_id: PaymentMethodID | None = None, ) -> WalletPaymentInitiated: """ @@ -157,38 +156,22 @@ async def init_creation_of_wallet_payment( user = await get_user_name_and_email(app, user_id=user_id) settings: PaymentsSettings = get_plugin_settings(app) payment_inited: WalletPaymentInitiated - if payment_method_id is None: - if settings.PAYMENTS_FAKE_COMPLETION: - payment_inited = await _fake_init_payment( - app, - price_dollars, - osparc_credits, - product_name, - wallet_id, - user_id, - user.email, - comment, - ) - else: - # call to payment-service - assert not settings.PAYMENTS_FAKE_COMPLETION # nosec - payment_inited = await _rpc.init_payment( - app, - amount_dollars=price_dollars, - target_credits=osparc_credits, - product_name=product_name, - wallet_id=wallet_id, - wallet_name=user_wallet.name, - user_id=user_id, - user_name=user.name, - user_email=user.email, - comment=comment, - ) + if settings.PAYMENTS_FAKE_COMPLETION: + payment_inited = await _fake_init_payment( + app, + price_dollars, + osparc_credits, + product_name, + wallet_id, + user_id, + user.email, + comment, + ) else: - assert payment_method_id is not None # nosec - payment_inited = await _rpc.init_payment_with_payment_method( + # call to payment-service + assert not settings.PAYMENTS_FAKE_COMPLETION # nosec + payment_inited = await _rpc.init_payment( app, - payment_method_id=payment_method_id, amount_dollars=price_dollars, target_credits=osparc_credits, product_name=product_name, @@ -210,6 +193,7 @@ async def _ack_creation_of_wallet_payment( completion_state: PaymentTransactionState, message: str | None = None, invoice_url: HttpUrl | None = None, + notify_enabled: bool = True, ) -> PaymentTransaction: # # NOTE: implements endpoint in payment service hit by the gateway @@ -231,7 +215,10 @@ async def _ack_creation_of_wallet_payment( payment = _to_api_model(transaction) # notifying front-end via web-sockets - await notify_payment_completed(app, user_id=transaction.user_id, payment=payment) + if notify_enabled: + await notify_payment_completed( + app, user_id=transaction.user_id, payment=payment + ) if completion_state == PaymentTransactionState.SUCCESS: # notifying RUT @@ -285,6 +272,100 @@ async def cancel_payment_to_wallet( ) +async def _fake_pay_with_payment_method( # noqa: PLR0913 pylint: disable=too-many-arguments + app, + amount_dollars, + target_credits, + product_name, + wallet_id, + wallet_name, + user_id, + user_name, + user_email, + payment_method_id: PaymentMethodID, + comment, +) -> PaymentTransaction: + + assert user_name # nosec + assert wallet_name # nosec + + inited = await _fake_init_payment( + app, + amount_dollars, + target_credits, + product_name, + wallet_id, + user_id, + user_email, + comment, + ) + return await _ack_creation_of_wallet_payment( + app, + payment_id=inited.payment_id, + completion_state=PaymentTransactionState.SUCCESS, + message=f"Fake payment completed with payment-id = {payment_method_id}", + invoice_url=f"https://fake-invoice.com/?id={inited.payment_id}", # type: ignore + notify_enabled=False, + ) + + +async def pay_with_payment_method( + app: web.Application, + *, + price_dollars: Decimal, + osparc_credits: Decimal, + product_name: str, + user_id: UserID, + wallet_id: WalletID, + payment_method_id: PaymentMethodID, + comment: str | None, +) -> PaymentTransaction: + + # wallet: check permissions + await raise_for_wallet_payments_permissions( + app, user_id=user_id, wallet_id=wallet_id, product_name=product_name + ) + user_wallet = await get_wallet_by_user( + app, user_id=user_id, wallet_id=wallet_id, product_name=product_name + ) + assert user_wallet.wallet_id == wallet_id # nosec + + # user info + user = await get_user_name_and_email(app, user_id=user_id) + + settings: PaymentsSettings = get_plugin_settings(app) + if settings.PAYMENTS_FAKE_COMPLETION: + return await _fake_pay_with_payment_method( + app, + payment_method_id=payment_method_id, + amount_dollars=price_dollars, + target_credits=osparc_credits, + product_name=product_name, + wallet_id=wallet_id, + wallet_name=user_wallet.name, + user_id=user_id, + user_name=user.name, + user_email=user.email, + comment=comment, + ) + + assert not settings.PAYMENTS_FAKE_COMPLETION # nosec + + return await _rpc.pay_with_payment_method( + app, + payment_method_id=payment_method_id, + amount_dollars=price_dollars, + target_credits=osparc_credits, + product_name=product_name, + wallet_id=wallet_id, + wallet_name=user_wallet.name, + user_id=user_id, + user_name=user.name, + user_email=user.email, + comment=comment, + ) + + async def list_user_payments_page( app: web.Application, product_name: str, diff --git a/services/web/server/src/simcore_service_webserver/payments/_rpc.py b/services/web/server/src/simcore_service_webserver/payments/_rpc.py index bd4a2ec74b9..e5c0d2680a3 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_rpc.py +++ b/services/web/server/src/simcore_service_webserver/payments/_rpc.py @@ -12,6 +12,7 @@ PaymentMethodGet, PaymentMethodID, PaymentMethodInitiated, + PaymentTransaction, WalletPaymentInitiated, ) from models_library.basic_types import IDStr @@ -211,7 +212,7 @@ async def delete_payment_method( @log_decorator(_logger, level=logging.DEBUG) -async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-arguments +async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-arguments app: web.Application, *, payment_method_id: PaymentMethodID, @@ -224,12 +225,13 @@ async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=t user_name: str, user_email: EmailStr, comment: str | None = None, -) -> WalletPaymentInitiated: +) -> PaymentTransaction: + rpc_client = app[_APP_PAYMENTS_RPC_CLIENT_KEY] result = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_payment_with_payment_method"), + parse_obj_as(RPCMethodName, "pay_with_payment_method"), payment_method_id=payment_method_id, amount_dollars=amount_dollars, target_credits=target_credits, @@ -241,5 +243,6 @@ async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=t user_email=user_email, comment=comment, ) - assert isinstance(result, WalletPaymentInitiated) # nosec + + assert isinstance(result, PaymentTransaction) # nosec return result diff --git a/services/web/server/src/simcore_service_webserver/payments/api.py b/services/web/server/src/simcore_service_webserver/payments/api.py index 4dbdd9d4a0b..171ebd5e574 100644 --- a/services/web/server/src/simcore_service_webserver/payments/api.py +++ b/services/web/server/src/simcore_service_webserver/payments/api.py @@ -13,18 +13,22 @@ cancel_payment_to_wallet, init_creation_of_wallet_payment, list_user_payments_page, + pay_with_payment_method, ) +from ._socketio import notify_payment_completed __all__: tuple[str, ...] = ( "cancel_creation_of_wallet_payment_method", "cancel_payment_to_wallet", - "init_creation_of_wallet_payment", "delete_wallet_payment_method", - "list_user_payments_page", "get_wallet_payment_autorecharge", "get_wallet_payment_method", "init_creation_of_wallet_payment_method", + "init_creation_of_wallet_payment", + "list_user_payments_page", "list_wallet_payment_methods", + "notify_payment_completed", + "pay_with_payment_method", "replace_wallet_payment_autorecharge", ) # nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 38a327a3a78..c4cac0680f2 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -11,8 +11,11 @@ ReplaceWalletAutoRecharge, WalletPaymentInitiated, ) +from models_library.basic_types import AmountDecimal, NonNegativeDecimal from models_library.rest_pagination import Page, PageQueryParameters from models_library.rest_pagination_utils import paginate_data +from pydantic import parse_obj_as +from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from servicelib.aiohttp.requests_validation import ( parse_request_body_as, parse_request_path_parameters_as, @@ -20,6 +23,7 @@ ) from servicelib.logging_utils import get_log_record_extra, log_context from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON +from servicelib.utils import fire_and_forget_task from .._meta import API_VTAG as VTAG from ..login.decorators import login_required @@ -33,6 +37,8 @@ init_creation_of_wallet_payment_method, list_user_payments_page, list_wallet_payment_methods, + notify_payment_completed, + pay_with_payment_method, replace_wallet_payment_autorecharge, ) from ..products.api import get_current_product_credit_price @@ -48,30 +54,16 @@ _logger = logging.getLogger(__name__) -async def _init_creation_of_payments( - request: web.Request, - user_id, - product_name, - wallet_id, - payment_method_id, - init: CreateWalletPayment, -) -> WalletPaymentInitiated: +async def _eval_total_credits_or_raise( + request: web.Request, amount_dollars: AmountDecimal +) -> NonNegativeDecimal: # Conversion usd_per_credit = await get_current_product_credit_price(request) if not usd_per_credit: # '0 or None' should raise raise web.HTTPConflict(reason=MSG_PRICE_NOT_DEFINED_ERROR) - return await init_creation_of_wallet_payment( - request.app, - user_id=user_id, - product_name=product_name, - wallet_id=wallet_id, - osparc_credits=init.price_dollars / usd_per_credit, - comment=init.comment, - price_dollars=init.price_dollars, - payment_method_id=payment_method_id, - ) + return parse_obj_as(NonNegativeDecimal, amount_dollars / usd_per_credit) routes = web.RouteTableDef() @@ -100,13 +92,18 @@ async def _create_payment(request: web.Request): extra=get_log_record_extra(user_id=req_ctx.user_id), ): - payment: WalletPaymentInitiated = await _init_creation_of_payments( - request, + osparc_credits = await _eval_total_credits_or_raise( + request, body_params.price_dollars + ) + + payment: WalletPaymentInitiated = await init_creation_of_wallet_payment( + request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name, wallet_id=wallet_id, - payment_method_id=None, - init=body_params, + osparc_credits=osparc_credits, + comment=body_params.comment, + price_dollars=body_params.price_dollars, ) return envelope_json_response(payment, web.HTTPCreated) @@ -317,15 +314,12 @@ async def _delete_payment_method(request: web.Request): @routes.post( f"/{VTAG}/wallets/{{wallet_id}}/payments-methods/{{payment_method_id}}:pay", - name="init_payment_with_payment_method", + name="pay_with_payment_method", ) @login_required @permission_required("wallets.*") @handle_wallets_exceptions -async def _init_payment_with_payment_method(request: web.Request): - """Triggers the creation of a new payment method. - Note that creating a payment-method follows the init-prompt-ack flow - """ +async def _pay_with_payment_method(request: web.Request): req_ctx = WalletsRequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(PaymentMethodsPathParams, request) body_params = await parse_request_body_as(CreateWalletPayment, request) @@ -341,16 +335,40 @@ async def _init_payment_with_payment_method(request: web.Request): extra=get_log_record_extra(user_id=req_ctx.user_id), ): - payment: WalletPaymentInitiated = await _init_creation_of_payments( - request, + osparc_credits = await _eval_total_credits_or_raise( + request, body_params.price_dollars + ) + + payment: PaymentTransaction = await pay_with_payment_method( + request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name, wallet_id=wallet_id, payment_method_id=path_params.payment_method_id, - init=body_params, + osparc_credits=osparc_credits, + comment=body_params.comment, + price_dollars=body_params.price_dollars, ) - return envelope_json_response(payment, web.HTTPAccepted) + # NOTE: Due to the design change in https://github.com/ITISFoundation/osparc-simcore/pull/5017 + # we decided not to change the return value to avoid changing the front-end logic + # instead we emulate a init-prompt-ack workflow by firing a background task that acks payment + + fire_and_forget_task( + notify_payment_completed( + request.app, user_id=req_ctx.user_id, payment=payment + ), + task_suffix_name=f"{__name__}._pay_with_payment_method", + fire_and_forget_tasks_collection=request.app[APP_FIRE_AND_FORGET_TASKS_KEY], + ) + + return envelope_json_response( + WalletPaymentInitiated( + payment_id=payment.payment_id, + payment_form_url=None, + ), + web.HTTPAccepted, + ) # diff --git a/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py b/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py index e94b6570b62..51df97d8567 100644 --- a/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py @@ -20,6 +20,7 @@ PaymentMethodGet, PaymentMethodID, PaymentMethodInitiated, + PaymentTransaction, WalletGet, ) from models_library.basic_types import IDStr @@ -41,6 +42,7 @@ from simcore_service_webserver.payments._onetime_api import ( _fake_cancel_payment, _fake_init_payment, + _fake_pay_with_payment_method, ) from simcore_service_webserver.payments.settings import ( PaymentsSettings, @@ -200,24 +202,27 @@ async def _pay( user_name: str, user_email: EmailStr, comment: str | None = None, - ): + ) -> PaymentTransaction: + assert await _get( app, payment_method_id=payment_method_id, user_id=user_id, wallet_id=wallet_id, ) - return await _init( + + return await _fake_pay_with_payment_method( app, - amount_dollars=amount_dollars, - target_credits=target_credits, - product_name=product_name, - wallet_id=wallet_id, - wallet_name=wallet_name, - user_id=user_id, - user_name=user_name, - user_email=user_email, - comment=comment, + amount_dollars, + target_credits, + product_name, + wallet_id, + wallet_name, + user_id, + user_name, + user_email, + payment_method_id, + comment, ) return { @@ -256,8 +261,8 @@ async def _pay( autospec=True, side_effect=_del, ), - "init_payment_with_payment_method": mocker.patch( - "simcore_service_webserver.payments._onetime_api._rpc.init_payment_with_payment_method", + "pay_with_payment_method": mocker.patch( + "simcore_service_webserver.payments._onetime_api._rpc.pay_with_payment_method", autospec=True, side_effect=_pay, ), diff --git a/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py b/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py index 9b1fe418560..c18ce0973dc 100644 --- a/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py +++ b/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py @@ -26,15 +26,9 @@ from pytest_mock import MockerFixture from pytest_simcore.helpers.utils_assert import assert_status from simcore_postgres_database.models.payments_methods import InitPromptAckFlowState -from simcore_postgres_database.models.payments_transactions import ( - PaymentTransactionState, -) from simcore_service_webserver.payments._methods_api import ( _ack_creation_of_wallet_payment_method, ) -from simcore_service_webserver.payments._onetime_api import ( - _ack_creation_of_wallet_payment, -) from simcore_service_webserver.payments.settings import PaymentsSettings from simcore_service_webserver.payments.settings import ( get_plugin_settings as get_payments_plugin_settings, @@ -354,7 +348,7 @@ async def test_one_time_payment_with_payment_method( ) assert ( - client.app.router["init_payment_with_payment_method"] + client.app.router["pay_with_payment_method"] .url_for( wallet_id=f"{logged_user_wallet.wallet_id}", payment_method_id=wallet_payment_method_id, @@ -373,21 +367,11 @@ async def test_one_time_payment_with_payment_method( data, error = await assert_status(response, web.HTTPAccepted) assert error is None payment = WalletPaymentInitiated.parse_obj(data) - assert mock_rpc_payments_service_api["init_payment_with_payment_method"].called + assert mock_rpc_payments_service_api["pay_with_payment_method"].called assert payment.payment_id - assert payment.payment_form_url - assert payment.payment_form_url.host == "some-fake-gateway.com" - assert payment.payment_form_url.query - assert payment.payment_form_url.query.endswith(payment.payment_id) + assert payment.payment_form_url is None - # Complete - await _ack_creation_of_wallet_payment( - client.app, - payment_id=payment.payment_id, - completion_state=PaymentTransactionState.SUCCESS, - invoice_url=faker.url(), - ) # check notification to RUT (fake) assert mock_rut_add_credits_to_wallet.called mock_rut_add_credits_to_wallet.assert_called_once() From 163d47fae0141cffa69738f8b9311894eff12fbb Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Thu, 16 Nov 2023 11:44:44 +0100 Subject: [PATCH 12/24] =?UTF-8?q?=E2=9C=A8=20Comp=20backend:=20allow=20mul?= =?UTF-8?q?tiple=20amis/bootscripts/pre-pull=20images=20(#5033)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env-devel | 6 + .pylintrc | 2 +- packages/aws-library/requirements/_base.in | 1 + packages/aws-library/requirements/_base.txt | 2 + packages/aws-library/requirements/ci.txt | 1 + packages/aws-library/requirements/dev.txt | 1 + .../aws-library/src/aws_library/ec2/models.py | 102 +++++++++++++- .../src/settings_library/ec2.py | 2 +- services/autoscaling/README.md | 19 ++- services/autoscaling/requirements/_base.txt | 79 +++++++++++ .../simcore_service_autoscaling/api/health.py | 4 +- .../core/settings.py | 64 +++++---- .../modules/auto_scaling_core.py | 20 ++- .../utils/auto_scaling_core.py | 20 +-- services/autoscaling/tests/manual/.env-devel | 14 +- services/autoscaling/tests/manual/README.md | 3 +- services/autoscaling/tests/unit/conftest.py | 74 ++++++---- .../tests/unit/test_core_settings.py | 107 ++++++++++---- .../unit/test_modules_auto_scaling_dynamic.py | 18 +-- .../unit/test_modules_auto_scaling_task.py | 3 +- .../tests/unit/test_modules_rabbitmq.py | 17 +-- .../unit/test_utils_auto_scaling_core.py | 130 +++++++++++++----- .../tests/unit/test_utils_docker.py | 4 +- .../tests/unit/test_utils_rabbitmq.py | 15 +- .../core/settings.py | 52 ++++--- .../data/docker-compose.yml | 14 +- .../utils/clusters.py | 14 +- .../clusters-keeper/tests/manual/README.md | 7 +- .../clusters-keeper/tests/unit/conftest.py | 48 ++----- .../tests/unit/test_utils_clusters.py | 11 +- services/docker-compose.yml | 40 +++++- 31 files changed, 626 insertions(+), 268 deletions(-) diff --git a/.env-devel b/.env-devel index f2e27bf1c32..a2fe30524ce 100644 --- a/.env-devel +++ b/.env-devel @@ -18,6 +18,12 @@ AGENT_VOLUMES_CLEANUP_S3_PROVIDER=MINIO API_SERVER_DEV_FEATURES_ENABLED=0 +AUTOSCALING_DASK=null +AUTOSCALING_EC2_ACCESS=null +AUTOSCALING_EC2_INSTANCES=null +AUTOSCALING_NODES_MONITORING=null +AUTOSCALING_POLL_INTERVAL=10 + BF_API_KEY=none BF_API_SECRET=none diff --git a/.pylintrc b/.pylintrc index bae6b832f9a..f3f789e9c63 100644 --- a/.pylintrc +++ b/.pylintrc @@ -243,7 +243,7 @@ contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members= +generated-members=sh # Tells whether to warn about missing members when the owner of the attribute # is inferred to be None. diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in index a0fd39eb41f..bff24c55562 100644 --- a/packages/aws-library/requirements/_base.in +++ b/packages/aws-library/requirements/_base.in @@ -10,3 +10,4 @@ aioboto3 aiocache pydantic[email] types-aiobotocore[ec2] +sh diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index 2e125c14ad8..9267ce51d19 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -154,6 +154,8 @@ rpds-py==0.12.0 # referencing s3transfer==0.7.0 # via boto3 +sh==2.0.6 + # via -r requirements/_base.in six==1.16.0 # via python-dateutil tenacity==8.2.3 diff --git a/packages/aws-library/requirements/ci.txt b/packages/aws-library/requirements/ci.txt index 604f51d486e..0672404f4ca 100644 --- a/packages/aws-library/requirements/ci.txt +++ b/packages/aws-library/requirements/ci.txt @@ -13,6 +13,7 @@ # installs this repo's packages ../pytest-simcore/ +../models-library/ ../service-library/ ../settings-library/ diff --git a/packages/aws-library/requirements/dev.txt b/packages/aws-library/requirements/dev.txt index b183acca92c..9017cb30880 100644 --- a/packages/aws-library/requirements/dev.txt +++ b/packages/aws-library/requirements/dev.txt @@ -13,6 +13,7 @@ # installs this repo's packages --editable ../pytest-simcore/ +--editable ../models-library/ --editable ../service-library/ --editable ../settings-library/ diff --git a/packages/aws-library/src/aws_library/ec2/models.py b/packages/aws-library/src/aws_library/ec2/models.py index d124763cebd..21b0f89b513 100644 --- a/packages/aws-library/src/aws_library/ec2/models.py +++ b/packages/aws-library/src/aws_library/ec2/models.py @@ -1,8 +1,19 @@ import datetime +import tempfile from dataclasses import dataclass -from typing import TypeAlias - -from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt +from typing import Any, ClassVar, TypeAlias + +import sh +from models_library.docker import DockerGenericTag +from pydantic import ( + BaseModel, + ByteSize, + Extra, + Field, + NonNegativeFloat, + PositiveInt, + validator, +) from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType @@ -74,3 +85,88 @@ class EC2InstanceConfig: key_name: str security_group_ids: list[str] subnet_id: str + + +AMIIdStr: TypeAlias = str +CommandStr: TypeAlias = str + + +class EC2InstanceBootSpecific(BaseModel): + ami_id: AMIIdStr + custom_boot_scripts: list[CommandStr] = Field( + default_factory=list, + description="script(s) to run on EC2 instance startup (be careful!), " + "each entry is run one after the other using '&&' operator", + ) + pre_pull_images: list[DockerGenericTag] = Field( + default_factory=list, + description="a list of docker image/tags to pull on instance cold start", + ) + pre_pull_images_cron_interval: datetime.timedelta = Field( + default=datetime.timedelta(minutes=30), + description="time interval between pulls of images (minimum is 1 minute) " + "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)", + ) + + class Config: + extra = Extra.forbid + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + { + # just AMI + "ami_id": "ami-123456789abcdef", + }, + { + # AMI + scripts + "ami_id": "ami-123456789abcdef", + "custom_boot_scripts": ["ls -tlah", "echo blahblah"], + }, + { + # AMI + scripts + pre-pull + "ami_id": "ami-123456789abcdef", + "custom_boot_scripts": ["ls -tlah", "echo blahblah"], + "pre_pull_images": [ + "nginx:latest", + "itisfoundation/my-very-nice-service:latest", + "simcore/services/dynamic/another-nice-one:2.4.5", + "asd", + ], + }, + { + # AMI + pre-pull + "ami_id": "ami-123456789abcdef", + "pre_pull_images": [ + "nginx:latest", + "itisfoundation/my-very-nice-service:latest", + "simcore/services/dynamic/another-nice-one:2.4.5", + "asd", + ], + }, + { + # AMI + pre-pull + cron + "ami_id": "ami-123456789abcdef", + "pre_pull_images": [ + "nginx:latest", + "itisfoundation/my-very-nice-service:latest", + "simcore/services/dynamic/another-nice-one:2.4.5", + "asd", + ], + "pre_pull_images_cron_interval": "01:00:00", + }, + ] + } + + @validator("custom_boot_scripts") + @classmethod + def validate_bash_calls(cls, v): + try: + with tempfile.NamedTemporaryFile(mode="wt", delete=True) as temp_file: + temp_file.writelines(v) + temp_file.flush() + # NOTE: this will not capture runtime errors, but at least some syntax errors such as invalid quotes + sh.bash("-n", temp_file.name) + except sh.ErrorReturnCode as exc: + msg = f"Invalid bash call in custom_boot_scripts: {v}, Error: {exc.stderr}" + raise ValueError(msg) from exc + + return v diff --git a/packages/settings-library/src/settings_library/ec2.py b/packages/settings-library/src/settings_library/ec2.py index 710db9c6e4f..2cd7cf0b9a6 100644 --- a/packages/settings-library/src/settings_library/ec2.py +++ b/packages/settings-library/src/settings_library/ec2.py @@ -18,7 +18,7 @@ class Config(BaseCustomSettings.Config): "examples": [ { "EC2_ACCESS_KEY_ID": "my_access_key_id", - "EC2_ENDPOINT": "http://my_ec2_endpoint.com", + "EC2_ENDPOINT": "https://my_ec2_endpoint.com", "EC2_REGION_NAME": "us-east-1", "EC2_SECRET_ACCESS_KEY": "my_secret_access_key", } diff --git a/services/autoscaling/README.md b/services/autoscaling/README.md index 70785f9659f..cd2abf6bcb8 100644 --- a/services/autoscaling/README.md +++ b/services/autoscaling/README.md @@ -1,16 +1,13 @@ # autoscaling -[![image-size]](https://microbadger.com/images/itisfoundation/autoscaling. "More on itisfoundation/autoscaling.:staging-latest image") +Service to auto-scale swarm for both dynamic and computational services -[![image-badge]](https://microbadger.com/images/itisfoundation/autoscaling "More on Auto scaling service image in registry") -[![image-version]](https://microbadger.com/images/itisfoundation/autoscaling "More on Auto scaling service image in registry") -[![image-commit]](https://microbadger.com/images/itisfoundation/autoscaling "More on Auto scaling service image in registry") -Service to auto-scale swarm +## development - -[image-size]:https://img.shields.io/microbadger/image-size/itisfoundation/autoscaling./staging-latest.svg?label=autoscaling.&style=flat -[image-badge]:https://images.microbadger.com/badges/image/itisfoundation/autoscaling.svg -[image-version]https://images.microbadger.com/badges/version/itisfoundation/autoscaling.svg -[image-commit]:https://images.microbadger.com/badges/commit/itisfoundation/autoscaling.svg - +``` +make install-dev +make test-dev-unit + +# NOTE: there are manual tests that need access to AWS EC2 instances! +``` diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index bb189297c0e..621da3d81fc 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -7,6 +7,7 @@ aio-pika==9.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aioboto3==12.0.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in @@ -21,18 +22,26 @@ aiocache==0.12.2 aiodebug==2.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiodocker==0.21.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in aiofiles==23.2.1 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiohttp==3.8.6 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -58,6 +67,9 @@ anyio==4.0.0 arrow==1.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in @@ -82,6 +94,11 @@ botocore-stubs==1.31.77 # via types-aiobotocore certifi==2023.7.22 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -125,6 +142,11 @@ exceptiongroup==1.1.3 # via anyio fastapi==0.99.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -153,6 +175,11 @@ httpcore==0.18.0 # via httpx httpx==0.25.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -176,6 +203,11 @@ importlib-metadata==6.8.0 # dask jinja2==3.1.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -194,6 +226,8 @@ jmespath==1.0.1 # botocore jsonschema==4.19.2 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in @@ -222,6 +256,8 @@ multidict==6.0.4 # yarl orjson==3.9.10 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in @@ -243,6 +279,11 @@ psutil==5.9.5 # distributed pydantic==1.10.13 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -254,6 +295,11 @@ pydantic==1.10.13 # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -268,6 +314,7 @@ pygments==2.16.1 pyinstrument==4.6.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.8.2 # via @@ -275,6 +322,11 @@ python-dateutil==2.8.2 # botocore pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -287,11 +339,17 @@ pyyaml==6.0.1 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # dask # distributed redis==5.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -303,15 +361,19 @@ redis==5.0.1 # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/./constraints.txt # -c requirements/../../../packages/service-library/requirements/././constraints.txt # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications rich==13.6.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -321,6 +383,8 @@ rpds-py==0.10.6 # referencing s3transfer==0.7.0 # via boto3 +sh==2.0.6 + # via -r requirements/../../../packages/aws-library/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.0 @@ -334,6 +398,11 @@ sortedcontainers==2.4.0 # distributed starlette==0.27.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -352,11 +421,13 @@ tblib==2.0.0 tenacity==8.2.3 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in toolz==0.12.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # dask # distributed @@ -368,9 +439,12 @@ tornado==6.3.3 tqdm==4.66.1 # via # -c requirements/../../../packages/service-library/requirements/./_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in typer==0.9.0 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -394,6 +468,11 @@ typing-extensions==4.8.0 # uvicorn urllib3==1.26.16 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt diff --git a/services/autoscaling/src/simcore_service_autoscaling/api/health.py b/services/autoscaling/src/simcore_service_autoscaling/api/health.py index 0091d6d877b..07c29f3a198 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/api/health.py +++ b/services/autoscaling/src/simcore_service_autoscaling/api/health.py @@ -5,6 +5,7 @@ """ import datetime +from typing import Annotated from fastapi import APIRouter, Depends, FastAPI from fastapi.responses import PlainTextResponse @@ -37,8 +38,7 @@ class _StatusGet(BaseModel): @router.get("/status", include_in_schema=True, response_model=_StatusGet) -async def get_status(app: FastAPI = Depends(get_app)) -> _StatusGet: - +async def get_status(app: Annotated[FastAPI, Depends(get_app)]) -> _StatusGet: return _StatusGet( rabbitmq=_ComponentStatus( is_enabled=bool(app.state.rabbitmq_client), diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py index be378286fad..12ce642d348 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py @@ -1,7 +1,8 @@ import datetime from functools import cached_property -from typing import cast +from typing import Any, ClassVar, Final, cast +from aws_library.ec2.models import EC2InstanceBootSpecific from fastapi import FastAPI from models_library.basic_types import ( BootModeEnum, @@ -9,7 +10,7 @@ LogLevel, VersionTag, ) -from models_library.docker import DockerGenericTag, DockerLabelKey +from models_library.docker import DockerLabelKey from pydantic import ( AnyUrl, Field, @@ -29,23 +30,31 @@ from .._meta import API_VERSION, API_VTAG, APP_NAME +AUTOSCALING_ENV_PREFIX: Final[str] = "AUTOSCALING_" + + +class AutoscalingEC2Settings(EC2Settings): + class Config(EC2Settings.Config): + env_prefix = AUTOSCALING_ENV_PREFIX + + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + { + f"{AUTOSCALING_ENV_PREFIX}EC2_ACCESS_KEY_ID": "my_access_key_id", + f"{AUTOSCALING_ENV_PREFIX}EC2_ENDPOINT": "https://my_ec2_endpoint.com", + f"{AUTOSCALING_ENV_PREFIX}EC2_REGION_NAME": "us-east-1", + f"{AUTOSCALING_ENV_PREFIX}EC2_SECRET_ACCESS_KEY": "my_secret_access_key", + } + ], + } + class EC2InstancesSettings(BaseCustomSettings): - EC2_INSTANCES_ALLOWED_TYPES: list[str] = Field( + EC2_INSTANCES_ALLOWED_TYPES: dict[str, EC2InstanceBootSpecific] = Field( ..., - min_items=1, - unique_items=True, - description="Defines which EC2 instances are considered as candidates for new EC2 instance", - ) - EC2_INSTANCES_AMI_ID: str = Field( - ..., - min_length=1, - description="Defines the AMI (Amazon Machine Image) ID used to start a new EC2 instance", - ) - EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS: list[str] = Field( - default_factory=list, - description="script(s) to run on EC2 instance startup (be careful!), each entry is run one after the other using '&&' operator", + description="Defines which EC2 instances are considered as candidates for new EC2 instance and their respective boot specific parameters", ) + EC2_INSTANCES_KEY_NAME: str = Field( ..., min_length=1, @@ -72,15 +81,7 @@ class EC2InstancesSettings(BaseCustomSettings): min_length=1, description="prefix used to name the EC2 instances created by this instance of autoscaling", ) - EC2_INSTANCES_PRE_PULL_IMAGES: list[DockerGenericTag] = Field( - default_factory=list, - description="a list of docker image/tags to pull on instance cold start", - ) - EC2_INSTANCES_PRE_PULL_IMAGES_CRON_INTERVAL: datetime.timedelta = Field( - default=datetime.timedelta(minutes=30), - description="time interval between pulls of images (minimum is 1 minute) " - "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)", - ) + EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field( ..., min_items=1, @@ -112,11 +113,16 @@ def ensure_time_is_in_range(cls, value): @validator("EC2_INSTANCES_ALLOWED_TYPES") @classmethod - def check_valid_intance_names(cls, value): + def check_valid_instance_names( + cls, value: dict[str, EC2InstanceBootSpecific] + ) -> dict[str, EC2InstanceBootSpecific]: # NOTE: needed because of a flaw in BaseCustomSettings # issubclass raises TypeError if used on Aliases - parse_obj_as(tuple[InstanceTypeType, ...], value) - return value + if all(parse_obj_as(InstanceTypeType, key) for key in value): + return value + + msg = "Invalid instance type name" + raise ValueError(msg) class NodesMonitoringSettings(BaseCustomSettings): @@ -184,7 +190,9 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) - AUTOSCALING_EC2_ACCESS: EC2Settings | None = Field(auto_default_from_env=True) + AUTOSCALING_EC2_ACCESS: AutoscalingEC2Settings | None = Field( + auto_default_from_env=True + ) AUTOSCALING_EC2_INSTANCES: EC2InstancesSettings | None = Field( auto_default_from_env=True diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py index 08cd815c5ad..1e0c3f28a9e 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py @@ -184,9 +184,9 @@ async def sorted_allowed_instance_types(app: FastAPI) -> list[EC2InstanceType]: def _sort_according_to_allowed_types(instance_type: EC2InstanceType) -> int: assert app_settings.AUTOSCALING_EC2_INSTANCES # nosec - return app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES.index( - f"{instance_type.name}" - ) + return list( + app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES + ).index(f"{instance_type.name}") allowed_instance_types.sort(key=_sort_according_to_allowed_types) return allowed_instance_types @@ -452,15 +452,21 @@ async def _start_instances( assert app_settings.AUTOSCALING_EC2_INSTANCES # nosec instance_tags = auto_scaling_mode.get_ec2_tags(app) - instance_startup_script = await ec2_startup_script(app_settings) results = await asyncio.gather( *[ ec2_client.start_aws_instance( EC2InstanceConfig( type=instance_type, tags=instance_tags, - startup_script=instance_startup_script, - ami_id=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_AMI_ID, + startup_script=await ec2_startup_script( + app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[ + instance_type.name + ], + app_settings, + ), + ami_id=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[ + instance_type.name + ].ami_id, key_name=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME, security_group_ids=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_SECURITY_GROUP_IDS, subnet_id=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_SUBNET_ID, @@ -618,7 +624,7 @@ async def _find_terminateable_instances( _logger.info( "%s has still %ss before being terminateable", f"{instance.ec2_instance.id=}", - f"{(elapsed_time_since_drained - app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION).total_seconds()}", + f"{(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION - elapsed_time_since_drained).total_seconds()}", ) if terminateable_nodes: diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py index 106b68ea352..f0fe674e9db 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py +++ b/services/autoscaling/src/simcore_service_autoscaling/utils/auto_scaling_core.py @@ -3,7 +3,12 @@ import re from typing import Final -from aws_library.ec2.models import EC2InstanceData, EC2InstanceType, Resources +from aws_library.ec2.models import ( + EC2InstanceBootSpecific, + EC2InstanceData, + EC2InstanceType, + Resources, +) from models_library.generated_models.docker_rest_api import Node from types_aiobotocore_ec2.literals import InstanceTypeType @@ -74,15 +79,14 @@ def _find_node_with_name(node: Node) -> bool: return associated_instances, non_associated_instances -async def ec2_startup_script(app_settings: ApplicationSettings) -> str: - assert app_settings.AUTOSCALING_EC2_INSTANCES # nosec - startup_commands = ( - app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS.copy() - ) +async def ec2_startup_script( + ec2_boot_specific: EC2InstanceBootSpecific, app_settings: ApplicationSettings +) -> str: + startup_commands = ec2_boot_specific.custom_boot_scripts.copy() startup_commands.append(await utils_docker.get_docker_swarm_join_bash_command()) if app_settings.AUTOSCALING_REGISTRY: # noqa: SIM102 if pull_image_cmd := utils_docker.get_docker_pull_images_on_start_bash_command( - app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES + ec2_boot_specific.pre_pull_images ): startup_commands.append( " && ".join( @@ -96,7 +100,7 @@ async def ec2_startup_script(app_settings: ApplicationSettings) -> str: ) startup_commands.append( utils_docker.get_docker_pull_images_crontab( - app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES_CRON_INTERVAL + ec2_boot_specific.pre_pull_images_cron_interval ), ) diff --git a/services/autoscaling/tests/manual/.env-devel b/services/autoscaling/tests/manual/.env-devel index 80e54412ce4..a56c8ab125b 100644 --- a/services/autoscaling/tests/manual/.env-devel +++ b/services/autoscaling/tests/manual/.env-devel @@ -1,14 +1,18 @@ AUTOSCALING_DEBUG=true AUTOSCALING_LOGLEVEL=INFO AUTOSCALING_TASK_INTERVAL=30 -EC2_ACCESS_KEY_ID=XXXXXXXXXX -EC2_INSTANCES_ALLOWED_TYPES="[\"t2.micro\"]" -EC2_INSTANCES_AMI_ID=XXXXXXXXXX +AUTOSCALING_EC2_ACCESS_KEY_ID=XXXXXXXXXX +AUTOSCALING_EC2_SECRET_ACCESS_KEY=XXXXXXXXXX +AUTOSCALING_EC2_ENDPOINT=https://ec2.amazonaws.com +AUTOSCALING_EC2_REGION_NAME=us-east-1 +EC2_INSTANCES_MACHINES_BUFFER=0 +EC2_INSTANCES_MAX_INSTANCES=20 +EC2_INSTANCES_TIME_BEFORE_TERMINATION="00:03:00" +EC2_INSTANCES_ALLOWED_TYPES='{"t2.micro": {"ami_id": "XXXXXXXX", "custom_boot_scripts": ["whoami"], "pre_pull_images": ["ubuntu:latest"]}}' EC2_INSTANCES_KEY_NAME=XXXXXXXXXX +EC2_INSTANCES_NAME_PREFIX=testing-osparc-computational-cluster EC2_INSTANCES_SECURITY_GROUP_IDS="[\"XXXXXXXXXX\"]" EC2_INSTANCES_SUBNET_ID=XXXXXXXXXX -EC2_SECRET_ACCESS_KEY=XXXXXXXXXX -EC2_INSTANCES_NAME_PREFIX=testing-osparc-computational-cluster LOG_FORMAT_LOCAL_DEV_ENABLED=True # define the following to activate dynamic autoscaling # NODES_MONITORING_NEW_NODES_LABELS="[\"testing.autoscaled-node\"]" diff --git a/services/autoscaling/tests/manual/README.md b/services/autoscaling/tests/manual/README.md index e44e739d182..baa8c88aa94 100644 --- a/services/autoscaling/tests/manual/README.md +++ b/services/autoscaling/tests/manual/README.md @@ -8,7 +8,7 @@ The dynamic mode is used directly with docker swarm facilities. ### requirements 1. AWS EC2 access -2. a machine running in EC2 with docker installed and access to osparc-simcore repository +2. a machine running in EC2 with docker installed and access to osparc-simcore repository (for example t2.xlarge to have some computational power) ## computational mode @@ -92,6 +92,7 @@ make build-devel # this will build the autoscaling devel image cd osparc-simcore/services/autoscaling/tests/manual make .env # generate an initial .env file nano .env # edit .env and set the variables as needed +# in particular NODES_MONITORING_NEW_NODES_LABELS, NODES_MONITORING_NODE_LABELS, NODES_MONITORING_SERVICE_LABELS must be activated ``` 3. start autoscaling stack diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index 14f1132c531..c6a1dbe8191 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -6,6 +6,7 @@ import dataclasses import datetime import json +import random from collections.abc import AsyncIterator, Awaitable, Callable from copy import deepcopy from pathlib import Path @@ -19,8 +20,7 @@ import pytest import simcore_service_autoscaling from asgi_lifespan import LifespanManager -from aws_library.ec2.client import SimcoreEC2API -from aws_library.ec2.models import EC2InstanceData +from aws_library.ec2.models import EC2InstanceBootSpecific, EC2InstanceData from deepdiff import DeepDiff from faker import Faker from fakeredis.aioredis import FakeRedis @@ -41,7 +41,11 @@ from pytest_simcore.helpers.utils_host import get_localhost_ip from settings_library.rabbit import RabbitSettings from simcore_service_autoscaling.core.application import create_app -from simcore_service_autoscaling.core.settings import ApplicationSettings, EC2Settings +from simcore_service_autoscaling.core.settings import ( + AUTOSCALING_ENV_PREFIX, + ApplicationSettings, + EC2Settings, +) from simcore_service_autoscaling.models import Cluster, DaskTaskResources from simcore_service_autoscaling.modules.docker import AutoscalingDocker from tenacity import retry @@ -86,6 +90,19 @@ def ec2_instances() -> list[InstanceTypeType]: return ["t2.nano", "m5.12xlarge"] +@pytest.fixture +def mocked_ec2_server_envs( + mocked_ec2_server_settings: EC2Settings, + monkeypatch: pytest.MonkeyPatch, +) -> EnvVarsDict: + # NOTE: overrides the EC2Settings with what autoscaling expects + changed_envs: EnvVarsDict = { + f"{AUTOSCALING_ENV_PREFIX}{k}": v + for k, v in mocked_ec2_server_settings.dict().items() + } + return setenvs_from_dict(monkeypatch, changed_envs) + + @pytest.fixture def app_environment( mock_env_devel_environment: EnvVarsDict, @@ -94,18 +111,27 @@ def app_environment( ec2_instances: list[InstanceTypeType], ) -> EnvVarsDict: # SEE https://faker.readthedocs.io/en/master/providers/faker.providers.internet.html?highlight=internet#faker-providers-internet + envs = setenvs_from_dict( monkeypatch, { - "EC2_ACCESS_KEY_ID": faker.pystr(), - "EC2_SECRET_ACCESS_KEY": faker.pystr(), + "AUTOSCALING_EC2_ACCESS": "{}", + "AUTOSCALING_EC2_ACCESS_KEY_ID": faker.pystr(), + "AUTOSCALING_EC2_SECRET_ACCESS_KEY": faker.pystr(), + "AUTOSCALING_EC2_INSTANCES": "{}", "EC2_INSTANCES_KEY_NAME": faker.pystr(), "EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps( faker.pylist(allowed_types=(str,)) ), "EC2_INSTANCES_SUBNET_ID": faker.pystr(), - "EC2_INSTANCES_AMI_ID": faker.pystr(), - "EC2_INSTANCES_ALLOWED_TYPES": json.dumps(ec2_instances), + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( + { + ec2_type_name: random.choice( # noqa: S311 + EC2InstanceBootSpecific.Config.schema_extra["examples"] + ) + for ec2_type_name in ec2_instances + } + ), }, ) return mock_env_devel_environment | envs @@ -128,7 +154,13 @@ def mocked_ec2_instances_envs( "EC2_INSTANCES_SUBNET_ID": aws_subnet_id, "EC2_INSTANCES_AMI_ID": aws_ami_id, "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( - aws_allowed_ec2_instance_type_names + { + ec2_type_name: random.choice( # noqa: S311 + EC2InstanceBootSpecific.Config.schema_extra["examples"] + ) + | {"ami_id": aws_ami_id} + for ec2_type_name in aws_allowed_ec2_instance_type_names + } ), }, ) @@ -155,6 +187,7 @@ def enabled_dynamic_mode( return app_environment | setenvs_from_dict( monkeypatch, { + "AUTOSCALING_NODES_MONITORING": "{}", "NODES_MONITORING_NODE_LABELS": json.dumps(["pytest.fake-node-label"]), "NODES_MONITORING_SERVICE_LABELS": json.dumps( ["pytest.fake-service-label"] @@ -173,6 +206,7 @@ def enabled_computational_mode( return app_environment | setenvs_from_dict( monkeypatch, { + "AUTOSCALING_DASK": "{}", "DASK_MONITORING_URL": faker.url(), "DASK_MONITORING_USER_NAME": faker.user_name(), "DASK_MONITORING_PASSWORD": faker.password(), @@ -181,16 +215,15 @@ def enabled_computational_mode( @pytest.fixture -def disabled_rabbitmq(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): - monkeypatch.delenv("RABBIT_HOST") - monkeypatch.delenv("RABBIT_USER") - monkeypatch.delenv("RABBIT_SECURE") - monkeypatch.delenv("RABBIT_PASSWORD") +def disabled_rabbitmq( + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("AUTOSCALING_RABBITMQ", "null") @pytest.fixture -def disabled_ec2(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): - monkeypatch.delenv("EC2_ACCESS_KEY_ID") +def disabled_ec2(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AUTOSCALING_EC2_ACCESS", "null") @pytest.fixture @@ -506,17 +539,6 @@ def aws_allowed_ec2_instance_type_names_env( return app_environment | setenvs_from_dict(monkeypatch, changed_envs) -@pytest.fixture -async def autoscaling_ec2( - app_environment: EnvVarsDict, -) -> AsyncIterator[SimcoreEC2API]: - settings = EC2Settings.create_from_envs() - ec2 = await SimcoreEC2API.create(settings) - assert ec2 - yield ec2 - await ec2.close() - - @pytest.fixture def host_cpu_count() -> int: return psutil.cpu_count() diff --git a/services/autoscaling/tests/unit/test_core_settings.py b/services/autoscaling/tests/unit/test_core_settings.py index 8576d5fdb35..1ccf0c2a574 100644 --- a/services/autoscaling/tests/unit/test_core_settings.py +++ b/services/autoscaling/tests/unit/test_core_settings.py @@ -6,9 +6,49 @@ import json import pytest +from faker import Faker from pydantic import ValidationError -from pytest_simcore.helpers.utils_envs import EnvVarsDict -from simcore_service_autoscaling.core.settings import ApplicationSettings +from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict +from simcore_service_autoscaling.core.settings import ( + ApplicationSettings, + EC2InstancesSettings, +) +from types_aiobotocore_ec2.literals import InstanceTypeType + + +def test_ec2_instances_settings(app_environment: EnvVarsDict): + settings = EC2InstancesSettings.create_from_envs() + assert isinstance(settings.EC2_INSTANCES_ALLOWED_TYPES, dict) + + +@pytest.fixture +def instance_type_with_invalid_boot_script( + mock_env_devel_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + faker: Faker, + ec2_instances: list[InstanceTypeType], +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( + { + ec2_type_name: { + "ami_id": faker.pystr(), + "custom_boot_scripts": ['ls"'], + } + for ec2_type_name in ec2_instances + } + ), + }, + ) + + +def test_ec2_instances_settings_with_invalid_custom_script_raises( + app_environment: EnvVarsDict, instance_type_with_invalid_boot_script: EnvVarsDict +): + with pytest.raises(ValidationError): + EC2InstancesSettings.create_from_envs() def test_settings(app_environment: EnvVarsDict): @@ -48,10 +88,10 @@ def test_defining_both_computational_and_dynamic_modes_is_invalid_and_raises( ApplicationSettings.create_from_envs() -def test_invalid_EC2_INSTANCES_TIME_BEFORE_TERMINATION( +def test_invalid_EC2_INSTANCES_TIME_BEFORE_TERMINATION( # noqa: N802 app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch ): - monkeypatch.setenv("EC2_INSTANCES_TIME_BEFORE_TERMINATION", "1:05:00") + setenvs_from_dict(monkeypatch, {"EC2_INSTANCES_TIME_BEFORE_TERMINATION": "1:05:00"}) settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES assert settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION @@ -59,8 +99,9 @@ def test_invalid_EC2_INSTANCES_TIME_BEFORE_TERMINATION( datetime.timedelta(minutes=59) == settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION ) - - monkeypatch.setenv("EC2_INSTANCES_TIME_BEFORE_TERMINATION", "-1:05:00") + setenvs_from_dict( + monkeypatch, {"EC2_INSTANCES_TIME_BEFORE_TERMINATION": "-1:05:00"} + ) settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES assert ( @@ -69,31 +110,47 @@ def test_invalid_EC2_INSTANCES_TIME_BEFORE_TERMINATION( ) -def test_EC2_INSTANCES_PRE_PULL_IMAGES( - app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch +def test_EC2_INSTANCES_PRE_PULL_IMAGES( # noqa: N802 + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker ): settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES - assert not settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES # passing an invalid image tag name will fail - monkeypatch.setenv( - "EC2_INSTANCES_PRE_PULL_IMAGES", json.dumps(["io.simcore.some234.cool-"]) + setenvs_from_dict( + monkeypatch, + { + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( + { + "t2.micro": { + "ami_id": faker.pystr(), + "pre_pull_images": ["io.simcore.some234.cool-"], + } + } + ) + }, ) - settings = ApplicationSettings.create_from_envs() - assert not settings.AUTOSCALING_EC2_INSTANCES + with pytest.raises(ValidationError): + ApplicationSettings.create_from_envs() # passing a valid will pass - monkeypatch.setenv( - "EC2_INSTANCES_PRE_PULL_IMAGES", - json.dumps( - [ - "nginx:latest", - "itisfoundation/my-very-nice-service:latest", - "simcore/services/dynamic/another-nice-one:2.4.5", - "asd", - ] - ), + setenvs_from_dict( + monkeypatch, + { + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( + { + "t2.micro": { + "ami_id": faker.pystr(), + "pre_pull_images": [ + "nginx:latest", + "itisfoundation/my-very-nice-service:latest", + "simcore/services/dynamic/another-nice-one:2.4.5", + "asd", + ], + } + } + ), + }, ) settings = ApplicationSettings.create_from_envs() assert settings.AUTOSCALING_EC2_INSTANCES @@ -102,4 +159,6 @@ def test_EC2_INSTANCES_PRE_PULL_IMAGES( "itisfoundation/my-very-nice-service:latest", "simcore/services/dynamic/another-nice-one:2.4.5", "asd", - ] == settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_PRE_PULL_IMAGES + ] == next( + iter(settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES.values()) + ).pre_pull_images diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py index c81c4968bb7..84a82229f08 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py @@ -262,9 +262,9 @@ async def test_cluster_scaling_from_labelled_services_with_no_services_and_machi ec2_client, num_reservations=1, num_instances=mock_machines_buffer, - instance_type=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[ - 0 - ], + instance_type=next( + iter(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES) + ), instance_state="running", ) _assert_rabbit_autoscaling_message_sent( @@ -282,9 +282,9 @@ async def test_cluster_scaling_from_labelled_services_with_no_services_and_machi ec2_client, num_reservations=1, num_instances=mock_machines_buffer, - instance_type=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[ - 0 - ], + instance_type=next( + iter(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES) + ), instance_state="running", ) assert fake_node.Description @@ -315,9 +315,9 @@ async def test_cluster_scaling_from_labelled_services_with_no_services_and_machi ec2_client, num_reservations=1, num_instances=mock_machines_buffer, - instance_type=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[ - 0 - ], + instance_type=next( + iter(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES) + ), instance_state="running", ) diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py index 7f3663ef7dc..8395792d052 100644 --- a/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py +++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_task.py @@ -32,11 +32,10 @@ def app_environment( @pytest.fixture def mock_background_task(mocker: MockerFixture) -> mock.Mock: - mocked_task = mocker.patch( + return mocker.patch( "simcore_service_autoscaling.modules.auto_scaling_task.auto_scale_cluster", autospec=True, ) - return mocked_task async def test_auto_scaling_task_not_created_if_no_mode_defined( diff --git a/services/autoscaling/tests/unit/test_modules_rabbitmq.py b/services/autoscaling/tests/unit/test_modules_rabbitmq.py index 13cb1f73ee9..f1cdf140174 100644 --- a/services/autoscaling/tests/unit/test_modules_rabbitmq.py +++ b/services/autoscaling/tests/unit/test_modules_rabbitmq.py @@ -3,7 +3,8 @@ # pylint:disable=redefined-outer-name import asyncio -from typing import Any, Callable, Mapping +from collections.abc import Callable, Mapping +from typing import Any import aiodocker import pytest @@ -28,12 +29,12 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -_TENACITY_RETRY_PARAMS = dict( - reraise=True, - retry=retry_if_exception_type(AssertionError), - stop=stop_after_delay(30), - wait=wait_fixed(0.1), -) +_TENACITY_RETRY_PARAMS = { + "reraise": True, + "retry": retry_if_exception_type(AssertionError), + "stop": stop_after_delay(30), + "wait": wait_fixed(0.1), +} # Selection of core and tool services started in this swarm fixture (integration) pytest_simcore_core_services_selection = [ @@ -86,7 +87,7 @@ def test_rabbitmq_does_not_initialize_if_deactivated( initialized_app: FastAPI, ): assert hasattr(initialized_app.state, "rabbitmq_client") - assert initialized_app.state.rabbitmq_client == None + assert initialized_app.state.rabbitmq_client is None with pytest.raises(ConfigurationError): get_rabbitmq_client(initialized_app) diff --git a/services/autoscaling/tests/unit/test_utils_auto_scaling_core.py b/services/autoscaling/tests/unit/test_utils_auto_scaling_core.py index 345a757f575..48ba0286cfb 100644 --- a/services/autoscaling/tests/unit/test_utils_auto_scaling_core.py +++ b/services/autoscaling/tests/unit/test_utils_auto_scaling_core.py @@ -4,6 +4,7 @@ # pylint: disable=unused-variable # pylint: disable=too-many-arguments +import datetime import json import re from collections.abc import Callable @@ -13,6 +14,8 @@ from models_library.docker import DockerGenericTag from models_library.generated_models.docker_rest_api import Node from pydantic import parse_obj_as +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.utils_envs import setenvs_from_dict from simcore_service_autoscaling.core.errors import Ec2InvalidDnsNameError from simcore_service_autoscaling.core.settings import ApplicationSettings from simcore_service_autoscaling.models import EC2InstanceData @@ -135,10 +138,37 @@ def minimal_configuration( ... -async def test_ec2_startup_script_no_pre_pulling( - minimal_configuration: None, app_settings: ApplicationSettings +@pytest.fixture +def ec2_instances_boot_just_ami( + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker +) -> EnvVarsDict: + envs = setenvs_from_dict( + monkeypatch, + { + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( + {"t2.micro": {"ami_id": faker.pystr()}} + ), + }, + ) + return app_environment | envs + + +async def test_ec2_startup_script_just_ami( + minimal_configuration: None, + ec2_instances_boot_just_ami: EnvVarsDict, + app_settings: ApplicationSettings, ): - startup_script = await ec2_startup_script(app_settings) + assert app_settings.AUTOSCALING_EC2_INSTANCES + instance_boot_specific = next( + iter( + app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES.values() + ) + ) + assert not instance_boot_specific.pre_pull_images + assert instance_boot_specific.pre_pull_images_cron_interval == datetime.timedelta( + minutes=30 + ) + startup_script = await ec2_startup_script(instance_boot_specific, app_settings) assert len(startup_script.split("&&")) == 1 assert re.fullmatch( r"^docker swarm join --availability=drain --token .*$", startup_script @@ -146,9 +176,30 @@ async def test_ec2_startup_script_no_pre_pulling( @pytest.fixture -def enabled_pre_pull_images( - minimal_configuration: None, monkeypatch: pytest.MonkeyPatch -) -> list[DockerGenericTag]: +def ec2_instances_boot_ami_scripts( + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker +) -> list[str]: + custom_scripts = faker.pylist(allowed_types=(str,)) + setenvs_from_dict( + monkeypatch, + { + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( + { + "t2.micro": { + "ami_id": faker.pystr(), + "custom_boot_scripts": custom_scripts, + } + } + ), + }, + ) + return custom_scripts + + +@pytest.fixture +def ec2_instances_boot_ami_pre_pull( + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker +) -> EnvVarsDict: images = parse_obj_as( list[DockerGenericTag], [ @@ -158,23 +209,15 @@ def enabled_pre_pull_images( "asd", ], ) - monkeypatch.setenv( - "EC2_INSTANCES_PRE_PULL_IMAGES", - json.dumps(images), - ) - return images - - -@pytest.fixture -def enabled_custom_boot_scripts( - minimal_configuration: None, monkeypatch: pytest.MonkeyPatch, faker: Faker -) -> list[str]: - custom_scripts = faker.pylist(allowed_types=(str,)) - monkeypatch.setenv( - "EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS", - json.dumps(custom_scripts), + envs = setenvs_from_dict( + monkeypatch, + { + "EC2_INSTANCES_ALLOWED_TYPES": json.dumps( + {"t2.micro": {"ami_id": faker.pystr(), "pre_pull_images": images}} + ), + }, ) - return custom_scripts + return app_environment | envs @pytest.fixture @@ -184,10 +227,18 @@ def disabled_registry(monkeypatch: pytest.MonkeyPatch) -> None: async def test_ec2_startup_script_with_pre_pulling( minimal_configuration: None, - enabled_pre_pull_images: None, + ec2_instances_boot_ami_pre_pull: EnvVarsDict, app_settings: ApplicationSettings, ): - startup_script = await ec2_startup_script(app_settings) + assert app_settings.AUTOSCALING_EC2_INSTANCES + instance_boot_specific = next( + iter( + app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES.values() + ) + ) + assert instance_boot_specific.pre_pull_images + assert instance_boot_specific.pre_pull_images_cron_interval + startup_script = await ec2_startup_script(instance_boot_specific, app_settings) assert len(startup_script.split("&&")) == 7 assert re.fullmatch( r"^(docker swarm join [^&&]+) && (echo [^\s]+ \| docker login [^&&]+) && (echo [^&&]+) && (echo [^&&]+) && (chmod \+x [^&&]+) && (./docker-pull-script.sh) && (echo .+)$", @@ -197,26 +248,43 @@ async def test_ec2_startup_script_with_pre_pulling( async def test_ec2_startup_script_with_custom_scripts( minimal_configuration: None, - enabled_pre_pull_images: None, - enabled_custom_boot_scripts: list[str], + ec2_instances_boot_ami_scripts: list[str], app_settings: ApplicationSettings, ): for _ in range(3): - startup_script = await ec2_startup_script(app_settings) - assert len(startup_script.split("&&")) == 7 + len(enabled_custom_boot_scripts) + assert app_settings.AUTOSCALING_EC2_INSTANCES + instance_boot_specific = next( + iter( + app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES.values() + ) + ) + assert not instance_boot_specific.pre_pull_images + assert instance_boot_specific.pre_pull_images_cron_interval + startup_script = await ec2_startup_script(instance_boot_specific, app_settings) + assert len(startup_script.split("&&")) == 1 + len( + ec2_instances_boot_ami_scripts + ) assert re.fullmatch( - rf"^([^&&]+ &&){{{len(enabled_custom_boot_scripts)}}} (docker swarm join [^&&]+) && (echo [^\s]+ \| docker login [^&&]+) && (echo [^&&]+) && (echo [^&&]+) && (chmod \+x [^&&]+) && (./docker-pull-script.sh) && (echo .+)$", + rf"^([^&&]+ &&){{{len(ec2_instances_boot_ami_scripts)}}} (docker swarm join .+)$", startup_script, ), f"{startup_script=}" async def test_ec2_startup_script_with_pre_pulling_but_no_registry( minimal_configuration: None, - enabled_pre_pull_images: None, + ec2_instances_boot_ami_pre_pull: EnvVarsDict, disabled_registry: None, app_settings: ApplicationSettings, ): - startup_script = await ec2_startup_script(app_settings) + assert app_settings.AUTOSCALING_EC2_INSTANCES + instance_boot_specific = next( + iter( + app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES.values() + ) + ) + assert instance_boot_specific.pre_pull_images + assert instance_boot_specific.pre_pull_images_cron_interval + startup_script = await ec2_startup_script(instance_boot_specific, app_settings) assert len(startup_script.split("&&")) == 1 assert re.fullmatch( r"^docker swarm join --availability=drain --token .*$", startup_script diff --git a/services/autoscaling/tests/unit/test_utils_docker.py b/services/autoscaling/tests/unit/test_utils_docker.py index e9ee5c21c27..73d8719cb18 100644 --- a/services/autoscaling/tests/unit/test_utils_docker.py +++ b/services/autoscaling/tests/unit/test_utils_docker.py @@ -605,7 +605,7 @@ async def test_compute_node_used_resources_with_service( faker: Faker, ): # 1. if we have services with no defined reservations, then we cannot know what they use... - service_with_no_resources = await create_service(task_template, {}, "running") + await create_service(task_template, {}, "running") node_used_resources = await compute_node_used_resources( autoscaling_docker, host_node ) @@ -638,7 +638,7 @@ async def test_compute_node_used_resources_with_service( node_used_resources = await compute_node_used_resources( autoscaling_docker, host_node, - service_labels=[random.choice(list(service_labels.keys()))], + service_labels=[random.choice(list(service_labels.keys()))], # noqa: S311 ) assert node_used_resources == Resources(cpus=host_cpu_count, ram=ByteSize(0)) # 4. if we look for services with all the correct labels, they should then become visible again diff --git a/services/autoscaling/tests/unit/test_utils_rabbitmq.py b/services/autoscaling/tests/unit/test_utils_rabbitmq.py index e5350a393b0..3ec04e7a1db 100644 --- a/services/autoscaling/tests/unit/test_utils_rabbitmq.py +++ b/services/autoscaling/tests/unit/test_utils_rabbitmq.py @@ -4,7 +4,8 @@ # pylint:disable=too-many-arguments -from typing import Any, Awaitable, Callable +from collections.abc import Awaitable, Callable +from typing import Any import aiodocker from faker import Faker @@ -29,12 +30,12 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -_TENACITY_RETRY_PARAMS = dict( - reraise=True, - retry=retry_if_exception_type(AssertionError), - stop=stop_after_delay(30), - wait=wait_fixed(0.1), -) +_TENACITY_RETRY_PARAMS = { + "reraise": True, + "retry": retry_if_exception_type(AssertionError), + "stop": stop_after_delay(30), + "wait": wait_fixed(0.1), +} # Selection of core and tool services started in this swarm fixture (integration) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index 491cdeec854..3702227670b 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -2,6 +2,7 @@ from functools import cached_property from typing import Any, ClassVar, Final, cast +from aws_library.ec2.models import EC2InstanceBootSpecific from fastapi import FastAPI from models_library.basic_types import ( BootModeEnum, @@ -31,7 +32,7 @@ class Config(EC2Settings.Config): "examples": [ { f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_ACCESS_KEY_ID": "my_access_key_id", - f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_ENDPOINT": "http://my_ec2_endpoint.com", + f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_ENDPOINT": "https://my_ec2_endpoint.com", f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_REGION_NAME": "us-east-1", f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_SECRET_ACCESS_KEY": "my_secret_access_key", } @@ -40,21 +41,29 @@ class Config(EC2Settings.Config): class WorkersEC2InstancesSettings(BaseCustomSettings): - WORKERS_EC2_INSTANCES_ALLOWED_TYPES: list[str] = Field( + WORKERS_EC2_INSTANCES_ALLOWED_TYPES: dict[str, EC2InstanceBootSpecific] = Field( ..., - min_items=1, - unique_items=True, - description="Defines which EC2 instances are considered as candidates for new EC2 instance", + description="Defines which EC2 instances are considered as candidates for new EC2 instance and their respective boot specific parameters", ) - WORKERS_EC2_INSTANCES_AMI_ID: str = Field( + + WORKERS_EC2_INSTANCES_KEY_NAME: str = Field( ..., min_length=1, - description="Defines the AMI (Amazon Machine Image) ID used to start a new EC2 instance", + description="SSH key filename (without ext) to access the instance through SSH" + " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html)," + "this is required to start a new EC2 instance", + ) + # BUFFER is not exposed since we set it to 0 + WORKERS_EC2_INSTANCES_MAX_START_TIME: datetime.timedelta = Field( + default=datetime.timedelta(minutes=3), + description="Usual time taken an EC2 instance with the given AMI takes to be in 'running' mode " + "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)", ) WORKERS_EC2_INSTANCES_MAX_INSTANCES: int = Field( default=10, description="Defines the maximum number of instances the clusters_keeper app may create", ) + # NAME PREFIX is not exposed since we override it anyway WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field( ..., min_items=1, @@ -69,13 +78,6 @@ class WorkersEC2InstancesSettings(BaseCustomSettings): " (https://docs.aws.amazon.com/vpc/latest/userguide/configure-subnets.html), " "this is required to start a new EC2 instance", ) - WORKERS_EC2_INSTANCES_KEY_NAME: str = Field( - ..., - min_length=1, - description="SSH key filename (without ext) to access the instance through SSH" - " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html)," - "this is required to start a new EC2 instance", - ) WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION: datetime.timedelta = Field( default=datetime.timedelta(minutes=3), @@ -83,24 +85,18 @@ class WorkersEC2InstancesSettings(BaseCustomSettings): "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)", ) - WORKERS_EC2_INSTANCES_MAX_START_TIME: datetime.timedelta = Field( - default=datetime.timedelta(minutes=3), - description="Usual time taken an EC2 instance with the given AMI takes to be in 'running' mode " - "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)", - ) - - WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS: list[str] = Field( - default_factory=list, - description="script(s) to run on EC2 instance startup (be careful!), each entry is run one after the other using '&&' operator", - ) - @validator("WORKERS_EC2_INSTANCES_ALLOWED_TYPES") @classmethod - def check_valid_intance_names(cls, value): + def check_valid_instance_names( + cls, value: dict[str, EC2InstanceBootSpecific] + ) -> dict[str, EC2InstanceBootSpecific]: # NOTE: needed because of a flaw in BaseCustomSettings # issubclass raises TypeError if used on Aliases - parse_obj_as(tuple[InstanceTypeType, ...], value) - return value + if all(parse_obj_as(InstanceTypeType, key) for key in value): + return value + + msg = "Invalid instance type name" + raise ValueError(msg) class PrimaryEC2InstancesSettings(BaseCustomSettings): diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml b/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml index 213aac41821..bb4e22973ee 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml @@ -47,20 +47,20 @@ services: hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}" environment: DASK_MONITORING_URL: tcp://dask-scheduler:8786 - EC2_ACCESS_KEY_ID: ${CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID} - EC2_ENDPOINT: ${CLUSTERS_KEEPER_EC2_ENDPOINT} + AUTOSCALING_EC2_ACCESS_KEY_ID: ${CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID} + AUTOSCALING_EC2_ENDPOINT: ${CLUSTERS_KEEPER_EC2_ENDPOINT} + AUTOSCALING_EC2_REGION_NAME: ${CLUSTERS_KEEPER_EC2_REGION_NAME} + AUTOSCALING_EC2_SECRET_ACCESS_KEY: ${CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY} + AUTOSCALING_NODES_MONITORING: null + AUTOSCALING_POLL_INTERVAL: 10 EC2_INSTANCES_ALLOWED_TYPES: ${WORKERS_EC2_INSTANCES_ALLOWED_TYPES} - EC2_INSTANCES_AMI_ID: ${WORKERS_EC2_INSTANCES_AMI_ID} - EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS: ${WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS} EC2_INSTANCES_KEY_NAME: ${WORKERS_EC2_INSTANCES_KEY_NAME} + EC2_INSTANCES_MACHINES_BUFFER: 0 EC2_INSTANCES_MAX_INSTANCES: ${WORKERS_EC2_INSTANCES_MAX_INSTANCES} - EC2_INSTANCES_MAX_START_TIME: ${WORKERS_EC2_INSTANCES_MAX_START_TIME} EC2_INSTANCES_NAME_PREFIX: ${EC2_INSTANCES_NAME_PREFIX} EC2_INSTANCES_SECURITY_GROUP_IDS: ${WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS} EC2_INSTANCES_SUBNET_ID: ${WORKERS_EC2_INSTANCES_SUBNET_ID} EC2_INSTANCES_TIME_BEFORE_TERMINATION: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION} - EC2_REGION_NAME: ${CLUSTERS_KEEPER_EC2_REGION_NAME} - EC2_SECRET_ACCESS_KEY: ${CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY} LOG_FORMAT_LOCAL_DEV_ENABLED: 1 LOG_LEVEL: ${LOG_LEVEL:-WARNING} REDIS_HOST: redis diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py index 805d793a919..5bc8a44aa17 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py @@ -1,9 +1,11 @@ import base64 import datetime import functools +import json from typing import Any, Final from aws_library.ec2.models import EC2InstanceData +from fastapi.encoders import jsonable_encoder from models_library.api_schemas_clusters_keeper.clusters import ( ClusterState, OnDemandCluster, @@ -38,22 +40,22 @@ def _convert_to_env_list(entries: list[Any]) -> str: entries_as_str = ",".join(rf"\"{k}\"" for k in entries) return f"[{entries_as_str}]" + def _convert_to_env_dict(entries: dict[str, Any]) -> str: + return f"'{json.dumps(jsonable_encoder(entries))}'" + environment_variables = [ f"DOCKER_IMAGE_TAG={app_settings.CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG}", f"CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ACCESS_KEY_ID}", f"CLUSTERS_KEEPER_EC2_ENDPOINT={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ENDPOINT}", - f"WORKERS_EC2_INSTANCES_ALLOWED_TYPES={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_ALLOWED_TYPES)}", - f"WORKERS_EC2_INSTANCES_AMI_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_AMI_ID}", - f"WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS)}", + f"CLUSTERS_KEEPER_EC2_REGION_NAME={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_REGION_NAME}", + f"CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_SECRET_ACCESS_KEY}", + f"WORKERS_EC2_INSTANCES_ALLOWED_TYPES={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_ALLOWED_TYPES)}", f"WORKERS_EC2_INSTANCES_KEY_NAME={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_KEY_NAME}", f"WORKERS_EC2_INSTANCES_MAX_INSTANCES={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_MAX_INSTANCES}", - f"WORKERS_EC2_INSTANCES_MAX_START_TIME={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_MAX_START_TIME}", f"EC2_INSTANCES_NAME_PREFIX={cluster_machines_name_prefix}", f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}", f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}", f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}", - f"CLUSTERS_KEEPER_EC2_REGION_NAME={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_REGION_NAME}", - f"CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_SECRET_ACCESS_KEY}", f"LOG_LEVEL={app_settings.LOG_LEVEL}", ] diff --git a/services/clusters-keeper/tests/manual/README.md b/services/clusters-keeper/tests/manual/README.md index a2ecd57c8a0..21f203c86c4 100644 --- a/services/clusters-keeper/tests/manual/README.md +++ b/services/clusters-keeper/tests/manual/README.md @@ -63,7 +63,7 @@ CLUSTERS_KEEPER_EC2_ACCESS={} CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID=XXXXXXX CLUSTERS_KEEPER_EC2_ENDPOINT=https://ec2.amazonaws.com CLUSTERS_KEEPER_EC2_REGION_NAME=us-east-1 -CLUSTERS_KEEPER_SECRET_EC2_ACCESS_KEY=XXXXXXX +CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY=XXXXXXX CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES={} PRIMARY_EC2_INSTANCES_ALLOWED_TYPES="[\"t2.micro\"]" @@ -74,13 +74,12 @@ PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS="[\"XXXXXXX\"]" PRIMARY_EC2_INSTANCES_SUBNET_ID=XXXXXXX CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES={} -WORKERS_EC2_INSTANCES_ALLOWED_TYPES="[\"g4dn.xlarge\"]" # will change in some next PR -WORKERS_EC2_INSTANCES_AMI_ID=XXXXXXX -WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS="[]" +WORKERS_EC2_INSTANCES_ALLOWED_TYPES='{"g4dn.xlarge": {"ami_id": "XXXXXXXX", "custom_boot_scripts": ["whoami"], "pre_pull_images": ["ubuntu:latest"]}}' WORKERS_EC2_INSTANCES_KEY_NAME=XXXXXXX WORKERS_EC2_INSTANCES_MAX_INSTANCES=10 WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS="[\"XXXXXXX\"]" WORKERS_EC2_INSTANCES_SUBNET_ID=XXXXXXX +WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION="00:03:00" ``` 5. start osparc diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index 26c97e27084..82803bc03da 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -4,6 +4,7 @@ import importlib.resources import json +import random from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from pathlib import Path from typing import Any @@ -15,7 +16,7 @@ import simcore_service_clusters_keeper.data import yaml from asgi_lifespan import LifespanManager -from aws_library.ec2.client import SimcoreEC2API +from aws_library.ec2.models import EC2InstanceBootSpecific from faker import Faker from fakeredis.aioredis import FakeRedis from fastapi import FastAPI @@ -106,8 +107,14 @@ def app_environment( "PRIMARY_EC2_INSTANCES_AMI_ID": faker.pystr(), "PRIMARY_EC2_INSTANCES_ALLOWED_TYPES": json.dumps(ec2_instances), "CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES": "{}", - "WORKERS_EC2_INSTANCES_ALLOWED_TYPES": json.dumps(ec2_instances), - "WORKERS_EC2_INSTANCES_AMI_ID": faker.pystr(), + "WORKERS_EC2_INSTANCES_ALLOWED_TYPES": json.dumps( + { + ec2_type_name: random.choice( # noqa: S311 + EC2InstanceBootSpecific.Config.schema_extra["examples"] + ) + for ec2_type_name in ec2_instances + } + ), "WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS": json.dumps( faker.pylist(allowed_types=(str,)) ), @@ -162,10 +169,7 @@ def disable_clusters_management_background_task( @pytest.fixture def disabled_rabbitmq(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): - monkeypatch.delenv("RABBIT_HOST") - monkeypatch.delenv("RABBIT_USER") - monkeypatch.delenv("RABBIT_SECURE") - monkeypatch.delenv("RABBIT_PASSWORD") + monkeypatch.setenv("CLUSTERS_KEEPER_RABBITMQ", "null") @pytest.fixture @@ -204,36 +208,6 @@ async def async_client(initialized_app: FastAPI) -> AsyncIterator[httpx.AsyncCli yield client -@pytest.fixture -def aws_allowed_ec2_instance_type_names_env( - app_environment: EnvVarsDict, - monkeypatch: pytest.MonkeyPatch, -) -> EnvVarsDict: - changed_envs = { - "PRIMARY_EC2_INSTANCES_ALLOWED_TYPES": json.dumps( - [ - "t2.xlarge", - "t2.2xlarge", - "g3.4xlarge", - "r5n.4xlarge", - "r5n.8xlarge", - ] - ), - } - return app_environment | setenvs_from_dict(monkeypatch, changed_envs) - - -@pytest.fixture -async def clusters_keeper_ec2( - app_environment: EnvVarsDict, -) -> AsyncIterator[SimcoreEC2API]: - settings = EC2Settings.create_from_envs() - ec2 = await SimcoreEC2API.create(settings) - assert ec2 - yield ec2 - await ec2.close() - - @pytest.fixture async def mocked_redis_server(mocker: MockerFixture) -> None: mock_redis = FakeRedis() diff --git a/services/clusters-keeper/tests/unit/test_utils_clusters.py b/services/clusters-keeper/tests/unit/test_utils_clusters.py index 3e772896da0..80bae98c0b7 100644 --- a/services/clusters-keeper/tests/unit/test_utils_clusters.py +++ b/services/clusters-keeper/tests/unit/test_utils_clusters.py @@ -49,10 +49,11 @@ def test_create_startup_script( startup_script.splitlines()[-1].split("docker stack deploy")[0].strip() ) assert startup_script_envs_definition - startup_script_env_keys_names = { - entry.split("=", maxsplit=1)[0]: entry.split("=", maxsplit=1)[1] - for entry in startup_script_envs_definition.split(" ") - } + # Use regular expression to split the string into key-value pairs (courtesy of chatGPT) + startup_script_key_value_pairs: list[tuple[str, str]] = re.findall( + r"(\S+)=([\S\s]+?)(?=\S+=|$)", startup_script_envs_definition + ) + startup_script_env_keys_names = [key for key, _ in startup_script_key_value_pairs] # docker-compose expected values assert "services" in clusters_keeper_docker_compose assert "autoscaling" in clusters_keeper_docker_compose["services"] @@ -79,8 +80,6 @@ def test_create_startup_script( # check lists have \" written in them list_settings = [ - "WORKERS_EC2_INSTANCES_ALLOWED_TYPES", - "WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS", "WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS", ] assert all( diff --git a/services/docker-compose.yml b/services/docker-compose.yml index e8f63bbaaf7..c97297d37d7 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -35,10 +35,44 @@ services: networks: - autoscaling_subnet environment: - - LOG_LEVEL=${LOG_LEVEL:-WARNING} + - AUTOSCALING_LOGLEVEL=${AUTOSCALING_LOGLEVEL:-${LOG_LEVEL:-WARNING}} + - AUTOSCALING_POLL_INTERVAL=${AUTOSCALING_POLL_INTERVAL} + + - AUTOSCALING_DASK=${AUTOSCALING_DASK} # comp autoscaling + - DASK_MONITORING_URL=${DASK_MONITORING_URL} + + - AUTOSCALING_EC2_ACCESS=${AUTOSCALING_EC2_ACCESS} # used to enable/disable + - AUTOSCALING_EC2_ACCESS_KEY_ID=${AUTOSCALING_EC2_ACCESS_KEY_ID} + - AUTOSCALING_EC2_SECRET_ACCESS_KEY=${AUTOSCALING_EC2_SECRET_ACCESS_KEY} + - AUTOSCALING_EC2_REGION_NAME=${AUTOSCALING_EC2_REGION_NAME} + + - AUTOSCALING_EC2_INSTANCES=${AUTOSCALING_EC2_INSTANCES} # used to enable/disable + - EC2_INSTANCES_ALLOWED_TYPES=${EC2_INSTANCES_ALLOWED_TYPES} + - EC2_INSTANCES_MACHINES_BUFFER=${EC2_INSTANCES_MACHINES_BUFFER} + - EC2_INSTANCES_MAX_INSTANCES=${EC2_INSTANCES_MAX_INSTANCES} + - EC2_INSTANCES_NAME_PREFIX=${EC2_INSTANCES_NAME_PREFIX} + - EC2_INSTANCES_SECURITY_GROUP_IDS=${EC2_INSTANCES_SECURITY_GROUP_IDS} + - EC2_INSTANCES_SUBNET_ID=${EC2_INSTANCES_SUBNET_ID} + - EC2_INSTANCES_KEY_NAME=${EC2_INSTANCES_KEY_NAME} + + - AUTOSCALING_NODES_MONITORING=${AUTOSCALING_NODES_MONITORING} # dyn autoscaling + - NODES_MONITORING_NODE_LABELS=${NODES_MONITORING_NODE_LABELS} + - NODES_MONITORING_SERVICE_LABELS=${NODES_MONITORING_SERVICE_LABELS} + - NODES_MONITORING_NEW_NODES_LABELS=${NODES_MONITORING_NEW_NODES_LABELS} + - LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED} + - RABBIT_HOST=${RABBIT_HOST} + - RABBIT_PASSWORD=${RABBIT_PASSWORD} + - RABBIT_PORT=${RABBIT_PORT} + - RABBIT_SECURE=${RABBIT_SECURE} + - RABBIT_USER=${RABBIT_USER} - REDIS_HOST=${REDIS_HOST} - REDIS_PORT=${REDIS_PORT} + - REGISTRY_USER=${REGISTRY_USER} + - REGISTRY_PW=${REGISTRY_PW} + - REGISTRY_URL=${REGISTRY_URL} + - REGISTRY_SSL=${REGISTRY_SSL} + - REGISTRY_AUTH=${REGISTRY_AUTH} volumes: - "/var/run/docker.sock:/var/run/docker.sock" deploy: @@ -89,7 +123,7 @@ services: - CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID=${CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID} - CLUSTERS_KEEPER_EC2_ENDPOINT=${CLUSTERS_KEEPER_EC2_ENDPOINT} - CLUSTERS_KEEPER_EC2_REGION_NAME=${CLUSTERS_KEEPER_EC2_REGION_NAME} - - CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY=${CLUSTERS_KEEPER_} + - CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY=${CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY} - LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED} - CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=${CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES} - PRIMARY_EC2_INSTANCES_ALLOWED_TYPES=${PRIMARY_EC2_INSTANCES_ALLOWED_TYPES} @@ -108,8 +142,6 @@ services: - SWARM_STACK_NAME=${SWARM_STACK_NAME} - CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=${CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES} - WORKERS_EC2_INSTANCES_ALLOWED_TYPES=${WORKERS_EC2_INSTANCES_ALLOWED_TYPES} - - WORKERS_EC2_INSTANCES_AMI_ID=${WORKERS_EC2_INSTANCES_AMI_ID} - - WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS=${WORKERS_EC2_INSTANCES_CUSTOM_BOOT_SCRIPTS} - WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION=${WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION} - WORKERS_EC2_INSTANCES_KEY_NAME=${WORKERS_EC2_INSTANCES_KEY_NAME} - WORKERS_EC2_INSTANCES_MAX_INSTANCES=${WORKERS_EC2_INSTANCES_MAX_INSTANCES} From 1a4b6144153ec89af32b6a377f1d7e069ccaa73f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 16 Nov 2023 12:30:55 +0100 Subject: [PATCH 13/24] =?UTF-8?q?=20=E2=9C=A8=20Invitations=20to=20registe?= =?UTF-8?q?r=20to=20product=20(#4739)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../invitations/_core.py | 56 +++++++++++- .../invitations/plugin.py | 1 + .../login/_auth_api.py | 60 ++++++++++++ .../login/_registration.py | 6 ++ .../login/handlers_auth.py | 35 ++----- .../login/handlers_registration.py | 74 ++++++++++----- .../simcore_service_webserver/login/utils.py | 4 + .../products/_api.py | 2 +- .../products/_events.py | 7 +- .../products/_middlewares.py | 3 +- .../unit/with_dbs/03/invitations/conftest.py | 40 ++++++-- .../03/invitations/test_invitations.py | 91 ++++++++++++------- .../test_login_registration_invitations.py | 4 +- 13 files changed, 284 insertions(+), 99 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/login/_auth_api.py diff --git a/services/web/server/src/simcore_service_webserver/invitations/_core.py b/services/web/server/src/simcore_service_webserver/invitations/_core.py index e2741148047..ded31e6f7ab 100644 --- a/services/web/server/src/simcore_service_webserver/invitations/_core.py +++ b/services/web/server/src/simcore_service_webserver/invitations/_core.py @@ -9,11 +9,14 @@ ApiInvitationContentAndLink, ApiInvitationInputs, ) +from models_library.users import GroupID from pydantic import AnyHttpUrl, ValidationError, parse_obj_as from servicelib.error_codes import create_error_code +from simcore_postgres_database.models.groups import user_to_groups from simcore_postgres_database.models.users import users from ..db.plugin import get_database_engine +from ..products.api import Product from ._client import InvitationsServiceApi, get_invitations_service_api from .errors import ( MSG_INVALID_INVITATION_URL, @@ -26,14 +29,31 @@ _logger = logging.getLogger(__name__) -async def _is_user_registered(app: web.Application, email: str) -> bool: +async def _is_user_registered_in_platform(app: web.Application, email: str) -> bool: pg_engine = get_database_engine(app=app) - async with pg_engine.acquire() as conn: user_id = await conn.scalar(sa.select(users.c.id).where(users.c.email == email)) return user_id is not None +async def _is_user_registered_in_product( + app: web.Application, email: str, product_group_id: GroupID +) -> bool: + pg_engine = get_database_engine(app=app) + + async with pg_engine.acquire() as conn: + user_id = await conn.scalar( + sa.select(users.c.id) + .select_from( + sa.join(user_to_groups, users, user_to_groups.c.uid == users.c.id) + ) + .where( + (users.c.email == email) & (user_to_groups.c.gid == product_group_id) + ) + ) + return user_id is not None + + @contextmanager def _handle_exceptions_as_invitations_errors(): try: @@ -56,6 +76,7 @@ def _handle_exceptions_as_invitations_errors(): raise InvitationsServiceUnavailable from err except (ValidationError, ClientError) as err: + _logger.debug("Invitations error %s", f"{err}") raise InvitationsServiceUnavailable from err except InvitationsErrors: @@ -80,12 +101,21 @@ def is_service_invitation_code(code: str): async def validate_invitation_url( - app: web.Application, guest_email: str, invitation_url: str + app: web.Application, + *, + current_product: Product, + guest_email: str, + invitation_url: str, ) -> ApiInvitationContent: """Validates invitation and associated email/user and returns content upon success raises InvitationsError """ + if current_product.group_id is None: + raise InvitationsServiceUnavailable( + reason="Current product is not configured for invitations" + ) + invitations_service: InvitationsServiceApi = get_invitations_service_api(app=app) with _handle_exceptions_as_invitations_errors(): @@ -99,13 +129,29 @@ async def validate_invitation_url( invitation_url=valid_url ) + # check email if invitation.guest != guest_email: raise InvalidInvitation( reason="This invitation was issued for a different email" ) - # existing users cannot be re-invited - if await _is_user_registered(app=app, email=invitation.guest): + # check product + assert current_product.group_id is not None # nosec + if ( + invitation.product is not None + and invitation.product != current_product.name + ): + raise InvalidInvitation( + reason="This invitation was issued for a different product. " + f"Got '{invitation.product}', expected '{current_product.name}'" + ) + + # check invitation used + assert invitation.product == current_product.name # nosec + if await _is_user_registered_in_product( + app=app, email=invitation.guest, product_group_id=current_product.group_id + ): + # NOTE: a user might be already registered but the invitation is for another product raise InvalidInvitation(reason=MSG_INVITATION_ALREADY_USED) return invitation diff --git a/services/web/server/src/simcore_service_webserver/invitations/plugin.py b/services/web/server/src/simcore_service_webserver/invitations/plugin.py index 1b865c9dac4..fd20f0f8601 100644 --- a/services/web/server/src/simcore_service_webserver/invitations/plugin.py +++ b/services/web/server/src/simcore_service_webserver/invitations/plugin.py @@ -10,6 +10,7 @@ from .._constants import APP_SETTINGS_KEY from ..db.plugin import setup_db +from ..products.plugin import setup_products from ._client import invitations_service_api_cleanup_ctx _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/login/_auth_api.py b/services/web/server/src/simcore_service_webserver/login/_auth_api.py new file mode 100644 index 00000000000..d22562b147d --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/login/_auth_api.py @@ -0,0 +1,60 @@ +from datetime import datetime + +from aiohttp import web +from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON + +from ..products.api import Product +from ..security.api import check_password, encrypt_password +from ._constants import MSG_UNKNOWN_EMAIL, MSG_WRONG_PASSWORD +from .storage import AsyncpgStorage, get_plugin_storage +from .utils import USER, get_user_name_from_email, validate_user_status + + +async def get_user_by_email(app: web.Application, *, email: str) -> dict: + db: AsyncpgStorage = get_plugin_storage(app) + user: dict = await db.get_user({"email": email}) + return user + + +async def create_user( + app: web.Application, + *, + email: str, + password: str, + status: str, + expires_at: datetime | None +) -> dict: + db: AsyncpgStorage = get_plugin_storage(app) + + user: dict = await db.create_user( + { + "name": get_user_name_from_email(email), + "email": email, + "password_hash": encrypt_password(password), + "status": status, + "role": USER, + "expires_at": expires_at, + } + ) + return user + + +async def check_authorized_user_or_raise( + user: dict, + password: str, + product: Product, +) -> dict: + + if not user: + raise web.HTTPUnauthorized( + reason=MSG_UNKNOWN_EMAIL, content_type=MIMETYPE_APPLICATION_JSON + ) + + validate_user_status(user=user, support_email=product.support_email) + + if not check_password(password, user["password_hash"]): + raise web.HTTPUnauthorized( + reason=MSG_WRONG_PASSWORD, content_type=MIMETYPE_APPLICATION_JSON + ) + + return user diff --git a/services/web/server/src/simcore_service_webserver/login/_registration.py b/services/web/server/src/simcore_service_webserver/login/_registration.py index 0f644548b0e..bfc1c9d0bab 100644 --- a/services/web/server/src/simcore_service_webserver/login/_registration.py +++ b/services/web/server/src/simcore_service_webserver/login/_registration.py @@ -12,6 +12,7 @@ from aiohttp import web from models_library.basic_types import IdInt from models_library.emails import LowerCaseEmailStr +from models_library.products import ProductName from pydantic import BaseModel, Field, Json, PositiveInt, ValidationError, validator from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from simcore_postgres_database.models.confirmations import ConfirmationAction @@ -23,6 +24,7 @@ validate_invitation_url, ) from ..invitations.errors import InvalidInvitation, InvitationsServiceUnavailable +from ..products.api import Product from ._confirmation import is_confirmation_expired, validate_confirmation_code from ._constants import MSG_EMAIL_EXISTS, MSG_INVITATIONS_CONTACT_SUFFIX from .settings import LoginOptions @@ -51,6 +53,7 @@ class InvitationData(BaseModel): "Sets the number of days from creation until the account expires", ) extra_credits_in_usd: PositiveInt | None = None + product: ProductName | None = None class _InvitationValidator(BaseModel): @@ -190,6 +193,7 @@ async def extract_email_from_invitation( async def check_and_consume_invitation( invitation_code: str, guest_email: str, + product: Product, db: AsyncpgStorage, cfg: LoginOptions, app: web.Application, @@ -207,6 +211,7 @@ async def check_and_consume_invitation( with _invitations_request_context(invitation_code=invitation_code) as url: content = await validate_invitation_url( app, + current_product=product, guest_email=guest_email, invitation_url=f"{url}", ) @@ -219,6 +224,7 @@ async def check_and_consume_invitation( guest=content.guest, trial_account_days=content.trial_account_days, extra_credits_in_usd=content.extra_credits_in_usd, + product=content.product, ) # database-type invitations diff --git a/services/web/server/src/simcore_service_webserver/login/handlers_auth.py b/services/web/server/src/simcore_service_webserver/login/handlers_auth.py index 4cde1cb741d..b18dff0ae45 100644 --- a/services/web/server/src/simcore_service_webserver/login/handlers_auth.py +++ b/services/web/server/src/simcore_service_webserver/login/handlers_auth.py @@ -14,7 +14,7 @@ from .._meta import API_VTAG from ..products.api import Product, get_current_product -from ..security.api import check_password, forget +from ..security.api import forget from ..session.access_policies import ( on_success_grant_session_access_to, session_access_required, @@ -27,6 +27,7 @@ mask_phone_number, send_sms_code, ) +from ._auth_api import check_authorized_user_or_raise, get_user_by_email from ._constants import ( CODE_2FA_CODE_REQUIRED, CODE_PHONE_NUMBER_REQUIRED, @@ -37,22 +38,14 @@ MSG_LOGGED_OUT, MSG_PHONE_MISSING, MSG_UNAUTHORIZED_LOGIN_2FA, - MSG_UNKNOWN_EMAIL, MSG_WRONG_2FA_CODE, - MSG_WRONG_PASSWORD, ) from ._models import InputSchema from ._security import login_granted_response from .decorators import login_required from .settings import LoginSettingsForProduct, get_plugin_settings from .storage import AsyncpgStorage, get_plugin_storage -from .utils import ( - ACTIVE, - envelope_response, - flash_response, - notify_user_logout, - validate_user_status, -) +from .utils import envelope_response, flash_response, notify_user_logout log = logging.getLogger(__name__) @@ -98,25 +91,13 @@ async def login(request: web.Request): settings: LoginSettingsForProduct = get_plugin_settings( request.app, product_name=product.name ) - db: AsyncpgStorage = get_plugin_storage(request.app) - login_ = await parse_request_body_as(LoginBody, request) - user = await db.get_user({"email": login_.email}) - if not user: - raise web.HTTPUnauthorized( - reason=MSG_UNKNOWN_EMAIL, content_type=MIMETYPE_APPLICATION_JSON - ) - - validate_user_status(user=user, support_email=product.support_email) - - if not check_password(login_.password.get_secret_value(), user["password_hash"]): - raise web.HTTPUnauthorized( - reason=MSG_WRONG_PASSWORD, content_type=MIMETYPE_APPLICATION_JSON - ) - - assert user["status"] == ACTIVE, "db corrupted. Invalid status" # nosec - assert user["email"] == login_.email, "db corrupted. Invalid email" # nosec + user = await check_authorized_user_or_raise( + user=await get_user_by_email(request.app, email=login_.email), + password=login_.password.get_secret_value(), + product=product, + ) # Some roles have login privileges has_privileges: Final[bool] = UserRole(user["role"]) > UserRole.USER diff --git a/services/web/server/src/simcore_service_webserver/login/handlers_registration.py b/services/web/server/src/simcore_service_webserver/login/handlers_registration.py index 601b3563217..24e1a5fa85b 100644 --- a/services/web/server/src/simcore_service_webserver/login/handlers_registration.py +++ b/services/web/server/src/simcore_service_webserver/login/handlers_registration.py @@ -14,7 +14,6 @@ from ..groups.api import auto_add_user_to_groups, auto_add_user_to_product_group from ..invitations.api import is_service_invitation_code from ..products.api import Product, get_current_product -from ..security.api import encrypt_password from ..session.access_policies import ( on_success_grant_session_access_to, session_access_required, @@ -22,6 +21,7 @@ from ..utils import MINUTE from ..utils_aiohttp import NextPage, envelope_json_response from ..utils_rate_limiting import global_rate_limit_route +from . import _auth_api from ._2fa import create_2fa_code, mask_phone_number, send_sms_code from ._confirmation import make_confirmation_link from ._constants import ( @@ -51,9 +51,9 @@ ACTIVE, CONFIRMATION_PENDING, REGISTRATION, - USER, envelope_response, flash_response, + get_user_name_from_email, notify_user_confirmation, ) from .utils_email import get_template_path, send_email_from_template @@ -61,10 +61,6 @@ log = logging.getLogger(__name__) -def _get_user_name(email: str) -> str: - return email.split("@")[0] - - routes = RouteTableDef() @@ -168,8 +164,24 @@ async def register(request: web.Request): content_type=MIMETYPE_APPLICATION_JSON, ) + # INVITATIONS expires_at: datetime | None = None # = does not expire invitation = None + # There are 3 possible states for an invitation: + # 1. Invitation is not required (i.e. the app has disabled invitations) + # 2. Invitation is invalid + # 3. Invitation is valid + # + # For those states the `invitation` variable get the following values + # 1. `None + # 2. no value, it raises and exception + # 3. gets `InvitationData` + # ` + # In addition, for 3. there are two types of invitations: + # 1. the invitation generated by the `invitation` service (new). + # 2. the invitation created by hand in the db confirmation table (deprecated). This + # one does not understand products. + # if settings.LOGIN_REGISTRATION_INVITATION_REQUIRED: # Only requests with INVITATION can register user # to either a permanent or to a trial account @@ -182,6 +194,7 @@ async def register(request: web.Request): invitation = await check_and_consume_invitation( invitation_code, + product=product, guest_email=registration.email, db=db, cfg=cfg, @@ -190,26 +203,39 @@ async def register(request: web.Request): if invitation.trial_account_days: expires_at = datetime.utcnow() + timedelta(invitation.trial_account_days) - username = _get_user_name(registration.email) - user: dict = await db.create_user( - { - "name": username, - "email": registration.email, - "password_hash": encrypt_password(registration.password.get_secret_value()), - "status": ( + # get authorized user or create new + user = await _auth_api.get_user_by_email(request.app, email=registration.email) + if user: + await _auth_api.check_authorized_user_or_raise( + user, + password=registration.password.get_secret_value(), + product=product, + ) + else: + user = await _auth_api.create_user( + request.app, + email=registration.email, + password=registration.password.get_secret_value(), + status=( CONFIRMATION_PENDING if settings.LOGIN_REGISTRATION_CONFIRMATION_REQUIRED else ACTIVE ), - "role": USER, - "expires_at": expires_at, - } + expires_at=expires_at, + ) + + # setup user groups + assert ( # nosec + product.name == invitation.product + if invitation and invitation.product + else True ) - # NOTE: PC->SAN: should this go here or when user is actually logged in? await auto_add_user_to_groups(app=request.app, user_id=user["id"]) await auto_add_user_to_product_group( - app=request.app, user_id=user["id"], product_name=product.name + app=request.app, + user_id=user["id"], + product_name=product.name, ) if settings.LOGIN_REGISTRATION_CONFIRMATION_REQUIRED: @@ -231,7 +257,7 @@ async def register(request: web.Request): context={ "host": request.host, "link": email_confirmation_url, # SEE email_confirmation handler (action=REGISTRATION) - "name": username, + "name": user["name"], "support_email": product.support_email, }, ) @@ -259,6 +285,12 @@ async def register(request: web.Request): # NOTE: Here confirmation is disabled assert settings.LOGIN_REGISTRATION_CONFIRMATION_REQUIRED is False # nosec + assert ( # nosec + product.name == invitation.product + if invitation and invitation.product + else True + ) + await notify_user_confirmation( request.app, user_id=user["id"], @@ -331,7 +363,7 @@ async def register_phone(request: web.Request): raise ValueError(msg) if await db.get_user({"phone": registration.phone}): - raise web.HTTPUnauthorized( + raise web.HTTPUnauthorized( # noqa: TRY301 reason="Cannot register this phone number because it is already assigned to an active user", content_type=MIMETYPE_APPLICATION_JSON, ) @@ -347,7 +379,7 @@ async def register_phone(request: web.Request): twilo_auth=settings.LOGIN_TWILIO, twilio_messaging_sid=product.twilio_messaging_sid, twilio_alpha_numeric_sender=product.twilio_alpha_numeric_sender_id, - user_name=_get_user_name(registration.email), + user_name=get_user_name_from_email(registration.email), ) message = MSG_2FA_CODE_SENT.format( diff --git a/services/web/server/src/simcore_service_webserver/login/utils.py b/services/web/server/src/simcore_service_webserver/login/utils.py index 3635225236a..f4732de0df6 100644 --- a/services/web/server/src/simcore_service_webserver/login/utils.py +++ b/services/web/server/src/simcore_service_webserver/login/utils.py @@ -144,3 +144,7 @@ def envelope_response( dumps=json_dumps, status=status, ) + + +def get_user_name_from_email(email: str) -> str: + return email.split("@")[0] diff --git a/services/web/server/src/simcore_service_webserver/products/_api.py b/services/web/server/src/simcore_service_webserver/products/_api.py index 9368627c17a..e124a37eb8e 100644 --- a/services/web/server/src/simcore_service_webserver/products/_api.py +++ b/services/web/server/src/simcore_service_webserver/products/_api.py @@ -30,7 +30,7 @@ def get_current_product(request: web.Request) -> Product: def list_products(app: web.Application) -> list[Product]: - products: list[Product] = app[APP_PRODUCTS_KEY].values() + products: list[Product] = list(app[APP_PRODUCTS_KEY].values()) return products diff --git a/services/web/server/src/simcore_service_webserver/products/_events.py b/services/web/server/src/simcore_service_webserver/products/_events.py index f0165a21cd8..48c23dc1ec1 100644 --- a/services/web/server/src/simcore_service_webserver/products/_events.py +++ b/services/web/server/src/simcore_service_webserver/products/_events.py @@ -1,5 +1,6 @@ import logging import tempfile +from collections import OrderedDict from pathlib import Path from aiohttp import web @@ -59,7 +60,9 @@ async def auto_create_products_groups(app: web.Application) -> None: def _set_app_state( - app: web.Application, app_products: dict[str, Product], default_product_name: str + app: web.Application, + app_products: OrderedDict[str, Product], + default_product_name: str, ): app[APP_PRODUCTS_KEY] = app_products assert default_product_name in app_products # nosec @@ -70,7 +73,7 @@ async def load_products_on_startup(app: web.Application): """ Loads info on products stored in the database into app's storage (i.e. memory) """ - app_products: dict[str, Product] = {} + app_products: OrderedDict[str, Product] = OrderedDict() engine: Engine = app[APP_DB_ENGINE_KEY] async with engine.acquire() as connection: async for row in iter_products(connection): diff --git a/services/web/server/src/simcore_service_webserver/products/_middlewares.py b/services/web/server/src/simcore_service_webserver/products/_middlewares.py index 4c3dbf78d27..1f69a3980f1 100644 --- a/services/web/server/src/simcore_service_webserver/products/_middlewares.py +++ b/services/web/server/src/simcore_service_webserver/products/_middlewares.py @@ -1,4 +1,5 @@ import logging +from collections import OrderedDict from aiohttp import web from servicelib.aiohttp.typing_extension import Handler @@ -12,7 +13,7 @@ def _discover_product_by_hostname(request: web.Request) -> str | None: - products: dict[str, Product] = request.app[APP_PRODUCTS_KEY] + products: OrderedDict[str, Product] = request.app[APP_PRODUCTS_KEY] for product in products.values(): if product.host_regex.search(request.host): product_name: str = product.name diff --git a/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py b/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py index 8779e0af913..6f312bf8a48 100644 --- a/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py @@ -24,6 +24,7 @@ InvitationsSettings, get_plugin_settings, ) +from simcore_service_webserver.products.api import Product, list_products from yarl import URL @@ -44,13 +45,27 @@ def invitations_service_openapi_specs( @pytest.fixture -def expected_invitation( +def current_product(client: TestClient) -> Product: + assert client.app + products = list_products(client.app) + assert products + assert products[0].name == "osparc" + return products[0] + + +@pytest.fixture +def fake_osparc_invitation( invitations_service_openapi_specs: dict[str, Any] ) -> ApiInvitationContent: + """ + Emulates an invitation for osparc product + """ oas = deepcopy(invitations_service_openapi_specs) - return ApiInvitationContent.parse_obj( + content = ApiInvitationContent.parse_obj( oas["components"]["schemas"]["ApiInvitationContent"]["example"] ) + content.product = "osparc" + return content @pytest.fixture() @@ -63,7 +78,7 @@ def mock_invitations_service_http_api( aioresponses_mocker: AioResponsesMock, invitations_service_openapi_specs: dict[str, Any], base_url: URL, - expected_invitation: ApiInvitationContent, + fake_osparc_invitation: ApiInvitationContent, ) -> AioResponsesMock: oas = deepcopy(invitations_service_openapi_specs) @@ -85,18 +100,24 @@ def mock_invitations_service_http_api( # extract assert "/v1/invitations:extract" in oas["paths"] + + def _extract(url, **kwargs): + return CallbackResult( + status=web.HTTPOk.status_code, + payload=jsonable_encoder(fake_osparc_invitation.dict()), + ) + aioresponses_mocker.post( f"{base_url}/v1/invitations:extract", - status=web.HTTPOk.status_code, - payload=jsonable_encoder(expected_invitation.dict()), + callback=_extract, + repeat=True, # NOTE: this can be used many times ) # generate assert "/v1/invitations" in oas["paths"] example = oas["components"]["schemas"]["ApiInvitationContentAndLink"]["example"] - def _side_effect(url, **kwargs): - + def _generate(url, **kwargs): body = kwargs["json"] assert isinstance(body, dict) if not body.get("product"): @@ -115,6 +136,9 @@ def _side_effect(url, **kwargs): ), ) - aioresponses_mocker.post(f"{base_url}/v1/invitations", callback=_side_effect) + aioresponses_mocker.post( + f"{base_url}/v1/invitations", + callback=_generate, + ) return aioresponses_mocker diff --git a/services/web/server/tests/unit/with_dbs/03/invitations/test_invitations.py b/services/web/server/tests/unit/with_dbs/03/invitations/test_invitations.py index 72330812e85..4eae544d8c1 100644 --- a/services/web/server/tests/unit/with_dbs/03/invitations/test_invitations.py +++ b/services/web/server/tests/unit/with_dbs/03/invitations/test_invitations.py @@ -11,6 +11,7 @@ from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict from pytest_simcore.helpers.utils_login import NewUser from simcore_service_webserver.application_settings import ApplicationSettings +from simcore_service_webserver.groups.api import auto_add_user_to_product_group from simcore_service_webserver.invitations._client import ( InvitationsServiceApi, get_invitations_service_api, @@ -20,6 +21,7 @@ InvalidInvitation, InvitationsServiceUnavailable, ) +from simcore_service_webserver.products.api import Product from yarl import URL @@ -29,9 +31,17 @@ def app_environment( env_devel_dict: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, ): - envs_plugins = setenvs_from_dict( + # ensures WEBSERVER_INVITATIONS is undefined + monkeypatch.delenv("WEBSERVER_INVITATIONS", raising=False) + app_environment.pop("WEBSERVER_INVITATIONS", None) + + # new envs + envs = setenvs_from_dict( monkeypatch, { + # as before + **app_environment, + # disable these plugins "WEBSERVER_ACTIVITY": "null", "WEBSERVER_DB_LISTENER": "0", "WEBSERVER_DIAGNOSTICS": "null", @@ -47,30 +57,24 @@ def app_environment( "WEBSERVER_TRACING": "null", "WEBSERVER_VERSION_CONTROL": "0", "WEBSERVER_WALLETS": "0", + # set INVITATIONS_* variables using those in .env-devel + **{ + key: value + for key, value in env_devel_dict.items() + if key.startswith("INVITATIONS_") + }, }, ) - # undefine WEBSERVER_INVITATIONS - app_environment.pop("WEBSERVER_INVITATIONS", None) - monkeypatch.delenv("WEBSERVER_INVITATIONS", raising=False) - - # set INVITATIONS_* variables using those in .devel-env - envs_invitations = setenvs_from_dict( - monkeypatch, - envs={ - name: value - for name, value in env_devel_dict.items() - if name.startswith("INVITATIONS_") - }, - ) - + # tests envs print(ApplicationSettings.create_from_envs().json(indent=2)) - return app_environment | envs_plugins | envs_invitations + return envs async def test_invitation_service_unavailable( client: TestClient, - expected_invitation: ApiInvitationContent, + fake_osparc_invitation: ApiInvitationContent, + current_product: Product, ): assert client.app invitations_api: InvitationsServiceApi = get_invitations_service_api(app=client.app) @@ -80,8 +84,9 @@ async def test_invitation_service_unavailable( with pytest.raises(InvitationsServiceUnavailable): await validate_invitation_url( app=client.app, - guest_email=expected_invitation.guest, + guest_email=fake_osparc_invitation.guest, invitation_url="https://server.com#/registration?invitation=1234", + current_product=current_product, ) @@ -99,53 +104,75 @@ async def test_invitation_service_api_ping( # second request is mocked to fail (since mock only works once) assert not await invitations_api.is_responsive() + mock_invitations_service_http_api.assert_called() + async def test_valid_invitation( client: TestClient, mock_invitations_service_http_api: AioResponsesMock, - expected_invitation: ApiInvitationContent, + fake_osparc_invitation: ApiInvitationContent, + current_product: Product, ): assert client.app invitation = await validate_invitation_url( app=client.app, - guest_email=expected_invitation.guest, + guest_email=fake_osparc_invitation.guest, invitation_url="https://server.com#register?invitation=1234", + current_product=current_product, ) assert invitation - assert invitation == expected_invitation + assert invitation == fake_osparc_invitation + mock_invitations_service_http_api.assert_called_once() -async def test_invalid_invitation_if_guest_is_already_registered( +async def test_invalid_invitation_if_guest_is_already_registered_in_product( client: TestClient, mock_invitations_service_http_api: AioResponsesMock, - expected_invitation: ApiInvitationContent, + fake_osparc_invitation: ApiInvitationContent, + current_product: Product, ): assert client.app + kwargs = { + "app": client.app, + "guest_email": fake_osparc_invitation.guest, + "invitation_url": "https://server.com#register?invitation=1234", + "current_product": current_product, + } + + # user exists async with NewUser( params={ "name": "test-user", - "email": expected_invitation.guest, + "email": fake_osparc_invitation.guest, }, app=client.app, - ): + ) as user: + + # valid because user is not assigned to product + assert await validate_invitation_url(**kwargs) == fake_osparc_invitation + + # now use is in product + await auto_add_user_to_product_group( + client.app, user_id=user["id"], product_name=current_product.name + ) + + # invitation is invalid now with pytest.raises(InvalidInvitation): - await validate_invitation_url( - app=client.app, - guest_email=expected_invitation.guest, - invitation_url="https://server.com#register?invitation=1234", - ) + await validate_invitation_url(**kwargs) async def test_invalid_invitation_if_not_guest( client: TestClient, mock_invitations_service_http_api: AioResponsesMock, - expected_invitation: ApiInvitationContent, + fake_osparc_invitation: ApiInvitationContent, + current_product: Product, ): assert client.app - assert expected_invitation.guest != "unexpected_guest@email.me" + assert fake_osparc_invitation.guest != "unexpected_guest@email.me" with pytest.raises(InvalidInvitation): await validate_invitation_url( app=client.app, guest_email="unexpected_guest@email.me", invitation_url="https://server.com#register?invitation=1234", + current_product=current_product, ) diff --git a/services/web/server/tests/unit/with_dbs/03/invitations/test_login_registration_invitations.py b/services/web/server/tests/unit/with_dbs/03/invitations/test_login_registration_invitations.py index 17eb63c1153..60f24d39510 100644 --- a/services/web/server/tests/unit/with_dbs/03/invitations/test_login_registration_invitations.py +++ b/services/web/server/tests/unit/with_dbs/03/invitations/test_login_registration_invitations.py @@ -100,7 +100,7 @@ async def test_check_registration_invitation_and_get_email( client: TestClient, mocker: MockerFixture, mock_invitations_service_http_api: AioResponsesMock, - expected_invitation: ApiInvitationContent, + fake_osparc_invitation: ApiInvitationContent, ): assert client.app mocker.patch( @@ -118,4 +118,4 @@ async def test_check_registration_invitation_and_get_email( data, _ = await assert_status(response, web.HTTPOk) invitation = InvitationInfo.parse_obj(data) - assert invitation.email == expected_invitation.guest + assert invitation.email == fake_osparc_invitation.guest From 35a6ef3ee342e016d52bb3b8627a63492843dafe Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Fri, 17 Nov 2023 07:10:02 +0000 Subject: [PATCH 14/24] =?UTF-8?q?=E2=9A=B0=EF=B8=8F=20removing=20node=5Fri?= =?UTF-8?q?ghts=20module=20from=20director-v2=20(#5039)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../core/application.py | 3 - .../scheduler/_core/_events_utils.py | 203 ++++-------- .../modules/node_rights.py | 246 -------------- .../unit/with_dbs/test_modules_node_rights.py | 307 ------------------ 4 files changed, 71 insertions(+), 688 deletions(-) delete mode 100644 services/director-v2/src/simcore_service_director_v2/modules/node_rights.py delete mode 100644 services/director-v2/tests/unit/with_dbs/test_modules_node_rights.py diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index 55a02280053..0af0a5d67bc 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -25,7 +25,6 @@ director_v0, dynamic_services, dynamic_sidecar, - node_rights, osparc_variables_substitutions, rabbitmq, remote_debug, @@ -180,8 +179,6 @@ def init_app(settings: AppSettings | None = None) -> FastAPI: if settings.DIRECTOR_V2_RESOURCE_USAGE_TRACKER: resource_usage_tracker_client.setup(app) - node_rights.setup(app) - # setup app -- app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index e5c83523f6c..0722c8a6faf 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -2,7 +2,7 @@ import json import logging -from typing import Any, Final +from typing import Any from fastapi import FastAPI from models_library.projects_networks import ProjectsNetworks @@ -30,7 +30,6 @@ from .....core.dynamic_services_settings.scheduler import ( DynamicServicesSchedulerSettings, ) -from .....core.errors import NodeRightsAcquireError from .....core.settings import AppSettings from .....models.dynamic_services_scheduler import ( DockerContainerInspect, @@ -42,11 +41,6 @@ from ....db.repositories.projects import ProjectsRepository from ....db.repositories.projects_networks import ProjectsNetworksRepository from ....director_v0 import DirectorV0Client -from ....node_rights import ( - NodeRightsManager, - ResourceName, - node_resource_limits_enabled, -) from ...api_client import ( BaseClientHTTPError, SidecarsClient, @@ -67,13 +61,6 @@ _logger = logging.getLogger(__name__) -# Used to ensure no more that X services per node pull or push data -# Locking is applied when: -# - study is being opened (state and outputs are pulled) -# - study is being closed (state and outputs are saved) -RESOURCE_STATE_AND_INPUTS: Final[ResourceName] = "state_and_inputs" - - def get_director_v0_client(app: FastAPI) -> DirectorV0Client: return DirectorV0Client.instance(app) @@ -260,88 +247,63 @@ async def attempt_pod_removal_and_data_saving( app_settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER ) - async def _remove_containers_save_state_and_outputs() -> None: - sidecars_client: SidecarsClient = get_sidecars_client( - app, scheduler_data.node_uuid - ) + sidecars_client: SidecarsClient = get_sidecars_client(app, scheduler_data.node_uuid) - await service_remove_containers(app, scheduler_data.node_uuid, sidecars_client) + await service_remove_containers(app, scheduler_data.node_uuid, sidecars_client) - # only try to save the status if : - # - it is requested to save the state - # - the dynamic-sidecar has finished booting correctly + # only try to save the status if : + # - it is requested to save the state + # - the dynamic-sidecar has finished booting correctly - can_really_save: bool = False - if scheduler_data.dynamic_sidecar.service_removal_state.can_save: - # if node is not present in the workbench it makes no sense - # to try and save the data, nodeports will raise errors - # and sidecar will hang + can_really_save: bool = False + if scheduler_data.dynamic_sidecar.service_removal_state.can_save: + # if node is not present in the workbench it makes no sense + # to try and save the data, nodeports will raise errors + # and sidecar will hang - projects_repository: ProjectsRepository = get_repository( - app, ProjectsRepository - ) + projects_repository: ProjectsRepository = get_repository( + app, ProjectsRepository + ) - can_really_save = await projects_repository.is_node_present_in_workbench( - project_id=scheduler_data.project_id, node_uuid=scheduler_data.node_uuid - ) + can_really_save = await projects_repository.is_node_present_in_workbench( + project_id=scheduler_data.project_id, node_uuid=scheduler_data.node_uuid + ) - if can_really_save and scheduler_data.dynamic_sidecar.were_containers_created: - _logger.info("Calling into dynamic-sidecar to save: state and output ports") - try: - tasks = [ - service_push_outputs(app, scheduler_data.node_uuid, sidecars_client) - ] + if can_really_save and scheduler_data.dynamic_sidecar.were_containers_created: + _logger.info("Calling into dynamic-sidecar to save: state and output ports") + try: + tasks = [ + service_push_outputs(app, scheduler_data.node_uuid, sidecars_client) + ] - # When enabled no longer uploads state via nodeports - # It uses rclone mounted volumes for this task. - if not app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: - tasks.append( - service_save_state( - app, scheduler_data.node_uuid, sidecars_client - ) - ) - - await logged_gather(*tasks, max_concurrency=2) - scheduler_data.dynamic_sidecar.were_state_and_outputs_saved = True - - _logger.info("dynamic-sidecar saved: state and output ports") - except (BaseClientHTTPError, TaskClientResultError) as e: - _logger.error( # noqa: TRY400 - ( - "Could not contact dynamic-sidecar to save service " - "state or output ports %s\n%s" - ), - scheduler_data.service_name, - f"{e}", + # When enabled no longer uploads state via nodeports + # It uses rclone mounted volumes for this task. + if not app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: + tasks.append( + service_save_state(app, scheduler_data.node_uuid, sidecars_client) ) - # ensure dynamic-sidecar does not get removed - # user data can be manually saved and manual - # cleanup of the dynamic-sidecar is required - scheduler_data.dynamic_sidecar.wait_for_manual_intervention_after_error = ( - True - ) - raise + await logged_gather(*tasks, max_concurrency=2) + scheduler_data.dynamic_sidecar.were_state_and_outputs_saved = True + + _logger.info("dynamic-sidecar saved: state and output ports") + except (BaseClientHTTPError, TaskClientResultError) as e: + _logger.error( # noqa: TRY400 + ( + "Could not contact dynamic-sidecar to save service " + "state or output ports %s\n%s" + ), + scheduler_data.service_name, + f"{e}", + ) + # ensure dynamic-sidecar does not get removed + # user data can be manually saved and manual + # cleanup of the dynamic-sidecar is required - if node_resource_limits_enabled(app): - node_rights_manager = await NodeRightsManager.instance(app) - assert scheduler_data.dynamic_sidecar.docker_node_id # nosec - try: - async with node_rights_manager.acquire( - scheduler_data.dynamic_sidecar.docker_node_id, - resource_name=RESOURCE_STATE_AND_INPUTS, - ): - await _remove_containers_save_state_and_outputs() - except NodeRightsAcquireError: - # Next observation cycle, the service will try again - _logger.debug( - "Skip saving service state for %s. Docker node %s is busy. Will try later.", - scheduler_data.node_uuid, - scheduler_data.dynamic_sidecar.docker_node_id, + scheduler_data.dynamic_sidecar.wait_for_manual_intervention_after_error = ( + True ) - return - else: - await _remove_containers_save_state_and_outputs() + raise await service_remove_sidecar_proxy_docker_networks_and_volumes( TaskProgress.create(), app, scheduler_data.node_uuid, settings.SWARM_STACK_NAME @@ -456,55 +418,32 @@ async def prepare_services_environment( ) ) - async def _pull_outputs_and_state(): - tasks = [sidecars_client.pull_service_output_ports(dynamic_sidecar_endpoint)] - # When enabled no longer downloads state via nodeports - # S3 is used to store state paths - if not app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: - tasks.append( - sidecars_client.restore_service_state(dynamic_sidecar_endpoint) - ) + tasks = [sidecars_client.pull_service_output_ports(dynamic_sidecar_endpoint)] + # When enabled no longer downloads state via nodeports + # S3 is used to store state paths + if not app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: + tasks.append(sidecars_client.restore_service_state(dynamic_sidecar_endpoint)) - await logged_gather(*tasks, max_concurrency=2) + await logged_gather(*tasks, max_concurrency=2) - # inside this directory create the missing dirs, fetch those form the labels - director_v0_client: DirectorV0Client = get_director_v0_client(app) - simcore_service_labels: SimcoreServiceLabels = ( - await director_v0_client.get_service_labels( - service=ServiceKeyVersion( - key=scheduler_data.key, version=scheduler_data.version - ) + # inside this directory create the missing dirs, fetch those form the labels + director_v0_client: DirectorV0Client = get_director_v0_client(app) + simcore_service_labels: SimcoreServiceLabels = ( + await director_v0_client.get_service_labels( + service=ServiceKeyVersion( + key=scheduler_data.key, version=scheduler_data.version ) ) - service_outputs_labels = json.loads( - simcore_service_labels.dict().get("io.simcore.outputs", "{}") - ).get("outputs", {}) - _logger.debug( - "Creating dirs from service outputs labels: %s", - service_outputs_labels, - ) - await sidecars_client.service_outputs_create_dirs( - dynamic_sidecar_endpoint, service_outputs_labels - ) - - scheduler_data.dynamic_sidecar.is_service_environment_ready = True + ) + service_outputs_labels = json.loads( + simcore_service_labels.dict().get("io.simcore.outputs", "{}") + ).get("outputs", {}) + _logger.debug( + "Creating dirs from service outputs labels: %s", + service_outputs_labels, + ) + await sidecars_client.service_outputs_create_dirs( + dynamic_sidecar_endpoint, service_outputs_labels + ) - if node_resource_limits_enabled(app): - node_rights_manager = await NodeRightsManager.instance(app) - assert scheduler_data.dynamic_sidecar.docker_node_id # nosec - try: - async with node_rights_manager.acquire( - scheduler_data.dynamic_sidecar.docker_node_id, - resource_name=RESOURCE_STATE_AND_INPUTS, - ): - await _pull_outputs_and_state() - except NodeRightsAcquireError: - # Next observation cycle, the service will try again - _logger.debug( - "Skip saving service state for %s. Docker node %s is busy. Will try later.", - scheduler_data.node_uuid, - scheduler_data.dynamic_sidecar.docker_node_id, - ) - return - else: - await _pull_outputs_and_state() + scheduler_data.dynamic_sidecar.is_service_environment_ready = True diff --git a/services/director-v2/src/simcore_service_director_v2/modules/node_rights.py b/services/director-v2/src/simcore_service_director_v2/modules/node_rights.py deleted file mode 100644 index cbfe7d996bc..00000000000 --- a/services/director-v2/src/simcore_service_director_v2/modules/node_rights.py +++ /dev/null @@ -1,246 +0,0 @@ -import asyncio -import logging -from contextlib import asynccontextmanager, suppress -from dataclasses import dataclass -from typing import AsyncIterator - -from fastapi import FastAPI -from pydantic import NonNegativeInt, PositiveFloat, PositiveInt -from redis.asyncio.lock import Lock -from servicelib.redis import RedisClientSDK -from settings_library.redis import RedisDatabase, RedisSettings - -from ..core.dynamic_services_settings.scheduler import DynamicServicesSchedulerSettings -from ..core.errors import ConfigurationError, NodeRightsAcquireError - -DockerNodeId = str -ResourceName = str - - -logger = logging.getLogger(__name__) - - -def node_resource_limits_enabled(app: FastAPI) -> bool: - dynamic_sidecars_scheduler_settings: DynamicServicesSchedulerSettings = ( - app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER - ) - enabled: bool = ( - dynamic_sidecars_scheduler_settings.DYNAMIC_SIDECAR_DOCKER_NODE_RESOURCE_LIMITS_ENABLED - ) - return enabled - - -def setup(app: FastAPI): - async def on_startup() -> None: - if node_resource_limits_enabled(app): - app.state.node_rights_manager = await NodeRightsManager.create(app) - await app.state.node_rights_manager.redis_client_sdk.setup() - - async def on_shutdown() -> None: - if node_resource_limits_enabled(app): - node_rights_manager: NodeRightsManager = app.state.node_rights_manager - await node_rights_manager.close() - - app.add_event_handler("startup", on_startup) - app.add_event_handler("shutdown", on_shutdown) - - -class ExtendLock: - def __init__( - self, - lock: Lock, - timeout_s: PositiveFloat, - extend_interval_s: PositiveFloat, - ) -> None: - self.timeout_s: PositiveFloat = timeout_s - self.extend_interval_s: PositiveFloat = extend_interval_s - self._redis_lock: Lock = lock - self.task: asyncio.Task | None = asyncio.create_task( - self._extend_task(), name=f"{self.__class__.__name__}" - ) - - async def _extend_task(self) -> None: - while True: - await asyncio.sleep(self.extend_interval_s) - await self._redis_lock.extend(self.timeout_s, replace_ttl=True) - - @property - def name(self) -> str: - return f"{self._redis_lock.name!r}" - - async def initialize(self) -> None: - await self._redis_lock.do_reacquire() - - async def release(self) -> None: - await self._redis_lock.release() - - -# acquire the rights to use a docker swarm node -@dataclass -class NodeRightsManager: - """ - A `slot` is used to limit `resource` usage. It can be viewed as a token - which has to be returned, once the user finished using the `resource`. - - A slot can be reserved via the `acquire` context manger. If no - `NodeRightsAcquireError` is raised, the user is free to use - the locked `resource`. If an error is raised the - user must try again at a later time. - """ - - app: FastAPI - redis_client_sdk: RedisClientSDK - is_enabled: bool - lock_timeout_s: PositiveFloat - concurrent_resource_slots: PositiveInt - - @classmethod - async def create(cls, app: FastAPI) -> "NodeRightsManager": - redis_settings: RedisSettings = app.state.settings.REDIS - dynamic_sidecars_scheduler_settings: DynamicServicesSchedulerSettings = ( - app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER - ) - return cls( - app=app, - redis_client_sdk=RedisClientSDK( - redis_settings.build_redis_dsn(RedisDatabase.LOCKS) - ), - is_enabled=dynamic_sidecars_scheduler_settings.DYNAMIC_SIDECAR_DOCKER_NODE_RESOURCE_LIMITS_ENABLED, - concurrent_resource_slots=dynamic_sidecars_scheduler_settings.DYNAMIC_SIDECAR_DOCKER_NODE_CONCURRENT_RESOURCE_SLOTS, - lock_timeout_s=dynamic_sidecars_scheduler_settings.DYNAMIC_SIDECAR_DOCKER_NODE_SAVES_LOCK_TIMEOUT_S, - ) - - @classmethod - async def instance(cls, app: FastAPI) -> "NodeRightsManager": - if not hasattr(app.state, "node_rights_manager"): - raise ConfigurationError( - "RedisLockManager client is not available. Please check the configuration." - ) - node_rights_manager: NodeRightsManager = app.state.node_rights_manager - return node_rights_manager - - @classmethod - def _get_key(cls, docker_node_id: DockerNodeId, resource_name: ResourceName) -> str: - return f"{cls.__name__}.{docker_node_id}.{resource_name}.lock_slots" - - @classmethod - def _get_lock_name( - cls, - docker_node_id: DockerNodeId, - resource_name: ResourceName, - slot_index: NonNegativeInt, - ) -> str: - return f"{cls._get_key(docker_node_id, resource_name)}.{slot_index}" - - async def _get_node_slots( - self, docker_node_id: DockerNodeId, resource_name: ResourceName - ) -> int: - """ - get the total amount of slots available for the provided - resource on the node - """ - - node_slots_key = self._get_key(docker_node_id, resource_name) - slots: bytes | None = await self.redis_client_sdk.redis.get(node_slots_key) - if slots is not None: - return int(slots) - - await self.redis_client_sdk.redis.set( - node_slots_key, self.concurrent_resource_slots - ) - return self.concurrent_resource_slots - - @staticmethod - async def _release_extend_lock(extend_lock: ExtendLock) -> None: - """ - Releases a lock and all its related resources. - Cancels the extend_task - """ - - if extend_lock.task: - extend_lock.task.cancel() - with suppress(asyncio.CancelledError): - - async def _await_task(task: asyncio.Task) -> None: - await task - - # NOTE: When the extension task is awaited it sometimes blocks - # we can safely timeout the task and ignore the error. - # The **most probable* cause of the error is when the extend_task - # and the release are called at the same time. Some internal locking - # is involved and the task is blocked forever. - - # it should not take more than `extend_interval_s` to cancel task - try: - await asyncio.wait_for( - _await_task(extend_lock.task), - timeout=extend_lock.extend_interval_s * 2, - ) - except asyncio.TimeoutError: - logger.warning( - "Timed out while awaiting for cancellation of '%s'", - extend_lock.name, - ) - - extend_lock.task = None - logger.info("Lock '%s' released", extend_lock.name) - else: - # Below will appear in the logs only if the logs was released twice, - # in which case a `redis.exceptions.LockError: Cannot release an unlocked lock` - # will be raised. - # Otherwise there might be some issues. - logger.warning( - "Lock '%s' has no associated `extend_task`.", extend_lock.name - ) - - await extend_lock.release() - - @asynccontextmanager - async def acquire( - self, docker_node_id: DockerNodeId, *, resource_name: ResourceName - ) -> AsyncIterator[ExtendLock]: - """ - Context manger to helo with acquire and release. If it is not possible - - raises: `NodeRightsAcquireError` if the lock was not acquired. - """ - slots = await self._get_node_slots(docker_node_id, resource_name) - acquired_lock: Lock | None = None - for slot in range(slots): - node_lock_name = self._get_lock_name(docker_node_id, resource_name, slot) - - lock = self.redis_client_sdk.redis.lock( - name=node_lock_name, timeout=self.lock_timeout_s - ) - lock_acquired = await lock.acquire(blocking=False) - - if lock_acquired: - acquired_lock = lock - logger.debug("Acquired %s/%s named '%s'", slot + 1, slots, lock.name) - break - - if acquired_lock is None: - raise NodeRightsAcquireError(docker_node_id=docker_node_id, slots=slots) - - # In order to avoid deadlock situations, where resources are not being - # released, a lock with a timeout will be acquired. - - # When the lock is acquired a background task which extends its - # validity will also be created. - - # Since the lifecycle of the extend task is tied to the one of the lock - # the task reference is attached to the lock. - extend_lock = ExtendLock( - lock=acquired_lock, - timeout_s=self.lock_timeout_s, - extend_interval_s=self.lock_timeout_s / 2, - ) - await extend_lock.initialize() - - try: - yield extend_lock - finally: - await self._release_extend_lock(extend_lock) - - async def close(self) -> None: - await self.redis_client_sdk.shutdown() diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_node_rights.py b/services/director-v2/tests/unit/with_dbs/test_modules_node_rights.py deleted file mode 100644 index d852c2f3c39..00000000000 --- a/services/director-v2/tests/unit/with_dbs/test_modules_node_rights.py +++ /dev/null @@ -1,307 +0,0 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=protected-access - -import asyncio -from asyncio import CancelledError, Task -from contextlib import suppress -from typing import Any, AsyncIterable, Final - -import pytest -from asgi_lifespan import LifespanManager -from faker import Faker -from fastapi import FastAPI -from pydantic import PositiveFloat -from pytest import MonkeyPatch -from redis.exceptions import LockError, LockNotOwnedError -from settings_library.redis import RedisSettings -from simcore_service_director_v2.core.errors import NodeRightsAcquireError -from simcore_service_director_v2.core.settings import AppSettings -from simcore_service_director_v2.modules import node_rights -from simcore_service_director_v2.modules.node_rights import ( - DockerNodeId, - ExtendLock, - NodeRightsManager, - ResourceName, -) - -pytest_simcore_core_services_selection = [ - "redis", -] - - -# UTILS -TEST_RESOURCE: Final[ResourceName] = "a_resource" - - -async def _assert_lock_acquired_and_released( - node_rights_manager: NodeRightsManager, - docker_node_id: DockerNodeId, - resource_name: ResourceName, - *, - sleep_before_release: PositiveFloat, -) -> ExtendLock: - async with node_rights_manager.acquire( - docker_node_id, resource_name=resource_name - ) as extend_lock: - assert await extend_lock._redis_lock.locked() is True - assert await extend_lock._redis_lock.owned() is True - - # task is running and not cancelled - assert extend_lock.task - assert extend_lock.task.done() is False - assert extend_lock.task.cancelled() is False - - await asyncio.sleep(sleep_before_release) - - # task was canceled and lock is unlocked and not owned - assert extend_lock.task is None - assert await extend_lock._redis_lock.locked() is False - assert await extend_lock._redis_lock.owned() is False - - return extend_lock - - -@pytest.fixture -async def minimal_app( - project_env_devel_environment: dict[str, Any], - redis_settings: RedisSettings, - monkeypatch: MonkeyPatch, -) -> AsyncIterable[FastAPI]: - monkeypatch.setenv("REDIS_HOST", redis_settings.REDIS_HOST) - monkeypatch.setenv("REDIS_PORT", f"{redis_settings.REDIS_PORT}") - monkeypatch.setenv("DYNAMIC_SIDECAR_DOCKER_NODE_RESOURCE_LIMITS_ENABLED", "true") - - app = FastAPI() - - # add expected redis_settings - app.state.settings = AppSettings.create_from_envs() - - # setup redis module - node_rights.setup(app) - - async with LifespanManager(app): - yield app - - -@pytest.fixture -async def node_rights_manager(minimal_app: FastAPI) -> AsyncIterable[NodeRightsManager]: - redis_lock_manger = await NodeRightsManager.instance(minimal_app) - await redis_lock_manger.redis_client_sdk.redis.flushall() - yield redis_lock_manger - await redis_lock_manger.redis_client_sdk.redis.flushall() - - -@pytest.fixture -def docker_node_id(faker: Faker) -> str: - return faker.uuid4() - - -async def test_redis_lock_working_as_expected( - node_rights_manager: NodeRightsManager, docker_node_id -) -> None: - lock = node_rights_manager.redis_client_sdk.redis.lock(docker_node_id) - - lock_acquired = await lock.acquire(blocking=False) - assert lock_acquired - assert await lock.locked() is True - - await lock.release() - assert await lock.locked() is False - - with pytest.raises(LockError): - await lock.release() - - -async def test_redis_two_lock_instances( - node_rights_manager: NodeRightsManager, docker_node_id: DockerNodeId -) -> None: - # NOTE: this test show cases how the locks work - # you have to acquire the lock from the same istance - # in order to avoid tricky situations - - lock = node_rights_manager.redis_client_sdk.redis.lock(docker_node_id) - - lock_acquired = await lock.acquire(blocking=False) - assert lock_acquired - assert await lock.locked() is True - - # we get a different instance - second_lock = node_rights_manager.redis_client_sdk.redis.lock(docker_node_id) - assert await second_lock.locked() is True - - # cannot release lock form different instance! - with pytest.raises(LockError): - await second_lock.release() - - assert await lock.locked() is True - # NOTE: this is confusing! One woudl expect the second lock to be unlocked - # but it actually is True - assert await second_lock.locked() is True - - await lock.release() - assert await lock.locked() is False - # NOTE: apparently it mirrors the first lock instance! - assert await second_lock.locked() is False - - -async def test_lock_extend_task_life_cycle( - node_rights_manager: NodeRightsManager, docker_node_id: DockerNodeId -) -> None: - extend_lock = await _assert_lock_acquired_and_released( - node_rights_manager, docker_node_id, TEST_RESOURCE, sleep_before_release=0 - ) - - # try to cancel again will not work! - with pytest.raises(LockError): - await node_rights_manager._release_extend_lock(extend_lock) - - -@pytest.mark.parametrize("resource_count", [1, 2, 10]) -async def test_no_more_locks_can_be_acquired( - node_rights_manager: NodeRightsManager, - docker_node_id: DockerNodeId, - resource_count: int, -) -> None: - # acquire all available locks - - async def _acquire_tasks(resource_name: ResourceName) -> tuple[int, list[Task]]: - slots = await node_rights_manager._get_node_slots(docker_node_id, resource_name) - assert slots == node_rights_manager.concurrent_resource_slots - - tasks = [ - asyncio.create_task( - _assert_lock_acquired_and_released( - node_rights_manager, - docker_node_id, - resource_name, - sleep_before_release=1, - ) - ) - for _ in range(slots) - ] - - # ensure locks are acquired - await asyncio.sleep(0.25) - - # no slots available - with pytest.raises(NodeRightsAcquireError) as exec_info: - await _assert_lock_acquired_and_released( - node_rights_manager, - docker_node_id, - resource_name, - sleep_before_release=0, - ) - assert ( - f"{exec_info.value}" - == f"Could not acquire a lock for {docker_node_id} since all {slots} slots are used." - ) - return slots, tasks - - tasks = [] - for r in range(resource_count): - resource_name = f"resource_{r}" - used_slots_r1, resource_tasks = await _acquire_tasks(resource_name) - assert len(resource_tasks) == used_slots_r1 - tasks += resource_tasks - - # wait for tasks to be released - await asyncio.gather(*tasks) - - -@pytest.mark.skip( - reason="fails will be removed by @GitHK, see https://github.com/ITISFoundation/osparc-simcore/issues/4405" -) -@pytest.mark.flaky(max_runs=3) -@pytest.mark.parametrize( - "locks_per_node", - [ - 4, - 10, - 100, - ], -) -async def test_acquire_all_available_node_locks_stress_test( - node_rights_manager: NodeRightsManager, - docker_node_id: DockerNodeId, - locks_per_node: int, -) -> None: - # NOTE: this test is designed to spot if there are any issues when - # acquiring and releasing locks in parallel with high concurrency - - # adds more stress with lower lock_timeout_s - node_rights_manager.lock_timeout_s = 0.5 - - node_rights_manager.concurrent_resource_slots = locks_per_node - - total_node_slots = await node_rights_manager._get_node_slots( - docker_node_id, TEST_RESOURCE - ) - assert total_node_slots == locks_per_node - - # THE extend task is causing things to hang!!! that is what is wrong here! - await asyncio.gather( - *[ - _assert_lock_acquired_and_released( - node_rights_manager, - docker_node_id, - TEST_RESOURCE, - sleep_before_release=node_rights_manager.lock_timeout_s * 2, - ) - for _ in range(total_node_slots) - ] - ) - print("all locks have been released") - - -async def test_lock_extension_expiration( - node_rights_manager: NodeRightsManager, docker_node_id: DockerNodeId -) -> None: - SHORT_INTERVAL = 0.10 - - node_rights_manager.lock_timeout_s = SHORT_INTERVAL - node_rights_manager.concurrent_resource_slots = 1 - - with pytest.raises(LockNotOwnedError, match="Cannot release a lock") as err_info: - async with node_rights_manager.acquire( - docker_node_id, resource_name=TEST_RESOURCE - ) as extend_lock: - # lock should have been extended at least 2 times - # and should still be locked - await asyncio.sleep(SHORT_INTERVAL * 4) - assert await extend_lock._redis_lock.locked() is True - assert await extend_lock._redis_lock.owned() is True - - # emulating process died (equivalent to no further renews) - assert extend_lock.task - extend_lock.task.cancel() - with suppress(CancelledError): - await extend_lock.task - - # lock is expected to be unlocked after timeout interval - await asyncio.sleep(node_rights_manager.lock_timeout_s) - assert await extend_lock._redis_lock.locked() is False - assert await extend_lock._redis_lock.owned() is False - - # the error must be raised by the release method inside the ExtendLock - assert ( - err_info.traceback[-2].statement.__str__().strip() - == "await self._redis_lock.release()" - ) - - -async def test_lock_raises_error_if_no_slots_are_available( - node_rights_manager: NodeRightsManager, docker_node_id: DockerNodeId -) -> None: - node_rights_manager.concurrent_resource_slots = 0 - - with pytest.raises(NodeRightsAcquireError) as err_info: - async with node_rights_manager.acquire( - docker_node_id, resource_name=TEST_RESOURCE - ): - pass - - assert f"{err_info.value}" == ( - f"Could not acquire a lock for {docker_node_id} since all " - f"{node_rights_manager.concurrent_resource_slots} slots are used." - ) From 8f207beddd568acdd3b7190d561a57062f8cb784 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 <60785969+matusdrobuliak66@users.noreply.github.com> Date: Fri, 17 Nov 2023 16:19:27 +0100 Subject: [PATCH 15/24] =?UTF-8?q?=E2=9C=A8=20Is922/add=20autorecharge=20fu?= =?UTF-8?q?nctionality=20(#5036)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> --- .env-devel | 1 + .../src/models_library/products.py | 17 +- .../src/models_library/rabbitmq_messages.py | 2 + .../test_models_payments_transactions.py | 7 +- .../pytest_simcore/helpers/rawdata_fakers.py | 29 ++ services/docker-compose.yml | 4 + .../api/rpc/_payments_methods.py | 4 +- .../simcore_service_payments/core/errors.py | 4 + .../simcore_service_payments/core/settings.py | 20 ++ .../db/auto_recharge_repo.py | 76 ++++ .../db/payments_methods_repo.py | 18 +- .../db/payments_transactions_repo.py | 44 ++- .../services/auto_recharge.py | 122 +++++++ .../services/auto_recharge_process_message.py | 160 ++++++++- .../services/payments.py | 3 + .../services/payments_methods.py | 2 +- .../services/rabbitmq.py | 11 + services/payments/tests/unit/conftest.py | 63 +++- .../tests/unit/test_rpc_payments_methods.py | 1 + .../test_services_auto_recharge_listener.py | 333 +++++++++++++++++- .../test_services_resource_usage_tracker.py | 64 +--- .../resource_tracker_utils.py | 1 + services/web/server/setup.cfg | 2 +- .../products/_api.py | 28 +- .../products/_rpc.py | 29 ++ .../simcore_service_webserver/products/api.py | 4 +- .../products/errors.py | 14 + .../products/plugin.py | 8 +- .../wallets/_handlers.py | 5 + .../wallets/_payments_handlers.py | 36 +- .../with_dbs/03/products/test_products_rpc.py | 98 ++++++ 31 files changed, 1102 insertions(+), 108 deletions(-) create mode 100644 services/payments/src/simcore_service_payments/db/auto_recharge_repo.py create mode 100644 services/payments/src/simcore_service_payments/services/auto_recharge.py create mode 100644 services/web/server/src/simcore_service_webserver/products/_rpc.py create mode 100644 services/web/server/src/simcore_service_webserver/products/errors.py create mode 100644 services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py diff --git a/.env-devel b/.env-devel index a2fe30524ce..d4b58c5796d 100644 --- a/.env-devel +++ b/.env-devel @@ -76,6 +76,7 @@ PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT=10000 PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT=100.0 PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS=100 +PAYMENTS_AUTORECHARGE_ENABLED=1 PAYMENTS_FAKE_COMPLETION_DELAY_SEC=10 PAYMENTS_FAKE_COMPLETION=0 PAYMENTS_GATEWAY_API_SECRET=replace-with-api-secret diff --git a/packages/models-library/src/models_library/products.py b/packages/models-library/src/models_library/products.py index 5fdb2a59209..fdd18ff3f3b 100644 --- a/packages/models-library/src/models_library/products.py +++ b/packages/models-library/src/models_library/products.py @@ -1,3 +1,18 @@ -from typing import TypeAlias +from decimal import Decimal +from typing import Any, ClassVar, TypeAlias + +from pydantic import BaseModel, Field ProductName: TypeAlias = str + + +class CreditResultGet(BaseModel): + product_name: ProductName + credit_amount: Decimal = Field(..., description="") + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + {"product_name": "s4l", "credit_amount": Decimal(15.5)}, + ] + } diff --git a/packages/models-library/src/models_library/rabbitmq_messages.py b/packages/models-library/src/models_library/rabbitmq_messages.py index a32a0b8dbe3..9d1731731d5 100644 --- a/packages/models-library/src/models_library/rabbitmq_messages.py +++ b/packages/models-library/src/models_library/rabbitmq_messages.py @@ -6,6 +6,7 @@ from typing import Any, Literal, TypeAlias import arrow +from models_library.products import ProductName from pydantic import BaseModel, Field from pydantic.types import NonNegativeFloat @@ -264,6 +265,7 @@ class WalletCreditsMessage(RabbitMessageBase): ) wallet_id: WalletID credits: Decimal + product_name: ProductName def routing_key(self) -> str | None: return f"{self.wallet_id}" diff --git a/packages/postgres-database/tests/test_models_payments_transactions.py b/packages/postgres-database/tests/test_models_payments_transactions.py index df6adee83bb..278dbb62fa5 100644 --- a/packages/postgres-database/tests/test_models_payments_transactions.py +++ b/packages/postgres-database/tests/test_models_payments_transactions.py @@ -47,7 +47,9 @@ def init_transaction(connection: SAConnection): async def _init(payment_id: str): # get payment_id from payment-gateway values = random_payment_transaction(payment_id=payment_id) - + # remove states + values.pop("state") + values.pop("completed_at") # init successful: set timestamp values["initiated_at"] = utcnow() @@ -178,6 +180,9 @@ async def _go(expected_total=5): payment_ids = [] for _ in range(expected_total): values = random_payment_transaction(user_id=user_id) + # remove states + values.pop("state") + values.pop("completed_at") payment_id = await insert_init_payment_transaction(connection, **values) assert payment_id payment_ids.append(payment_id) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py index 036d0a9c126..22fb55a620d 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py @@ -23,6 +23,10 @@ from faker import Faker from simcore_postgres_database.models.api_keys import api_keys from simcore_postgres_database.models.comp_pipeline import StateType +from simcore_postgres_database.models.payments_methods import InitPromptAckFlowState +from simcore_postgres_database.models.payments_transactions import ( + PaymentTransactionState, +) from simcore_postgres_database.models.products import products from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.users import users @@ -199,6 +203,8 @@ def random_payment_method( "user_id": FAKE.pyint(), "wallet_id": FAKE.pyint(), "initiated_at": utcnow(), + "state": InitPromptAckFlowState.PENDING, + "completed_at": None, } # state is not added on purpose assert set(data.keys()).issubset({c.name for c in payments_methods.columns}) @@ -226,6 +232,8 @@ def random_payment_transaction( "wallet_id": 1, "comment": "Free starting credits", "initiated_at": utcnow(), + "state": PaymentTransactionState.PENDING, + "completed_at": None, } # state is not added on purpose assert set(data.keys()).issubset({c.name for c in payments_transactions.columns}) @@ -234,6 +242,27 @@ def random_payment_transaction( return data +def random_payment_autorecharge( + primary_payment_method_id: str = FAKE.uuid4(), + **overrides, +) -> dict[str, Any]: + from simcore_postgres_database.models.payments_autorecharge import ( + payments_autorecharge, + ) + + data = { + "wallet_id": FAKE.pyint(), + "enabled": True, + "primary_payment_method_id": primary_payment_method_id, + "top_up_amount_in_usd": 100, + "monthly_limit_in_usd": 1000, + } + assert set(data.keys()).issubset({c.name for c in payments_autorecharge.columns}) + + data.update(overrides) + return data + + def random_api_key(product_name: str, user_id: int, **overrides) -> dict[str, Any]: data = { "display_name": FAKE.word(), diff --git a/services/docker-compose.yml b/services/docker-compose.yml index c97297d37d7..35f8529b826 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -270,6 +270,10 @@ services: - LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED} - PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES=${PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES} - PAYMENTS_ACCESS_TOKEN_SECRET_KEY=${PAYMENTS_ACCESS_TOKEN_SECRET_KEY} + - PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT=${PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT} + - PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT=${PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT} + - PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS=${PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS} + - PAYMENTS_AUTORECHARGE_ENABLED=${PAYMENTS_AUTORECHARGE_ENABLED} - PAYMENTS_GATEWAY_API_SECRET=${PAYMENTS_GATEWAY_API_SECRET} - PAYMENTS_GATEWAY_URL=${PAYMENTS_GATEWAY_URL} - PAYMENTS_LOGLEVEL=${PAYMENTS_LOGLEVEL} diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py index ef531a5b585..18262bacae1 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py @@ -17,6 +17,7 @@ from ...db.payments_transactions_repo import PaymentsTransactionsRepo from ...services import payments, payments_methods from ...services.payments_gateway import PaymentsGatewayApi +from ...services.resource_usage_tracker import ResourceUsageTrackerApi _logger = logging.getLogger(__name__) @@ -69,7 +70,7 @@ async def list_payment_methods( user_id: UserID, wallet_id: WalletID, ) -> list[PaymentMethodGet]: - return await payments_methods.list_payments_methods( + return await payments_methods.list_payment_methods( gateway=PaymentsGatewayApi.get_from_app_state(app), repo=PaymentsMethodsRepo(db_engine=app.state.engine), user_id=user_id, @@ -128,6 +129,7 @@ async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-a ): return await payments.pay_with_payment_method( gateway=PaymentsGatewayApi.get_from_app_state(app), + rut=ResourceUsageTrackerApi.get_from_app_state(app), repo_transactions=PaymentsTransactionsRepo(db_engine=app.state.engine), repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine), payment_method_id=payment_method_id, diff --git a/services/payments/src/simcore_service_payments/core/errors.py b/services/payments/src/simcore_service_payments/core/errors.py index d7133697760..1bc228f173f 100644 --- a/services/payments/src/simcore_service_payments/core/errors.py +++ b/services/payments/src/simcore_service_payments/core/errors.py @@ -33,3 +33,7 @@ class PaymentMethodAlreadyAckedError(PaymentsMethodsError): class PaymentMethodUniqueViolationError(PaymentsMethodsError): msg_template = "Payment method '{payment_method_id}' aready exists" + + +class InvalidPaymentMethodError(PaymentsMethodsError): + msg_template = "Invalid payment method '{payment_method_id}'" diff --git a/services/payments/src/simcore_service_payments/core/settings.py b/services/payments/src/simcore_service_payments/core/settings.py index f0632fef47c..ada7260d2bb 100644 --- a/services/payments/src/simcore_service_payments/core/settings.py +++ b/services/payments/src/simcore_service_payments/core/settings.py @@ -1,6 +1,7 @@ from functools import cached_property from typing import cast +from models_library.basic_types import NonNegativeDecimal from pydantic import Field, HttpUrl, PositiveFloat, SecretStr, parse_obj_as, validator from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag @@ -77,6 +78,25 @@ class ApplicationSettings(_BaseApplicationSettings): ) PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES: PositiveFloat = Field(default=30) + PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: NonNegativeDecimal = Field( + default=100, + description="Minimum balance in credits to top-up for auto-recharge", + ) + + PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT: NonNegativeDecimal = Field( + default=100, + description="Default value in USD on the amount to top-up for auto-recharge (`top_up_amount_in_usd`)", + ) + + PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT: NonNegativeDecimal | None = Field( + default=10000, + description="Default value in USD for the montly limit for auto-recharge (`monthly_limit_in_usd`)", + ) + PAYMENTS_AUTORECHARGE_ENABLED: bool = Field( + default=False, + description="Based on this variable is the auto recharge functionality in Payment service enabled", + ) + PAYMENTS_RABBITMQ: RabbitSettings = Field( auto_default_from_env=True, description="settings for service/rabbitmq" ) diff --git a/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py b/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py new file mode 100644 index 00000000000..d4bc5634e54 --- /dev/null +++ b/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py @@ -0,0 +1,76 @@ +from typing import TypeAlias + +from models_library.api_schemas_webserver.wallets import PaymentMethodID +from models_library.basic_types import NonNegativeDecimal +from models_library.users import UserID +from models_library.wallets import WalletID +from pydantic import BaseModel, PositiveInt +from simcore_postgres_database.utils_payments_autorecharge import AutoRechargeStmts + +from ..core.errors import InvalidPaymentMethodError +from .base import BaseRepository + +AutoRechargeID: TypeAlias = PositiveInt + + +class PaymentsAutorechargeDB(BaseModel): + wallet_id: WalletID + enabled: bool + primary_payment_method_id: PaymentMethodID + top_up_amount_in_usd: NonNegativeDecimal + monthly_limit_in_usd: NonNegativeDecimal | None + + class Config: + orm_mode = True + + +class AutoRechargeRepo(BaseRepository): + async def get_wallet_autorecharge( + self, + wallet_id: WalletID, + ) -> PaymentsAutorechargeDB | None: + """Annotates init-payment transaction + Raises: + PaymentAlreadyExistsError + """ + + async with self.db_engine.begin() as conn: + stmt = AutoRechargeStmts.get_wallet_autorecharge(wallet_id) + result = await conn.execute(stmt) + row = result.first() + return PaymentsAutorechargeDB.from_orm(row) if row else None + + async def replace_wallet_autorecharge( + self, + user_id: UserID, + wallet_id: WalletID, + new: PaymentsAutorechargeDB, + ) -> PaymentsAutorechargeDB: + """ + Raises: + InvalidPaymentMethodError: if `new` includes some invalid 'primary_payment_method_id' + + """ + async with self.db_engine.begin() as conn: + stmt = AutoRechargeStmts.is_valid_payment_method( + user_id=user_id, + wallet_id=new.wallet_id, + payment_method_id=new.primary_payment_method_id, + ) + + if await conn.scalar(stmt) != new.primary_payment_method_id: + raise InvalidPaymentMethodError( + payment_method_id=new.primary_payment_method_id + ) + + stmt = AutoRechargeStmts.upsert_wallet_autorecharge( + wallet_id=wallet_id, + enabled=new.enabled, + primary_payment_method_id=new.primary_payment_method_id, + top_up_amount_in_usd=new.top_up_amount_in_usd, + monthly_limit_in_usd=new.monthly_limit_in_usd, + ) + result = await conn.execute(stmt) + row = result.first() + assert row # nosec + return PaymentsAutorechargeDB.from_orm(row) diff --git a/services/payments/src/simcore_service_payments/db/payments_methods_repo.py b/services/payments/src/simcore_service_payments/db/payments_methods_repo.py index 26a901abbfd..f1c3f791efd 100644 --- a/services/payments/src/simcore_service_payments/db/payments_methods_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_methods_repo.py @@ -102,7 +102,6 @@ async def insert_payment_method( completion_state: InitPromptAckFlowState, state_message: str | None, ) -> PaymentsMethodsDB: - await self.insert_init_payment_method( payment_method_id, user_id=user_id, @@ -135,6 +134,23 @@ async def list_user_payment_methods( rows = result.fetchall() or [] return parse_obj_as(list[PaymentsMethodsDB], rows) + async def get_payment_method_by_id( + self, + payment_method_id: PaymentMethodID, + ) -> PaymentsMethodsDB: + async with self.db_engine.begin() as conn: + result = await conn.execute( + payments_methods.select().where( + (payments_methods.c.payment_method_id == payment_method_id) + & (payments_methods.c.state == InitPromptAckFlowState.SUCCESS) + ) + ) + row = result.first() + if row is None: + raise PaymentMethodNotFoundError(payment_method_id=payment_method_id) + + return PaymentsMethodsDB.from_orm(row) + async def get_payment_method( self, payment_method_id: PaymentMethodID, diff --git a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py index 8cfde93980d..84ee8b8543c 100644 --- a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime, timezone from decimal import Decimal import sqlalchemy as sa @@ -33,7 +33,7 @@ async def insert_init_payment_transaction( user_email: str, wallet_id: WalletID, comment: str | None, - initiated_at: datetime.datetime, + initiated_at: datetime, ) -> PaymentID: """Annotates init-payment transaction @@ -177,3 +177,43 @@ async def get_payment_transaction( ) row = result.fetchone() return PaymentsTransactionsDB.from_orm(row) if row else None + + async def sum_current_month_dollars(self, *, wallet_id: WalletID) -> Decimal: + _current_timestamp = datetime.now(tz=timezone.utc) + _current_month_start_timestamp = _current_timestamp.replace( + day=1, hour=0, minute=0, second=0, microsecond=0 + ) + + async with self.db_engine.begin() as conn: + sum_stmt = sa.select( + sa.func.sum(payments_transactions.c.price_dollars) + ).where( + (payments_transactions.c.wallet_id == wallet_id) + & ( + payments_transactions.c.state.in_( + [ + PaymentTransactionState.SUCCESS, + ] + ) + ) + & ( + payments_transactions.c.completed_at + >= _current_month_start_timestamp + ) + ) + result = await conn.execute(sum_stmt) + row = result.first() + return Decimal(0) if row is None or row[0] is None else Decimal(row[0]) + + async def get_last_payment_transaction_for_wallet( + self, *, wallet_id: WalletID + ) -> PaymentsTransactionsDB | None: + async with self.db_engine.begin() as connection: + result = await connection.execute( + payments_transactions.select() + .where(payments_transactions.c.wallet_id == wallet_id) + .order_by(payments_transactions.c.initiated_at.desc()) + .limit(1) + ) + row = result.fetchone() + return PaymentsTransactionsDB.from_orm(row) if row else None diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge.py b/services/payments/src/simcore_service_payments/services/auto_recharge.py new file mode 100644 index 00000000000..aa72715816d --- /dev/null +++ b/services/payments/src/simcore_service_payments/services/auto_recharge.py @@ -0,0 +1,122 @@ +import logging + +from fastapi import FastAPI +from models_library.api_schemas_webserver.wallets import ( + GetWalletAutoRecharge, + ReplaceWalletAutoRecharge, +) +from models_library.basic_types import NonNegativeDecimal +from models_library.users import UserID +from models_library.wallets import WalletID + +from ..core.settings import ApplicationSettings +from ..db.auto_recharge_repo import AutoRechargeRepo, PaymentsAutorechargeDB +from ..db.payments_methods_repo import PaymentsMethodsRepo + +_logger = logging.getLogger(__name__) + + +def _from_db_to_api_model( + db_model: PaymentsAutorechargeDB, min_balance_in_credits: NonNegativeDecimal +) -> GetWalletAutoRecharge: + return GetWalletAutoRecharge( + enabled=db_model.enabled, + payment_method_id=db_model.primary_payment_method_id, + min_balance_in_credits=min_balance_in_credits, + top_up_amount_in_usd=db_model.top_up_amount_in_usd, + monthly_limit_in_usd=db_model.monthly_limit_in_usd, + ) + + +def _from_api_to_db_model( + wallet_id: WalletID, api_model: ReplaceWalletAutoRecharge +) -> PaymentsAutorechargeDB: + return PaymentsAutorechargeDB( + wallet_id=wallet_id, + enabled=api_model.enabled, + primary_payment_method_id=api_model.payment_method_id, + top_up_amount_in_usd=api_model.top_up_amount_in_usd, + monthly_limit_in_usd=api_model.monthly_limit_in_usd, + ) + + +# +# payment-autorecharge api +# + +_NEWEST = 0 + + +async def get_wallet_auto_recharge( + settings: ApplicationSettings, + auto_recharge_repo: AutoRechargeRepo, + *, + wallet_id: WalletID, +) -> GetWalletAutoRecharge | None: + payments_autorecharge_db: PaymentsAutorechargeDB | None = ( + await auto_recharge_repo.get_wallet_autorecharge(wallet_id=wallet_id) + ) + if payments_autorecharge_db: + return GetWalletAutoRecharge( + enabled=payments_autorecharge_db.enabled, + payment_method_id=payments_autorecharge_db.primary_payment_method_id, + min_balance_in_credits=settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS, + top_up_amount_in_usd=payments_autorecharge_db.top_up_amount_in_usd, + monthly_limit_in_usd=payments_autorecharge_db.monthly_limit_in_usd, + ) + return None + + +async def get_user_wallet_payment_autorecharge_with_default( + app: FastAPI, + auto_recharge_repo: AutoRechargeRepo, + payments_method_repo: PaymentsMethodsRepo, + *, + user_id: UserID, + wallet_id: WalletID, +) -> GetWalletAutoRecharge: + settings: ApplicationSettings = app.state.settings + + wallet_autorecharge = await get_wallet_auto_recharge( + settings, + auto_recharge_repo, + wallet_id=wallet_id, + ) + if not wallet_autorecharge: + payment_method_id = None + wallet_payment_methods = await payments_method_repo.list_user_payment_methods( + user_id=user_id, + wallet_id=wallet_id, + ) + if wallet_payment_methods: + payment_method_id = wallet_payment_methods[_NEWEST].payment_method_id + + return GetWalletAutoRecharge( + enabled=False, + payment_method_id=payment_method_id, + min_balance_in_credits=settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS, + top_up_amount_in_usd=settings.PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT, + monthly_limit_in_usd=settings.PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT, + ) + return wallet_autorecharge + + +async def replace_wallet_payment_autorecharge( + app: FastAPI, + repo: AutoRechargeRepo, + *, + user_id: UserID, + wallet_id: WalletID, + new: ReplaceWalletAutoRecharge, +) -> GetWalletAutoRecharge: + settings: ApplicationSettings = app.state.settings + got: PaymentsAutorechargeDB = await repo.replace_wallet_autorecharge( + user_id=user_id, + wallet_id=wallet_id, + new=_from_api_to_db_model(wallet_id, new), + ) + + return _from_db_to_api_model( + got, + min_balance_in_credits=settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS, + ) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index 8480063b19b..a4dae4eefe7 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -1,20 +1,168 @@ import logging +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from typing import cast from fastapi import FastAPI +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.api_schemas_webserver.wallets import ( + GetWalletAutoRecharge, + PaymentMethodID, +) +from models_library.basic_types import NonNegativeDecimal +from models_library.products import CreditResultGet +from models_library.rabbitmq_basic_types import RPCMethodName from models_library.rabbitmq_messages import WalletCreditsMessage -from pydantic import parse_raw_as +from models_library.wallets import WalletID +from pydantic import EmailStr, parse_obj_as, parse_raw_as +from simcore_service_payments.db.auto_recharge_repo import AutoRechargeRepo +from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo +from simcore_service_payments.db.payments_transactions_repo import ( + PaymentsTransactionsRepo, +) +from simcore_service_payments.models.db import PaymentsMethodsDB +from simcore_service_payments.services.resource_usage_tracker import ( + ResourceUsageTrackerApi, +) + +from ..core.settings import ApplicationSettings +from .auto_recharge import get_wallet_auto_recharge +from .payments import pay_with_payment_method +from .payments_gateway import PaymentsGatewayApi +from .rabbitmq import get_rabbitmq_rpc_client _logger = logging.getLogger(__name__) async def process_message(app: FastAPI, data: bytes) -> bool: - assert app # nosec rabbit_message = parse_raw_as(WalletCreditsMessage, data) _logger.debug("Process msg: %s", rabbit_message) - # 1. Check if auto-recharge functionality is ON for wallet_id - # 2. Check if wallet credits are bellow the threshold - # 3. Get Payment method - # 4. Pay with payment method + settings: ApplicationSettings = app.state.settings + + # Step 1: Check if wallet credits are above the threshold + if await _check_wallet_credits_above_threshold( + settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS, rabbit_message.credits + ): + return True # We do not auto recharge + + # Step 2: Check auto-recharge conditions + _auto_recharge_repo: AutoRechargeRepo = AutoRechargeRepo(db_engine=app.state.engine) + wallet_auto_recharge: GetWalletAutoRecharge | None = await get_wallet_auto_recharge( + settings, _auto_recharge_repo, wallet_id=rabbit_message.wallet_id + ) + if await _check_autorecharge_conditions_not_met(wallet_auto_recharge): + return True # We do not auto recharge + assert wallet_auto_recharge is not None # nosec + assert wallet_auto_recharge.payment_method_id is not None # nosec + + # Step 3: Get Payment method + _payments_repo = PaymentsMethodsRepo(db_engine=app.state.engine) + payment_method_db = await _payments_repo.get_payment_method_by_id( + payment_method_id=wallet_auto_recharge.payment_method_id + ) + + # Step 4: Check spending limits + _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + if await _exceeds_monthly_limit( + _payments_transactions_repo, rabbit_message.wallet_id, wallet_auto_recharge + ): + return True # We do not auto recharge + # Step 5: Check last top-up time + if await _recently_topped_up(_payments_transactions_repo, rabbit_message.wallet_id): + return True # We do not auto recharge + + # Step 6: Perform auto-recharge + if settings.PAYMENTS_AUTORECHARGE_ENABLED: + await _perform_auto_recharge( + app, rabbit_message, payment_method_db, wallet_auto_recharge + ) return True + + +async def _check_wallet_credits_above_threshold( + threshold_in_credits: NonNegativeDecimal, _credits: Decimal +) -> bool: + return bool(_credits > threshold_in_credits) + + +async def _check_autorecharge_conditions_not_met( + wallet_auto_recharge: GetWalletAutoRecharge | None, +) -> bool: + return ( + wallet_auto_recharge is None + or wallet_auto_recharge.enabled is False + or wallet_auto_recharge.payment_method_id is None + ) + + +async def _exceeds_monthly_limit( + payments_transactions_repo: PaymentsTransactionsRepo, + wallet_id: WalletID, + wallet_auto_recharge: GetWalletAutoRecharge, +): + cumulative_current_month_spending = ( + await payments_transactions_repo.sum_current_month_dollars(wallet_id=wallet_id) + ) + return ( + wallet_auto_recharge.monthly_limit_in_usd is not None + and cumulative_current_month_spending + + wallet_auto_recharge.top_up_amount_in_usd + > wallet_auto_recharge.monthly_limit_in_usd + ) + + +async def _recently_topped_up( + payments_transactions_repo: PaymentsTransactionsRepo, wallet_id: WalletID +): + last_wallet_transaction = ( + await payments_transactions_repo.get_last_payment_transaction_for_wallet( + wallet_id=wallet_id + ) + ) + + current_timestamp = datetime.now(tz=timezone.utc) + current_timestamp_minus_5_minutes = current_timestamp - timedelta(minutes=5) + + return ( + last_wallet_transaction + and last_wallet_transaction.initiated_at > current_timestamp_minus_5_minutes + ) + + +async def _perform_auto_recharge( + app: FastAPI, + rabbit_message: WalletCreditsMessage, + payment_method_db: PaymentsMethodsDB, + wallet_auto_recharge: GetWalletAutoRecharge, +): + rabbitmq_rpc_client = get_rabbitmq_rpc_client(app) + result = await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "get_credit_amount"), + dollar_amount=wallet_auto_recharge.top_up_amount_in_usd, + product_name=rabbit_message.product_name, + ) + credit_result = parse_obj_as(CreditResultGet, result) + + payments_gateway = PaymentsGatewayApi.get_from_app_state(app) + payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + rut_api = ResourceUsageTrackerApi.get_from_app_state(app) + + await pay_with_payment_method( + gateway=payments_gateway, + rut=rut_api, + repo_transactions=payments_transactions_repo, + repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine), + payment_method_id=cast(PaymentMethodID, wallet_auto_recharge.payment_method_id), + amount_dollars=wallet_auto_recharge.top_up_amount_in_usd, + target_credits=credit_result.credit_amount, + product_name=rabbit_message.product_name, + wallet_id=rabbit_message.wallet_id, + wallet_name=f"id={rabbit_message.wallet_id}", + user_id=payment_method_db.user_id, + user_name=f"id={payment_method_db.user_id}", + user_email=EmailStr(f"placeholder_{payment_method_db.user_id}@example.itis"), + comment="Payment generated by auto recharge", + ) diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index 5fa4f374c33..c4ed342afc6 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -182,6 +182,7 @@ async def on_payment_completed( async def pay_with_payment_method( # noqa: PLR0913 gateway: PaymentsGatewayApi, + rut: ResourceUsageTrackerApi, repo_transactions: PaymentsTransactionsRepo, repo_methods: PaymentsMethodsRepo, *, @@ -246,6 +247,8 @@ async def pay_with_payment_method( # noqa: PLR0913 invoice_url=ack.invoice_url, ) + await on_payment_completed(transaction, rut) + return transaction.to_api_model() diff --git a/services/payments/src/simcore_service_payments/services/payments_methods.py b/services/payments/src/simcore_service_payments/services/payments_methods.py index bbfef17dab2..24b9988177b 100644 --- a/services/payments/src/simcore_service_payments/services/payments_methods.py +++ b/services/payments/src/simcore_service_payments/services/payments_methods.py @@ -148,7 +148,7 @@ async def create_payment_method( ) -async def list_payments_methods( +async def list_payment_methods( gateway: PaymentsGatewayApi, repo: PaymentsMethodsRepo, *, diff --git a/services/payments/src/simcore_service_payments/services/rabbitmq.py b/services/payments/src/simcore_service_payments/services/rabbitmq.py index 58facd39868..8938b52cbc2 100644 --- a/services/payments/src/simcore_service_payments/services/rabbitmq.py +++ b/services/payments/src/simcore_service_payments/services/rabbitmq.py @@ -27,6 +27,9 @@ async def _on_startup() -> None: app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( client_name="payments_rpc_server", settings=settings ) + app.state.rabbitmq_rpc_client = await RabbitMQRPCClient.create( + client_name="payments_rpc_client", settings=settings + ) async def _on_shutdown() -> None: if app.state.rabbitmq_client: @@ -35,6 +38,9 @@ async def _on_shutdown() -> None: if app.state.rabbitmq_rpc_server: await app.state.rabbitmq_rpc_server.close() app.state.rabbitmq_rpc_server = None + if app.state.rabbitmq_rpc_client: + await app.state.rabbitmq_rpc_client.close() + app.state.rabbitmq_rpc_client = None app.add_event_handler("startup", _on_startup) app.add_event_handler("shutdown", _on_shutdown) @@ -50,5 +56,10 @@ def get_rabbitmq_rpc_server(app: FastAPI) -> RabbitMQRPCClient: return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_server) +def get_rabbitmq_rpc_client(app: FastAPI) -> RabbitMQRPCClient: + assert app.state.rabbitmq_rpc_client # nosec + return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_client) + + async def post_message(app: FastAPI, message: RabbitMessageBase) -> None: await get_rabbitmq_client(app).publish(message.channel_name, message) diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index 086f6052d98..79bfe37ef7f 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -7,10 +7,11 @@ from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from pathlib import Path -from typing import NamedTuple +from typing import Any, NamedTuple from unittest.mock import Mock import httpx +import jsonref import pytest import respx import sqlalchemy as sa @@ -40,6 +41,7 @@ from simcore_service_payments.models.schemas.acknowledgements import ( AckPaymentWithPaymentMethod, ) +from toolz.dicttoolz import get_in @pytest.fixture(scope="session") @@ -361,3 +363,62 @@ def mock_payments_gateway_service_or_none( mock_payments_routes(mock_payments_gateway_service_api_base) mock_payments_methods_routes(mock_payments_gateway_service_api_base) return mock_payments_gateway_service_api_base + + +# +# mock resource-usage-tracker API +# + + +@pytest.fixture +def rut_service_openapi_specs( + osparc_simcore_services_dir: Path, +) -> dict[str, Any]: + openapi_path = ( + osparc_simcore_services_dir / "resource-usage-tracker" / "openapi.json" + ) + return jsonref.loads(openapi_path.read_text()) + + +@pytest.fixture +def mock_resource_usage_tracker_service_api_base( + app: FastAPI, rut_service_openapi_specs: dict[str, Any] +) -> Iterator[MockRouter]: + settings: ApplicationSettings = app.state.settings + with respx.mock( + base_url=settings.PAYMENTS_RESOURCE_USAGE_TRACKER.base_url, + assert_all_called=False, + assert_all_mocked=True, # IMPORTANT: KEEP always True! + ) as respx_mock: + assert "healthcheck" in get_in( + ["paths", "/", "get", "operationId"], + rut_service_openapi_specs, + no_default=True, + ) # type: ignore + respx_mock.get( + path="/", + name="healthcheck", + ).respond(status.HTTP_200_OK) + + yield respx_mock + + +@pytest.fixture +def mock_resoruce_usage_tracker_service_api( + faker: Faker, + mock_resource_usage_tracker_service_api_base: MockRouter, + rut_service_openapi_specs: dict[str, Any], +) -> MockRouter: + # check it exists + get_in( + ["paths", "/v1/credit-transactions", "post", "operationId"], + rut_service_openapi_specs, + no_default=True, + ) + + # fake successful response + mock_resource_usage_tracker_service_api_base.post( + "/v1/credit-transactions" + ).respond(json={"credit_transaction_id": faker.pyint()}) + + return mock_resource_usage_tracker_service_api_base diff --git a/services/payments/tests/unit/test_rpc_payments_methods.py b/services/payments/tests/unit/test_rpc_payments_methods.py index 7cce4ae6f95..49a3a59a1ef 100644 --- a/services/payments/tests/unit/test_rpc_payments_methods.py +++ b/services/payments/tests/unit/test_rpc_payments_methods.py @@ -201,6 +201,7 @@ async def test_webserver_pay_with_payment_method_workflow( is_pdb_enabled: bool, app: FastAPI, rpc_client: RabbitMQRPCClient, + mock_resoruce_usage_tracker_service_api: None, mock_payments_gateway_service_or_none: MockRouter | None, faker: Faker, product_name: ProductName, diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 9557dba382d..7b3098ddf63 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -5,16 +5,53 @@ # pylint: disable=unused-variable from collections.abc import Callable +from datetime import datetime, timedelta, timezone from decimal import Decimal +from typing import Awaitable, Iterator from unittest import mock import pytest +import sqlalchemy as sa +from faker import Faker from fastapi import FastAPI +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.api_schemas_webserver.wallets import ( + GetWalletAutoRecharge, + PaymentMethodID, +) +from models_library.basic_types import NonNegativeDecimal +from models_library.products import CreditResultGet, ProductName from models_library.rabbitmq_messages import WalletCreditsMessage from pytest_mock.plugin import MockerFixture +from pytest_simcore.helpers.rawdata_fakers import ( + random_payment_autorecharge, + random_payment_method, + random_payment_transaction, +) from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict -from servicelib.rabbitmq import RabbitMQClient +from respx import MockRouter +from servicelib.rabbitmq import RabbitMQClient, RabbitMQRPCClient, RPCRouter +from simcore_postgres_database.models.payments_autorecharge import payments_autorecharge +from simcore_postgres_database.models.payments_methods import payments_methods +from simcore_postgres_database.models.payments_transactions import ( + PaymentTransactionState, + payments_transactions, +) +from simcore_service_payments.core.settings import ApplicationSettings +from simcore_service_payments.db.payments_transactions_repo import ( + PaymentsTransactionsRepo, +) +from simcore_service_payments.models.db import PaymentsTransactionsDB +from simcore_service_payments.models.schemas.acknowledgements import ( + AckPaymentWithPaymentMethod, +) +from simcore_service_payments.services.auto_recharge_process_message import ( + _check_autorecharge_conditions_not_met, + _check_wallet_credits_above_threshold, + _exceeds_monthly_limit, + _recently_topped_up, +) from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay @@ -50,6 +87,7 @@ def app_environment( **postgres_env_vars_dict, **external_environment, "POSTGRES_CLIENT_NAME": "payments-service-pg-client", + "PAYMENTS_AUTORECHARGE_ENABLED": "1", }, ) @@ -61,13 +99,13 @@ async def mocked_message_parser(mocker: MockerFixture) -> mock.AsyncMock: ) -async def test_process_event_functions( +async def test_process_message__called( mocked_message_parser: mock.AsyncMock, app: FastAPI, rabbitmq_client: Callable[[str], RabbitMQClient], ): publisher = rabbitmq_client("publisher") - msg = WalletCreditsMessage(wallet_id=1, credits=Decimal(120.5)) + msg = WalletCreditsMessage(wallet_id=1, credits=Decimal(80.5), product_name="s4l") await publisher.publish(WalletCreditsMessage.get_channel_name(), msg) async for attempt in AsyncRetrying( @@ -78,3 +116,292 @@ async def test_process_event_functions( ): with attempt: mocked_message_parser.assert_called_once() + + +@pytest.fixture() +def populate_test_db( + postgres_db: sa.engine.Engine, faker: Faker, wallet_id: int +) -> Iterator[None]: + with postgres_db.connect() as con: + _primary_payment_method_id = faker.uuid4() + _completed_at = datetime.now(tz=timezone.utc) + timedelta(minutes=1) + + con.execute( + payments_methods.insert().values( + **random_payment_method( + payment_method_id=_primary_payment_method_id, + wallet_id=wallet_id, + state="SUCCESS", + completed_at=_completed_at, + ) + ) + ) + con.execute( + payments_autorecharge.insert().values( + **random_payment_autorecharge( + primary_payment_method_id=_primary_payment_method_id, + wallet_id=wallet_id, + ) + ) + ) + + yield + + con.execute(payments_methods.delete()) + con.execute(payments_autorecharge.delete()) + + +@pytest.fixture() +def wallet_id(faker: Faker): + return faker.pyint() + + +@pytest.fixture +async def mocked_pay_with_payment_method(mocker: MockerFixture) -> mock.AsyncMock: + return mocker.patch( + "simcore_service_payments.services.payments.PaymentsGatewayApi.pay_with_payment_method", + return_value=AckPaymentWithPaymentMethod.construct( + **AckPaymentWithPaymentMethod.Config.schema_extra["example"] + ), + ) + + +@pytest.fixture +async def mock_rpc_client( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], + mocker: MockerFixture, +) -> RabbitMQRPCClient: + rpc_client = await rabbitmq_rpc_client("client") + + # mock returned client + mocker.patch( + "simcore_service_payments.services.auto_recharge_process_message.get_rabbitmq_rpc_client", + return_value=rpc_client, + ) + + return rpc_client + + +@pytest.fixture +async def mock_rpc_server( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], + mocker: MockerFixture, +) -> RabbitMQRPCClient: + rpc_server = await rabbitmq_rpc_client("mock_server") + + router = RPCRouter() + + # mocks the interface defined in the webserver + + @router.expose() + async def get_credit_amount( + dollar_amount: ProductName, product_name: ProductName + ) -> CreditResultGet: + return CreditResultGet.parse_obj( + CreditResultGet.Config.schema_extra["examples"][0] + ) + + await rpc_server.register_router(router, namespace=WEBSERVER_RPC_NAMESPACE) + + return rpc_server + + +async def _assert_payments_transactions_db_row(postgres_db) -> PaymentsTransactionsDB: + async for attempt in AsyncRetrying( + wait=wait_fixed(0.2), + stop=stop_after_delay(10), + retry=retry_if_exception_type(AssertionError), + reraise=True, + ): + with attempt, postgres_db.connect() as con: + result = con.execute(sa.select(payments_transactions)) + row = result.first() + assert row + return PaymentsTransactionsDB.from_orm(row) + + +async def test_process_message__whole_autorecharge_flow_success( + app: FastAPI, + rabbitmq_client: Callable[[str], RabbitMQClient], + wallet_id: int, + populate_test_db: None, + mocked_pay_with_payment_method: mock.AsyncMock, + mock_rpc_server: RabbitMQRPCClient, + mock_rpc_client: RabbitMQRPCClient, + mock_resoruce_usage_tracker_service_api: MockRouter, + postgres_db: sa.engine.Engine, +): + publisher = rabbitmq_client("publisher") + msg = WalletCreditsMessage( + wallet_id=wallet_id, credits=Decimal(80.5), product_name="s4l" + ) + await publisher.publish(WalletCreditsMessage.get_channel_name(), msg) + + row = await _assert_payments_transactions_db_row(postgres_db) + assert row.wallet_id == wallet_id + assert row.state == PaymentTransactionState.SUCCESS + assert row.comment == "Payment generated by auto recharge" + assert len(mock_resoruce_usage_tracker_service_api.calls) == 1 + + +@pytest.mark.parametrize( + "_credits,expected", [(Decimal(10001), True), (Decimal(9999), False)] +) +async def test_check_wallet_credits_above_threshold( + app: FastAPI, _credits: Decimal, expected: bool +): + settings: ApplicationSettings = app.state.settings + assert settings.PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT + + assert expected == await _check_wallet_credits_above_threshold( + settings.PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT, _credits + ) + + +@pytest.mark.parametrize( + "get_wallet_auto_recharge,expected", + [ + ( + GetWalletAutoRecharge( + enabled=True, + payment_method_id=PaymentMethodID("123"), + min_balance_in_credits=NonNegativeDecimal(10), + top_up_amount_in_usd=NonNegativeDecimal(10), + monthly_limit_in_usd=NonNegativeDecimal(10), + ), + False, + ), + ( + GetWalletAutoRecharge( + enabled=False, + payment_method_id=PaymentMethodID("123"), + min_balance_in_credits=NonNegativeDecimal(10), + top_up_amount_in_usd=NonNegativeDecimal(10), + monthly_limit_in_usd=NonNegativeDecimal(10), + ), + True, + ), + ( + GetWalletAutoRecharge( + enabled=True, + payment_method_id=None, + min_balance_in_credits=NonNegativeDecimal(10), + top_up_amount_in_usd=NonNegativeDecimal(10), + monthly_limit_in_usd=NonNegativeDecimal(10), + ), + True, + ), + (None, True), + ], +) +async def test_check_autorecharge_conditions_not_met( + app: FastAPI, get_wallet_auto_recharge: GetWalletAutoRecharge, expected: bool +): + assert expected == await _check_autorecharge_conditions_not_met( + get_wallet_auto_recharge + ) + + +@pytest.fixture() +def populate_payment_transaction_db( + postgres_db: sa.engine.Engine, wallet_id: int +) -> Iterator[None]: + with postgres_db.connect() as con: + con.execute( + payments_transactions.insert().values( + **random_payment_transaction( + price_dollars=Decimal(9500), + wallet_id=wallet_id, + state=PaymentTransactionState.SUCCESS, + completed_at=datetime.now(tz=timezone.utc), + initiated_at=datetime.now(tz=timezone.utc) - timedelta(seconds=10), + ) + ) + ) + + yield + + con.execute(payments_transactions.delete()) + + +@pytest.mark.parametrize( + "get_wallet_auto_recharge,expected", + [ + ( + GetWalletAutoRecharge( + enabled=True, + payment_method_id=PaymentMethodID("123"), + min_balance_in_credits=NonNegativeDecimal(10), + top_up_amount_in_usd=NonNegativeDecimal(300), + monthly_limit_in_usd=NonNegativeDecimal(10000), + ), + False, + ), + ( + GetWalletAutoRecharge( + enabled=False, + payment_method_id=PaymentMethodID("123"), + min_balance_in_credits=NonNegativeDecimal(10), + top_up_amount_in_usd=NonNegativeDecimal(1000), + monthly_limit_in_usd=NonNegativeDecimal(10000), + ), + True, + ), + ], +) +async def test_exceeds_monthly_limit( + app: FastAPI, + wallet_id: int, + populate_payment_transaction_db: None, + get_wallet_auto_recharge: GetWalletAutoRecharge, + expected: bool, +): + _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + + assert expected == await _exceeds_monthly_limit( + _payments_transactions_repo, wallet_id, get_wallet_auto_recharge + ) + + +async def test_recently_topped_up_true( + app: FastAPI, + wallet_id: int, + populate_payment_transaction_db: None, +): + _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + + assert await _recently_topped_up(_payments_transactions_repo, wallet_id) is True + + +@pytest.fixture() +def populate_payment_transaction_db_with_older_trans( + postgres_db: sa.engine.Engine, wallet_id: int +) -> Iterator[None]: + with postgres_db.connect() as con: + current_timestamp = datetime.now(tz=timezone.utc) + current_timestamp_minus_10_minutes = current_timestamp - timedelta(minutes=10) + + con.execute( + payments_transactions.insert().values( + **random_payment_transaction( + price_dollars=Decimal(9500), + wallet_id=wallet_id, + state=PaymentTransactionState.SUCCESS, + initiated_at=current_timestamp_minus_10_minutes, + ) + ) + ) + + yield + + con.execute(payments_transactions.delete()) + + +async def test_recently_topped_up_false( + app: FastAPI, + wallet_id: int, + populate_payment_transaction_db_with_older_trans: None, +): + _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + + assert await _recently_topped_up(_payments_transactions_repo, wallet_id) is False diff --git a/services/payments/tests/unit/test_services_resource_usage_tracker.py b/services/payments/tests/unit/test_services_resource_usage_tracker.py index 26aa9a8a015..868cdf65661 100644 --- a/services/payments/tests/unit/test_services_resource_usage_tracker.py +++ b/services/payments/tests/unit/test_services_resource_usage_tracker.py @@ -5,17 +5,12 @@ # pylint: disable=unused-variable -from collections.abc import Iterator from datetime import datetime, timezone -from pathlib import Path -from typing import Any -import jsonref import pytest -import respx from asgi_lifespan import LifespanManager from faker import Faker -from fastapi import FastAPI, status +from fastapi import FastAPI from pytest_simcore.helpers.utils_envs import EnvVarsDict from respx import MockRouter from simcore_service_payments.core.application import create_app @@ -24,7 +19,6 @@ ResourceUsageTrackerApi, setup_resource_usage_tracker, ) -from toolz.dicttoolz import get_in async def test_setup_rut_api(app_environment: EnvVarsDict): @@ -61,62 +55,8 @@ def app( return create_app() -@pytest.fixture -def rut_service_openapi_specs( - osparc_simcore_services_dir: Path, -) -> dict[str, Any]: - openapi_path = ( - osparc_simcore_services_dir / "resource-usage-tracker" / "openapi.json" - ) - return jsonref.loads(openapi_path.read_text()) - - -@pytest.fixture -def mock_resource_usage_tracker_service_api_base( - app: FastAPI, rut_service_openapi_specs: dict[str, Any] -) -> Iterator[MockRouter]: - settings: ApplicationSettings = app.state.settings - with respx.mock( - base_url=settings.PAYMENTS_RESOURCE_USAGE_TRACKER.base_url, - assert_all_called=False, - assert_all_mocked=True, # IMPORTANT: KEEP always True! - ) as respx_mock: - assert "healthcheck" in get_in( - ["paths", "/", "get", "operationId"], - rut_service_openapi_specs, - no_default=True, - ) # type: ignore - respx_mock.get( - path="/", - name="healthcheck", - ).respond(status.HTTP_200_OK) - - yield respx_mock - - -@pytest.fixture -def mock_rut_service_api( - faker: Faker, - mock_resource_usage_tracker_service_api_base: MockRouter, - rut_service_openapi_specs: dict[str, Any], -): - # check it exists - get_in( - ["paths", "/v1/credit-transactions", "post", "operationId"], - rut_service_openapi_specs, - no_default=True, - ) - - # fake successful response - mock_resource_usage_tracker_service_api_base.post( - "/v1/credit-transactions" - ).respond(json={"credit_transaction_id": faker.pyint()}) - - return mock_resource_usage_tracker_service_api_base - - async def test_add_credits_to_wallet( - app: FastAPI, faker: Faker, mock_rut_service_api: MockRouter + app: FastAPI, faker: Faker, mock_resoruce_usage_tracker_service_api: MockRouter ): # test rut_api = ResourceUsageTrackerApi.get_from_app_state(app) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py index 4c54c8b0628..d2ffc4ad425 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py @@ -45,6 +45,7 @@ async def sum_credit_transactions_and_publish_to_rabbitmq( wallet_id=wallet_id, created_at=datetime.now(tz=timezone.utc), credits=wallet_total_credits.available_osparc_credits, + product_name=product_name, ) await rabbitmq_client.publish(publish_message.channel_name, publish_message) return wallet_total_credits diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 91dfe674a06..9e00e568cd6 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -12,7 +12,7 @@ commit_args = --no-verify [tool:pytest] addopts = --strict-markers asyncio_mode = auto -markers = +markers = slow: marks tests as slow (deselect with '-m "not slow"') acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." testit: "marks test to run during development" diff --git a/services/web/server/src/simcore_service_webserver/products/_api.py b/services/web/server/src/simcore_service_webserver/products/_api.py index e124a37eb8e..a1b1b6b3ebb 100644 --- a/services/web/server/src/simcore_service_webserver/products/_api.py +++ b/services/web/server/src/simcore_service_webserver/products/_api.py @@ -1,15 +1,17 @@ +from decimal import Decimal from pathlib import Path import aiofiles from aiohttp import web from models_library.basic_types import NonNegativeDecimal -from models_library.products import ProductName +from models_library.products import CreditResultGet, ProductName from .._constants import APP_PRODUCTS_KEY, RQ_PRODUCT_KEY from .._resources import webserver_resources from ._db import ProductRepository from ._events import APP_PRODUCTS_TEMPLATES_DIR_KEY from ._model import Product +from .errors import ProductPriceNotDefinedError def get_product_name(request: web.Request) -> str: @@ -49,6 +51,30 @@ async def get_current_product_credit_price( return await repo.get_product_latest_credit_price_or_none(current_product_name) +async def get_credit_amount( + app: web.Application, *, dollar_amount: Decimal, product_name: ProductName +) -> CreditResultGet: + """For provided dollars and product gets credit amount. + + NOTE: Contrary to other product api functions (e.g. get_current_product) this function + gets the latest update from the database. Otherwise, products are loaded + on startup and cached therefore in those cases would require a restart + of the service for the latest changes to take effect. + """ + repo = ProductRepository.create_from_app(app) + usd_per_credit: NonNegativeDecimal | None = ( + await repo.get_product_latest_credit_price_or_none(product_name) + ) + if not usd_per_credit: + # '0 or None' should raise + raise ProductPriceNotDefinedError( + reason=f"Product {product_name} usd_per_credit is either not defined or zero" + ) + + credit_amount = dollar_amount / usd_per_credit + return CreditResultGet(product_name=product_name, credit_amount=credit_amount) + + # # helpers for get_product_template_path # diff --git a/services/web/server/src/simcore_service_webserver/products/_rpc.py b/services/web/server/src/simcore_service_webserver/products/_rpc.py new file mode 100644 index 00000000000..4a4ee46a655 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/products/_rpc.py @@ -0,0 +1,29 @@ +from decimal import Decimal + +from aiohttp import web +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.products import CreditResultGet, ProductName +from servicelib.rabbitmq import RPCRouter + +from ..rabbitmq import get_rabbitmq_rpc_server +from . import _api + +router = RPCRouter() + + +@router.expose() +async def get_credit_amount( + app: web.Application, + *, + dollar_amount: Decimal, + product_name: ProductName, +) -> CreditResultGet: + credit_result_get: CreditResultGet = await _api.get_credit_amount( + app, dollar_amount=dollar_amount, product_name=product_name + ) + return credit_result_get + + +async def register_rpc_routes_on_startup(app: web.Application): + rpc_server = get_rabbitmq_rpc_server(app) + await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/products/api.py b/services/web/server/src/simcore_service_webserver/products/api.py index 58d82f2a872..eb4b2bab505 100644 --- a/services/web/server/src/simcore_service_webserver/products/api.py +++ b/services/web/server/src/simcore_service_webserver/products/api.py @@ -1,8 +1,8 @@ from models_library.products import ProductName from ._api import ( + get_credit_amount, get_current_product, - get_current_product_credit_price, get_product, get_product_name, get_product_template_path, @@ -11,7 +11,7 @@ from ._model import Product __all__: tuple[str, ...] = ( - "get_current_product_credit_price", + "get_credit_amount", "get_current_product", "get_product_name", "get_product_template_path", diff --git a/services/web/server/src/simcore_service_webserver/products/errors.py b/services/web/server/src/simcore_service_webserver/products/errors.py new file mode 100644 index 00000000000..a4696b01dc3 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/products/errors.py @@ -0,0 +1,14 @@ +""" + API plugin errors +""" + + +from pydantic.errors import PydanticErrorMixin + + +class ProductError(PydanticErrorMixin, ValueError): + ... + + +class ProductPriceNotDefinedError(ProductError): + msg_template = "Product price not defined. {reason}" diff --git a/services/web/server/src/simcore_service_webserver/products/plugin.py b/services/web/server/src/simcore_service_webserver/products/plugin.py index 823c8fb2718..70483623419 100644 --- a/services/web/server/src/simcore_service_webserver/products/plugin.py +++ b/services/web/server/src/simcore_service_webserver/products/plugin.py @@ -15,7 +15,8 @@ from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup from .._constants import APP_SETTINGS_KEY -from . import _handlers, _invitations_handlers +from ..rabbitmq import setup_rabbitmq +from . import _handlers, _invitations_handlers, _rpc from ._events import ( auto_create_products_groups, load_products_on_startup, @@ -43,6 +44,11 @@ def setup_products(app: web.Application): app.router.add_routes(_handlers.routes) app.router.add_routes(_invitations_handlers.routes) + # rpc api + setup_rabbitmq(app) + if app[APP_SETTINGS_KEY].WEBSERVER_RABBITMQ: + app.on_startup.append(_rpc.register_rpc_routes_on_startup) + # events app.on_startup.append( # NOTE: must go BEFORE load_products_on_startup diff --git a/services/web/server/src/simcore_service_webserver/wallets/_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_handlers.py index beaba7fc4f7..c1d4d8bf154 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_handlers.py @@ -33,10 +33,12 @@ PaymentNotFoundError, PaymentUniqueViolationError, ) +from ..products.errors import ProductPriceNotDefinedError from ..security.decorators import permission_required from ..users.exceptions import UserDefaultWalletNotFoundError from ..utils_aiohttp import envelope_json_response from . import _api +from ._constants import MSG_PRICE_NOT_DEFINED_ERROR from .errors import WalletAccessForbiddenError, WalletNotFoundError _logger = logging.getLogger(__name__) @@ -68,6 +70,9 @@ async def wrapper(request: web.Request) -> web.StreamResponse: except WalletAccessForbiddenError as exc: raise web.HTTPForbidden(reason=f"{exc}") from exc + except ProductPriceNotDefinedError as exc: + raise web.HTTPConflict(reason=MSG_PRICE_NOT_DEFINED_ERROR) + return wrapper diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index c4cac0680f2..0efde369a95 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -11,10 +11,9 @@ ReplaceWalletAutoRecharge, WalletPaymentInitiated, ) -from models_library.basic_types import AmountDecimal, NonNegativeDecimal +from models_library.products import CreditResultGet from models_library.rest_pagination import Page, PageQueryParameters from models_library.rest_pagination_utils import paginate_data -from pydantic import parse_obj_as from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -41,10 +40,9 @@ pay_with_payment_method, replace_wallet_payment_autorecharge, ) -from ..products.api import get_current_product_credit_price +from ..products.api import get_credit_amount from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from ._constants import MSG_PRICE_NOT_DEFINED_ERROR from ._handlers import ( WalletsPathParams, WalletsRequestContext, @@ -54,18 +52,6 @@ _logger = logging.getLogger(__name__) -async def _eval_total_credits_or_raise( - request: web.Request, amount_dollars: AmountDecimal -) -> NonNegativeDecimal: - # Conversion - usd_per_credit = await get_current_product_credit_price(request) - if not usd_per_credit: - # '0 or None' should raise - raise web.HTTPConflict(reason=MSG_PRICE_NOT_DEFINED_ERROR) - - return parse_obj_as(NonNegativeDecimal, amount_dollars / usd_per_credit) - - routes = web.RouteTableDef() @@ -91,9 +77,10 @@ async def _create_payment(request: web.Request): log_duration=True, extra=get_log_record_extra(user_id=req_ctx.user_id), ): - - osparc_credits = await _eval_total_credits_or_raise( - request, body_params.price_dollars + credit_result: CreditResultGet = await get_credit_amount( + request.app, + dollar_amount=body_params.price_dollars, + product_name=req_ctx.product_name, ) payment: WalletPaymentInitiated = await init_creation_of_wallet_payment( @@ -101,7 +88,7 @@ async def _create_payment(request: web.Request): user_id=req_ctx.user_id, product_name=req_ctx.product_name, wallet_id=wallet_id, - osparc_credits=osparc_credits, + osparc_credits=credit_result.credit_amount, comment=body_params.comment, price_dollars=body_params.price_dollars, ) @@ -334,9 +321,10 @@ async def _pay_with_payment_method(request: web.Request): log_duration=True, extra=get_log_record_extra(user_id=req_ctx.user_id), ): - - osparc_credits = await _eval_total_credits_or_raise( - request, body_params.price_dollars + credit_result: CreditResultGet = await get_credit_amount( + request.app, + dollar_amount=body_params.price_dollars, + product_name=req_ctx.product_name, ) payment: PaymentTransaction = await pay_with_payment_method( @@ -345,7 +333,7 @@ async def _pay_with_payment_method(request: web.Request): product_name=req_ctx.product_name, wallet_id=wallet_id, payment_method_id=path_params.payment_method_id, - osparc_credits=osparc_credits, + osparc_credits=credit_result.credit_amount, comment=body_params.comment, price_dollars=body_params.price_dollars, ) diff --git a/services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py b/services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py new file mode 100644 index 00000000000..c359d07a7e7 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py @@ -0,0 +1,98 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +from collections.abc import Awaitable, Callable +from decimal import Decimal + +import pytest +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.products import CreditResultGet, ProductName +from models_library.rabbitmq_basic_types import RPCMethodName +from pydantic import parse_obj_as +from pytest_mock import MockerFixture +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_login import UserInfoDict +from servicelib.rabbitmq import RabbitMQRPCClient, RPCServerError +from settings_library.rabbit import RabbitSettings +from simcore_postgres_database.models.users import UserRole +from simcore_service_webserver.application_settings import ApplicationSettings + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +@pytest.fixture +def app_environment( + rabbit_service: RabbitSettings, + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, +): + new_envs = setenvs_from_dict( + monkeypatch, + { + **app_environment, + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + }, + ) + + settings = ApplicationSettings.create_from_envs() + assert settings.WEBSERVER_RABBITMQ + + return new_envs + + +@pytest.fixture +def user_role() -> UserRole: + return UserRole.USER + + +@pytest.fixture +async def rpc_client( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], + mocker: MockerFixture, + all_product_prices, +) -> RabbitMQRPCClient: + return await rabbitmq_rpc_client("client") + + +async def test_get_credit_amount( + rpc_client: RabbitMQRPCClient, + osparc_product_name: ProductName, + logged_user: UserInfoDict, +): + result = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "get_credit_amount"), + dollar_amount=Decimal(900), + product_name="s4l", + ) + credit_result = parse_obj_as(CreditResultGet, result) + assert credit_result.credit_amount == 100 + + result = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "get_credit_amount"), + dollar_amount=Decimal(900), + product_name="tis", + ) + credit_result = parse_obj_as(CreditResultGet, result) + assert credit_result.credit_amount == 180 + + with pytest.raises(RPCServerError) as exc_info: + await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "get_credit_amount"), + dollar_amount=Decimal(900), + product_name="osparc", + ) + exc = exc_info.value + assert exc.method_name == "get_credit_amount" + assert exc.msg From 58eb4fdef026a3408ec8aabd8c987e4d055df714 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 17 Nov 2023 17:08:29 +0100 Subject: [PATCH 16/24] =?UTF-8?q?=F0=9F=90=9B=F0=9F=8E=A8=20Fixes=20format?= =?UTF-8?q?ting=20error=20in=20payments=20and=20enhances=20logging=20and?= =?UTF-8?q?=20error=20handling=20(#5037)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/models_library/healthchecks.py | 28 +++++ .../src/servicelib/fastapi/http_client.py | 39 ++++++ .../tests/fastapi/test_http_client.py | 48 +++++++- .../api/rpc/_payments.py | 114 +++++------------- .../api/rpc/_payments_methods.py | 87 ++++++++----- .../services/healthchecks.py | 27 +++++ .../services/payments.py | 10 +- .../services/payments_gateway.py | 58 ++++++--- .../services/postgres.py | 15 +++ .../api/test__one_time_payment_workflows.py | 21 +++- .../unit/test_services_payments_gateway.py | 21 ++++ .../payments/_onetime_api.py | 36 ++++-- .../payments/_rpc.py | 23 ++++ .../with_dbs/03/wallets/payments/conftest.py | 17 +++ 14 files changed, 396 insertions(+), 148 deletions(-) create mode 100644 packages/models-library/src/models_library/healthchecks.py create mode 100644 services/payments/src/simcore_service_payments/services/healthchecks.py diff --git a/packages/models-library/src/models_library/healthchecks.py b/packages/models-library/src/models_library/healthchecks.py new file mode 100644 index 00000000000..6e78b31a5dd --- /dev/null +++ b/packages/models-library/src/models_library/healthchecks.py @@ -0,0 +1,28 @@ +# +# healthcheck models for liveness probes and readiness probes +# +# SEE https://medium.com/polarsquad/how-should-i-answer-a-health-check-aa1fcf6e858e +# SEE https://docs.docker.com/engine/reference/builder/#healthcheck + + +from datetime import timedelta +from typing import TypeAlias + +from pydantic import BaseModel + + +class IsResponsive(BaseModel): + elapsed: timedelta # time elapsed to respond + + def __bool__(self) -> bool: + return True + + +class IsNonResponsive(BaseModel): + reason: str + + def __bool__(self) -> bool: + return False + + +LivenessResult: TypeAlias = IsResponsive | IsNonResponsive diff --git a/packages/service-library/src/servicelib/fastapi/http_client.py b/packages/service-library/src/servicelib/fastapi/http_client.py index 2eb87400eee..781533a1e68 100644 --- a/packages/service-library/src/servicelib/fastapi/http_client.py +++ b/packages/service-library/src/servicelib/fastapi/http_client.py @@ -3,6 +3,7 @@ import httpx from fastapi import FastAPI +from models_library.healthchecks import IsNonResponsive, IsResponsive, LivenessResult _logger = logging.getLogger(__name__) @@ -51,6 +52,13 @@ async def is_healhy(self) -> bool: except httpx.HTTPError: return False + async def check_liveness(self) -> LivenessResult: + try: + response = await self.client.get("/") + return IsResponsive(elapsed=response.elapsed) + except httpx.RequestError as err: + return IsNonResponsive(reason=f"{err}") + class AppStateMixin: """ @@ -77,3 +85,34 @@ def pop_from_app_state(cls, app: FastAPI): old = getattr(app.state, cls.app_state_name, None) delattr(app.state, cls.app_state_name) return old + + +def to_curl_command(request: httpx.Request, *, use_short_options: bool = True) -> str: + """Composes a curl command from a given request + + Can be used to reproduce a request in a separate terminal (e.g. debugging) + """ + # Adapted from https://github.com/marcuxyz/curlify2/blob/master/curlify2/curlify.py + method = request.method + url = request.url + + # https://curl.se/docs/manpage.html#-X + # -X, --request {method} + _x = "-X" if use_short_options else "--request" + request_option = f"{_x} {method}" + + # https://curl.se/docs/manpage.html#-d + # -d, --data HTTP POST data + data_option = "" + if body := request.read().decode(): + _d = "-d" if use_short_options else "--data" + data_option = f"{_d} '{body}'" + + # https://curl.se/docs/manpage.html#-H + # H, --header
Pass custom header(s) to server + headers_option = "" + if headers := [f'"{k}: {v}"' for k, v in request.headers.items()]: + _h = "-H" if use_short_options else "--header" + headers_option = f"{_h} {f' {_h} '.join(headers)}" + + return f"curl {request_option} {headers_option} {data_option} {url}" diff --git a/packages/service-library/tests/fastapi/test_http_client.py b/packages/service-library/tests/fastapi/test_http_client.py index 483051476cf..9815bf7223f 100644 --- a/packages/service-library/tests/fastapi/test_http_client.py +++ b/packages/service-library/tests/fastapi/test_http_client.py @@ -12,7 +12,8 @@ import respx from asgi_lifespan import LifespanManager from fastapi import FastAPI, status -from servicelib.fastapi.http_client import AppStateMixin, BaseHttpApi +from models_library.healthchecks import IsResponsive +from servicelib.fastapi.http_client import AppStateMixin, BaseHttpApi, to_curl_command def test_using_app_state_mixin(): @@ -98,5 +99,50 @@ class MyClientApi(BaseHttpApi, AppStateMixin): assert await api.ping() assert await api.is_healhy() + alive = await api.check_liveness() + assert bool(alive) + assert isinstance(alive, IsResponsive) + assert alive.elapsed.total_seconds() < 1 + # shutdown event assert api.client.is_closed + + +async def test_to_curl_command(mock_server_api: respx.MockRouter, base_url: str): + + mock_server_api.post(path__startswith="/foo").respond(status.HTTP_200_OK) + mock_server_api.get(path__startswith="/foo").respond(status.HTTP_200_OK) + mock_server_api.delete(path__startswith="/foo").respond(status.HTTP_200_OK) + + async with httpx.AsyncClient(base_url=base_url) as client: + response = await client.post("/foo", params={"x": "3"}, json={"y": 12}) + assert response.status_code == 200 + + cmd = to_curl_command(response.request) + + assert ( + cmd + == 'curl -X POST -H "host: test_base_http_api" -H "accept: */*" -H "accept-encoding: gzip, deflate" -H "connection: keep-alive" -H "user-agent: python-httpx/0.25.0" -H "content-length: 9" -H "content-type: application/json" -d \'{"y": 12}\' https://test_base_http_api/foo?x=3' + ) + + cmd = to_curl_command(response.request, use_short_options=False) + assert ( + cmd + == 'curl --request POST --header "host: test_base_http_api" --header "accept: */*" --header "accept-encoding: gzip, deflate" --header "connection: keep-alive" --header "user-agent: python-httpx/0.25.0" --header "content-length: 9" --header "content-type: application/json" --data \'{"y": 12}\' https://test_base_http_api/foo?x=3' + ) + + # with GET + response = await client.get("/foo", params={"x": "3"}) + cmd = to_curl_command(response.request) + + assert ( + cmd + == 'curl -X GET -H "host: test_base_http_api" -H "accept: */*" -H "accept-encoding: gzip, deflate" -H "connection: keep-alive" -H "user-agent: python-httpx/0.25.0" https://test_base_http_api/foo?x=3' + ) + + # with DELETE + response = await client.delete("/foo", params={"x": "3"}) + cmd = to_curl_command(response.request) + + assert "DELETE" in cmd + assert " -d " not in cmd diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments.py b/services/payments/src/simcore_service_payments/api/rpc/_payments.py index 5271bdfe017..550f5fb6977 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments.py @@ -1,10 +1,10 @@ import logging from decimal import Decimal -import arrow from fastapi import FastAPI from models_library.api_schemas_webserver.wallets import ( PaymentID, + PaymentTransaction, WalletPaymentInitiated, ) from models_library.users import UserID @@ -12,14 +12,9 @@ from pydantic import EmailStr from servicelib.logging_utils import get_log_record_extra, log_context from servicelib.rabbitmq import RPCRouter -from simcore_postgres_database.models.payments_transactions import ( - PaymentTransactionState, -) -from ..._constants import PAG, PGDB -from ...core.errors import PaymentAlreadyAckedError, PaymentNotFoundError from ...db.payments_transactions_repo import PaymentsTransactionsRepo -from ...models.payments_gateway import InitPayment, PaymentInitiated +from ...services import payments from ...services.payments_gateway import PaymentsGatewayApi _logger = logging.getLogger(__name__) @@ -42,58 +37,27 @@ async def init_payment( user_email: EmailStr, comment: str | None = None, ) -> WalletPaymentInitiated: - initiated_at = arrow.utcnow().datetime - # Payment-Gateway with log_context( _logger, logging.INFO, - "%s: Init payment %s in payments-gateway", - PAG, + "Init payment to %s", f"{wallet_id=}", extra=get_log_record_extra(user_id=user_id), ): - payments_gateway_api = PaymentsGatewayApi.get_from_app_state(app) - - init = await payments_gateway_api.init_payment( - payment=InitPayment( - amount_dollars=amount_dollars, - credits=target_credits, - user_name=user_name, - user_email=user_email, - wallet_name=wallet_name, - ) - ) - - submission_link = payments_gateway_api.get_form_payment_url(init.payment_id) - - # Database - with log_context( - _logger, - logging.INFO, - "%s: Annotate INIT transaction %s in db", - PGDB, - f"{init.payment_id=}", - extra=get_log_record_extra(user_id=user_id), - ): - repo = PaymentsTransactionsRepo(db_engine=app.state.engine) - payment_id = await repo.insert_init_payment_transaction( - payment_id=init.payment_id, - price_dollars=amount_dollars, - osparc_credits=target_credits, + return await payments.init_one_time_payment( + gateway=PaymentsGatewayApi.get_from_app_state(app), + repo=PaymentsTransactionsRepo(db_engine=app.state.engine), + amount_dollars=amount_dollars, + target_credits=target_credits, product_name=product_name, + wallet_id=wallet_id, + wallet_name=wallet_name, user_id=user_id, + user_name=user_name, user_email=user_email, - wallet_id=wallet_id, comment=comment, - initiated_at=initiated_at, ) - assert payment_id == init.payment_id # nosec - - return WalletPaymentInitiated( - payment_id=f"{payment_id}", - payment_form_url=f"{submission_link}", - ) @router.expose() @@ -104,46 +68,34 @@ async def cancel_payment( user_id: UserID, wallet_id: WalletID, ) -> None: - # validation - repo = PaymentsTransactionsRepo(db_engine=app.state.engine) - payment = await repo.get_payment_transaction( - payment_id=payment_id, user_id=user_id, wallet_id=wallet_id - ) - - if payment is None: - raise PaymentNotFoundError(payment_id=payment_id) - - if payment.state.is_completed(): - if payment.state == PaymentTransactionState.CANCELED: - # Avoids error if multiple cancel calls - return - raise PaymentAlreadyAckedError(payment_id=payment_id) - # execution with log_context( _logger, logging.INFO, - "%s: Cancelling payment %s in payments-gateway", - PAG, + "Cancel payment in %s", f"{wallet_id=}", extra=get_log_record_extra(user_id=user_id), ): - payments_gateway_api = PaymentsGatewayApi.get_from_app_state(app) - - init = PaymentInitiated(payment_id=payment_id) - payment_cancelled = await payments_gateway_api.cancel_payment(init) - - with log_context( - _logger, - logging.INFO, - "%s: Annotate CANCEL transaction %s in db", - PGDB, - f"{payment_id=}", - extra=get_log_record_extra(user_id=user_id), - ): - await repo.update_ack_payment_transaction( + await payments.cancel_one_time_payment( + gateway=PaymentsGatewayApi.get_from_app_state(app), + repo=PaymentsTransactionsRepo(db_engine=app.state.engine), payment_id=payment_id, - completion_state=PaymentTransactionState.CANCELED, - state_message=payment_cancelled.message, - invoice_url=None, + user_id=user_id, + wallet_id=wallet_id, ) + + +@router.expose() +async def get_payments_page( + app: FastAPI, + *, + user_id: UserID, + limit: int | None = None, + offset: int | None = None, +) -> tuple[int, list[PaymentTransaction]]: + return await payments.get_payments_page( + repo=PaymentsTransactionsRepo(db_engine=app.state.engine), + user_id=user_id, + limit=limit, + offset=offset, + ) diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py index 18262bacae1..9d959374291 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py @@ -11,6 +11,7 @@ from models_library.users import UserID from models_library.wallets import WalletID from pydantic import EmailStr +from servicelib.logging_utils import get_log_record_extra, log_context from servicelib.rabbitmq import RPCRouter from ...db.payments_methods_repo import PaymentsMethodsRepo @@ -35,15 +36,22 @@ async def init_creation_of_payment_method( user_name: IDStr, user_email: EmailStr, ) -> PaymentMethodInitiated: - return await payments_methods.init_creation_of_payment_method( - gateway=PaymentsGatewayApi.get_from_app_state(app), - repo=PaymentsMethodsRepo(db_engine=app.state.engine), - wallet_id=wallet_id, - wallet_name=wallet_name, - user_id=user_id, - user_name=user_name, - user_email=user_email, - ) + with log_context( + _logger, + logging.INFO, + "Init creation of payment-method to %s", + f"{wallet_id=}", + extra=get_log_record_extra(user_id=user_id), + ): + return await payments_methods.init_creation_of_payment_method( + gateway=PaymentsGatewayApi.get_from_app_state(app), + repo=PaymentsMethodsRepo(db_engine=app.state.engine), + wallet_id=wallet_id, + wallet_name=wallet_name, + user_id=user_id, + user_name=user_name, + user_email=user_email, + ) @router.expose() @@ -54,13 +62,20 @@ async def cancel_creation_of_payment_method( user_id: UserID, wallet_id: WalletID, ) -> None: - await payments_methods.cancel_creation_of_payment_method( - gateway=PaymentsGatewayApi.get_from_app_state(app), - repo=PaymentsMethodsRepo(db_engine=app.state.engine), - payment_method_id=payment_method_id, - user_id=user_id, - wallet_id=wallet_id, - ) + with log_context( + _logger, + logging.INFO, + "Cancel creation of payment-method in %s", + f"{wallet_id=}", + extra=get_log_record_extra(user_id=user_id), + ): + await payments_methods.cancel_creation_of_payment_method( + gateway=PaymentsGatewayApi.get_from_app_state(app), + repo=PaymentsMethodsRepo(db_engine=app.state.engine), + payment_method_id=payment_method_id, + user_id=user_id, + wallet_id=wallet_id, + ) @router.expose() @@ -127,19 +142,27 @@ async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-a user_email: EmailStr, comment: str | None = None, ): - return await payments.pay_with_payment_method( - gateway=PaymentsGatewayApi.get_from_app_state(app), - rut=ResourceUsageTrackerApi.get_from_app_state(app), - repo_transactions=PaymentsTransactionsRepo(db_engine=app.state.engine), - repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine), - payment_method_id=payment_method_id, - amount_dollars=amount_dollars, - target_credits=target_credits, - product_name=product_name, - wallet_id=wallet_id, - wallet_name=wallet_name, - user_id=user_id, - user_name=user_name, - user_email=user_email, - comment=comment, - ) + with log_context( + _logger, + logging.INFO, + "Pay w/ %s to %s", + f"{payment_method_id=}", + f"{wallet_id=}", + extra=get_log_record_extra(user_id=user_id), + ): + return await payments.pay_with_payment_method( + gateway=PaymentsGatewayApi.get_from_app_state(app), + rut=ResourceUsageTrackerApi.get_from_app_state(app), + repo_transactions=PaymentsTransactionsRepo(db_engine=app.state.engine), + repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine), + payment_method_id=payment_method_id, + amount_dollars=amount_dollars, + target_credits=target_credits, + product_name=product_name, + wallet_id=wallet_id, + wallet_name=wallet_name, + user_id=user_id, + user_name=user_name, + user_email=user_email, + comment=comment, + ) diff --git a/services/payments/src/simcore_service_payments/services/healthchecks.py b/services/payments/src/simcore_service_payments/services/healthchecks.py new file mode 100644 index 00000000000..98774700f44 --- /dev/null +++ b/services/payments/src/simcore_service_payments/services/healthchecks.py @@ -0,0 +1,27 @@ +import asyncio +import logging + +from models_library.healthchecks import LivenessResult +from sqlalchemy.ext.asyncio import AsyncEngine + +from .payments_gateway import PaymentsGatewayApi +from .postgres import check_postgres_liveness +from .resource_usage_tracker import ResourceUsageTrackerApi + +_logger = logging.getLogger(__name__) + + +async def create_health_report( + gateway: PaymentsGatewayApi, + rut: ResourceUsageTrackerApi, + engine: AsyncEngine, +) -> dict[str, LivenessResult]: + gateway_liveness, rut_liveness, db_liveness = await asyncio.gather( + gateway.check_liveness(), rut.check_liveness(), check_postgres_liveness(engine) + ) + + return { + "payments_gateway": gateway_liveness, + "resource_usage_tracker": rut_liveness, + "postgres": db_liveness, + } diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index c4ed342afc6..58a0b592d53 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -18,7 +18,7 @@ ) from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import EmailStr +from pydantic import EmailStr, PositiveInt from servicelib.logging_utils import log_context from simcore_postgres_database.models.payments_transactions import ( PaymentTransactionState, @@ -256,13 +256,13 @@ async def get_payments_page( repo: PaymentsTransactionsRepo, *, user_id: UserID, - limit: int, - offset: int, -) -> tuple[int, list[PaymentsTransactionsDB]]: + limit: PositiveInt | None = None, + offset: PositiveInt | None = None, +) -> tuple[int, list[PaymentTransaction]]: """All payments associated to a user (i.e. including all the owned wallets)""" total_number_of_items, page = await repo.list_user_payment_transactions( user_id=user_id, offset=offset, limit=limit ) - return total_number_of_items, page + return total_number_of_items, [t.to_api_model() for t in page] diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index c9d4174e054..7dd2b47de76 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -16,9 +16,9 @@ from fastapi.encoders import jsonable_encoder from httpx import URL, HTTPStatusError from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID -from pydantic import ValidationError +from pydantic import ValidationError, parse_raw_as from pydantic.errors import PydanticErrorMixin -from servicelib.fastapi.http_client import AppStateMixin, BaseHttpApi +from servicelib.fastapi.http_client import AppStateMixin, BaseHttpApi, to_curl_command from simcore_service_payments.models.schemas.acknowledgements import ( AckPaymentWithPaymentMethod, ) @@ -39,38 +39,58 @@ _logger = logging.getLogger(__name__) -class PaymentsGatewayError(PydanticErrorMixin, ValueError): - msg_template = "Payment-gateway got {status_code} for {operation_id}: {reason}" - - -def _parse_raw_or_none(text: str | None): +def _parse_raw_as_or_none(cls: type, text: str | None): if text: with suppress(ValidationError): - return ErrorModel.parse_raw(text) + return parse_raw_as(cls, text) return None +class PaymentsGatewayError(PydanticErrorMixin, ValueError): + msg_template = "{operation_id} error {status_code}: {reason}" + + @classmethod + def from_http_status_error( + cls, err: HTTPStatusError, operation_id: str + ) -> "PaymentsGatewayError": + return cls( + operation_id=f"PaymentsGatewayApi.{operation_id}", + reason=f"{err}", + status_code=err.response.status_code, + # extra context for details + http_status_error=err, + model=_parse_raw_as_or_none(ErrorModel, err.response.text), + ) + + def get_detailed_message(self) -> str: + err_json = "null" + if model := getattr(self, "model", None): + err_json = model.json(indent=1) + + curl_cmd = "null" + if http_status_error := getattr(self, "http_status_error", None): + curl_cmd = to_curl_command(http_status_error.request) + + return f"{self}\nREQ: '{curl_cmd}'\nRESP: {err_json}" + + @contextlib.contextmanager -def _raise_if_error(operation_id: str): +def _raise_as_payments_gateway_error(operation_id: str): try: - yield except HTTPStatusError as err: - model = _parse_raw_or_none(err.response.text) - - raise PaymentsGatewayError( - operation_id=f"PaymentsGatewayApi.{operation_id}", - reason=model.message if model else f"{err}", - http_status_error=err, - model=model, - ) from err + error = PaymentsGatewayError.from_http_status_error( + err, operation_id=operation_id + ) + _logger.warning(error.get_detailed_message()) + raise error from err def _handle_status_errors(coro: Callable): @functools.wraps(coro) async def _wrapper(self, *args, **kwargs): - with _raise_if_error(operation_id=coro.__name__): + with _raise_as_payments_gateway_error(operation_id=coro.__name__): return await coro(self, *args, **kwargs) return _wrapper diff --git a/services/payments/src/simcore_service_payments/services/postgres.py b/services/payments/src/simcore_service_payments/services/postgres.py index 59d1c6f18c4..5a731551611 100644 --- a/services/payments/src/simcore_service_payments/services/postgres.py +++ b/services/payments/src/simcore_service_payments/services/postgres.py @@ -1,5 +1,9 @@ +import time + from fastapi import FastAPI +from models_library.healthchecks import IsNonResponsive, IsResponsive, LivenessResult from servicelib.db_async_engine import close_db_connection, connect_to_db +from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncEngine from ..core.settings import ApplicationSettings @@ -11,6 +15,17 @@ def get_engine(app: FastAPI) -> AsyncEngine: return engine +async def check_postgres_liveness(engine: AsyncEngine) -> LivenessResult: + try: + tic = time.time() + # test + async with engine.connect(): + ... + return IsResponsive(elapsed=time.time() - tic) + except SQLAlchemyError as err: + return IsNonResponsive(reason=f"{err}") + + def setup_postgres(app: FastAPI): app.state.engine = None diff --git a/services/payments/tests/unit/api/test__one_time_payment_workflows.py b/services/payments/tests/unit/api/test__one_time_payment_workflows.py index f21d94acbcc..f4697b7ab3d 100644 --- a/services/payments/tests/unit/api/test__one_time_payment_workflows.py +++ b/services/payments/tests/unit/api/test__one_time_payment_workflows.py @@ -82,7 +82,7 @@ async def test_successful_one_time_payment_workflow( ) # ACK via api/rest - result = await rpc_client.request( + inited = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, parse_obj_as(RPCMethodName, "init_payment"), amount_dollars=1000, @@ -96,15 +96,30 @@ async def test_successful_one_time_payment_workflow( timeout_s=None if is_pdb_enabled else 5, ) - assert isinstance(result, WalletPaymentInitiated) + assert isinstance(inited, WalletPaymentInitiated) assert mock_payments_gateway_service_or_none.routes["init_payment"].called # ACK response = await client.post( - f"/v1/payments/{result.payment_id}:ack", + f"/v1/payments/{inited.payment_id}:ack", json=AckPayment(success=True, invoice_url=faker.url()).dict(), headers=auth_headers, ) assert response.status_code == status.HTTP_200_OK assert mock_on_payment_completed.called + + # LIST payments via api/rest + got = await rpc_client.request( + PAYMENTS_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "get_payments_page"), + user_id=user_id, + timeout_s=None if is_pdb_enabled else 5, + ) + + total_number_of_items, transactions = got + assert total_number_of_items == 1 + assert len(transactions) == 1 + + assert transactions[0].state == "SUCCESS" + assert transactions[0].payment_id == inited.payment_id diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index 8a0b34f9350..ce2a858cc52 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -4,6 +4,7 @@ # pylint: disable=too-many-arguments +import httpx import pytest from faker import Faker from fastapi import FastAPI, status @@ -17,6 +18,7 @@ from simcore_service_payments.services.payments_gateway import ( PaymentsGatewayApi, PaymentsGatewayError, + _raise_as_payments_gateway_error, setup_payments_gateway, ) @@ -188,6 +190,7 @@ async def test_payment_methods_workflow( with pytest.raises(PaymentsGatewayError) as err_info: await payments_gateway_api.get_payment_method(payment_method_id) + assert str(err_info.value) assert err_info.value.operation_id == "PaymentsGatewayApi.get_payment_method" http_status_error = err_info.value.http_status_error @@ -198,3 +201,21 @@ async def test_payment_methods_workflow( for route in mock_payments_gateway_service_or_none.routes: if route.name and "payment_method" in route.name: assert route.called + + +async def test_payments_gateway_error_exception(): + async def _go(): + with _raise_as_payments_gateway_error(operation_id="foo"): + async with httpx.AsyncClient( + app=FastAPI(), + base_url="http://payments.testserver.io", + ) as client: + response = await client.post("/foo", params={"x": "3"}, json={"y": 12}) + response.raise_for_status() + + with pytest.raises(PaymentsGatewayError) as err_info: + await _go() + err = err_info.value + assert isinstance(err, PaymentsGatewayError) + + assert "curl -X POST" in err.get_detailed_message() diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index 00ce812f7fa..208aeebd413 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -366,6 +366,23 @@ async def pay_with_payment_method( ) +async def _fake_get_payments_page( + app: web.Application, + user_id: UserID, + limit: int, + offset: int, +): + + ( + total_number_of_items, + transactions, + ) = await _onetime_db.list_user_payment_transactions( + app, user_id=user_id, offset=offset, limit=limit + ) + + return total_number_of_items, [_to_api_model(t) for t in transactions] + + async def list_user_payments_page( app: web.Application, product_name: str, @@ -378,11 +395,16 @@ async def list_user_payments_page( assert offset >= 0 # nosec assert product_name # nosec - ( - total_number_of_items, - transactions, - ) = await _onetime_db.list_user_payment_transactions( - app, user_id=user_id, offset=offset, limit=limit - ) + settings: PaymentsSettings = get_plugin_settings(app) + if settings.PAYMENTS_FAKE_COMPLETION: + total_number_of_items, payments = await _fake_get_payments_page( + app, user_id=user_id, offset=offset, limit=limit + ) + + else: + assert not settings.PAYMENTS_FAKE_COMPLETION # nosec + total_number_of_items, payments = await _rpc.get_payments_page( + app, user_id=user_id, offset=offset, limit=limit + ) - return [_to_api_model(t) for t in transactions], total_number_of_items + return payments, total_number_of_items diff --git a/services/web/server/src/simcore_service_webserver/payments/_rpc.py b/services/web/server/src/simcore_service_webserver/payments/_rpc.py index e5c0d2680a3..05e5d70e259 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_rpc.py +++ b/services/web/server/src/simcore_service_webserver/payments/_rpc.py @@ -106,6 +106,29 @@ async def cancel_payment( ) +@log_decorator(_logger, level=logging.DEBUG) +async def get_payments_page( + app: web.Application, + *, + user_id: UserID, + limit: int | None, + offset: int | None, +) -> tuple[int, list[PaymentTransaction]]: + rpc_client = app[_APP_PAYMENTS_RPC_CLIENT_KEY] + + result: tuple[int, list[PaymentTransaction]] = await rpc_client.request( + PAYMENTS_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "get_payments_page"), + user_id=user_id, + limit=limit, + offset=offset, + ) + assert ( # nosec + parse_obj_as(tuple[int, list[PaymentTransaction]], result) is not None + ) + return result + + @log_decorator(_logger, level=logging.DEBUG) async def init_creation_of_payment_method( app: web.Application, diff --git a/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py b/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py index 51df97d8567..0963daa3f27 100644 --- a/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py @@ -41,6 +41,7 @@ ) from simcore_service_webserver.payments._onetime_api import ( _fake_cancel_payment, + _fake_get_payments_page, _fake_init_payment, _fake_pay_with_payment_method, ) @@ -131,6 +132,17 @@ async def _cancel( ): await _fake_cancel_payment(app, payment_id) + async def _get_page( + app: web.Application, + *, + user_id: UserID, + limit: int | None, + offset: int | None, + ): + assert limit is not None + assert offset is not None + return await _fake_get_payments_page(app, user_id, limit, offset) + # payment-methods ---- async def _init_pm( app: web.Application, @@ -236,6 +248,11 @@ async def _pay( autospec=True, side_effect=_cancel, ), + "get_payments_page": mocker.patch( + "simcore_service_webserver.payments._onetime_api._rpc.get_payments_page", + autospec=True, + side_effect=_get_page, + ), "init_creation_of_payment_method": mocker.patch( "simcore_service_webserver.payments._methods_api._rpc.init_creation_of_payment_method", autospec=True, From 33380f12b132ccb17777b91a164cee496ae91661 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard <126242332+bisgaard-itis@users.noreply.github.com> Date: Fri, 17 Nov 2023 23:03:37 +0100 Subject: [PATCH 17/24] =?UTF-8?q?=E2=9C=A8=20Introduce=20fake=20API=20serv?= =?UTF-8?q?er=20entrypoint=20for=20logstreaming=20(#5047)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/routes/solvers_jobs_getters.py | 49 +++++++++++++- .../models/schemas/jobs.py | 10 +++ .../test_api_routers_solvers_jobs_logs.py | 65 +++++++++++++++++++ 3 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index fc7b64030d2..b4a86a9a5bf 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -8,19 +8,26 @@ from fastapi import APIRouter, Depends, status from fastapi.exceptions import HTTPException -from fastapi.responses import RedirectResponse +from fastapi.responses import RedirectResponse, StreamingResponse from fastapi_pagination.api import create_page from models_library.api_schemas_webserver.projects import ProjectGet from models_library.api_schemas_webserver.resource_usage import PricingUnitGet from models_library.api_schemas_webserver.wallets import WalletGetWithAvailableCredits -from models_library.projects_nodes_io import BaseFileLink +from models_library.projects_nodes_io import BaseFileLink, NodeID, NodeIDStr from pydantic.types import PositiveInt from servicelib.logging_utils import log_context from ...models.basic_types import VersionStr from ...models.pagination import Page, PaginationParams from ...models.schemas.files import File -from ...models.schemas.jobs import ArgumentTypes, Job, JobID, JobMetadata, JobOutputs +from ...models.schemas.jobs import ( + ArgumentTypes, + Job, + JobID, + JobLog, + JobMetadata, + JobOutputs, +) from ...models.schemas.solvers import SolverKeyId from ...services.catalog import CatalogApi from ...services.director_v2 import DirectorV2Api, DownloadLink, NodeName @@ -354,3 +361,39 @@ async def get_job_pricing_unit( return await webserver_api.get_project_node_pricing_unit( project_id=job_id, node_id=node_id ) + + +@router.get( + "/{solver_key:path}/releases/{version}/jobs/{job_id:uuid}/logstream", + response_class=StreamingResponse, + include_in_schema=API_SERVER_DEV_FEATURES_ENABLED, +) +async def get_log_stream( + solver_key: SolverKeyId, + version: VersionStr, + job_id: JobID, + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], +): + async def _fake_generator(node_id: NodeIDStr): + for ii in range(100): + job_log: JobLog = JobLog( + job_id=job_id, + node_id=NodeID(node_id), + log_level=logging.DEBUG, + messages=[f"Hi Manuel. Gruss from the API-server {ii/100}"], + ) + yield job_log.json() + "\n" + + job_name = _compose_job_resource_name(solver_key, version, job_id) + + with log_context(_logger, logging.DEBUG, f"Stream logs for {job_name=}"): + project: ProjectGet = await webserver_api.get_project(project_id=job_id) + _raise_if_job_not_associated_with_solver(solver_key, version, project) + + node_ids = list(project.workbench.keys()) + assert len(node_ids) == 1 # nosec + + return StreamingResponse( + _fake_generator(node_id=node_ids[0]), + media_type="application/x-ndjson", + ) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py index ff8c5993251..5bb46b1da4e 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py @@ -3,6 +3,8 @@ from typing import Any, ClassVar, TypeAlias from uuid import UUID, uuid4 +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID from models_library.projects_state import RunningState from pydantic import ( BaseModel, @@ -18,6 +20,7 @@ parse_obj_as, validator, ) +from servicelib.logging_utils import LogLevelInt, LogMessageStr from starlette.datastructures import Headers from ...models.schemas.files import File @@ -296,3 +299,10 @@ def create_from_headers(cls, headers: Headers) -> "JobPricingSpecification | Non return parse_obj_as(JobPricingSpecification, headers) except ValidationError: return None + + +class JobLog(BaseModel): + job_id: ProjectID + node_id: NodeID | None + log_level: LogLevelInt + messages: list[LogMessageStr] diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py new file mode 100644 index 00000000000..e78a39d32e0 --- /dev/null +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py @@ -0,0 +1,65 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +import logging +from typing import Iterable + +import httpx +import pytest +from faker import Faker +from fastapi import FastAPI +from models_library.api_schemas_webserver.projects import ProjectGet +from pytest_mock import MockFixture +from pytest_simcore.simcore_webserver_projects_rest_api import GET_PROJECT +from simcore_service_api_server.models.schemas.jobs import JobID, JobLog + +_logger = logging.getLogger(__name__) + + +@pytest.fixture +def fake_project_for_streaming( + mocker: MockFixture, faker: Faker +) -> Iterable[ProjectGet]: + + assert isinstance(response_body := GET_PROJECT.response_body, dict) + assert (data := response_body.get("data")) is not None + fake_project = ProjectGet.parse_obj(data) + fake_project.workbench = {faker.uuid4(): faker.uuid4()} + mocker.patch( + "simcore_service_api_server.api.dependencies.webserver.AuthSession.get_project", + return_value=fake_project, + ) + + mocker.patch( + "simcore_service_api_server.api.routes.solvers_jobs_getters._raise_if_job_not_associated_with_solver" + ) + yield fake_project + + +async def test_log_streaming( + app: FastAPI, + auth: httpx.BasicAuth, + client: httpx.AsyncClient, + solver_key: str, + solver_version: str, + fake_project_for_streaming: ProjectGet, +): + + job_id: JobID = fake_project_for_streaming.uuid + + ii: int = 0 + async with client.stream( + "GET", + f"/v0/solvers/{solver_key}/releases/{solver_version}/jobs/{job_id}/logstream", + auth=auth, + ) as response: + response.raise_for_status() + async for line in response.aiter_lines(): + job_log = JobLog.parse_raw(line) + _logger.debug(job_log.json()) + ii += 1 + assert ii > 0 From 63414c26d06656148c4faf7cc77f2ff161fae637 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Mon, 20 Nov 2023 10:37:36 +0100 Subject: [PATCH 18/24] =?UTF-8?q?=E2=9C=A8Computational=20backend:=20Trace?= =?UTF-8?q?=20parent=20node=20(#5042)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../utils_projects_nodes.py | 24 ++++++-- .../tests/test_utils_projects_nodes.py | 59 ++++++++++++++++++- .../api/routes/computations.py | 48 +++++++++++++-- .../models/comp_runs.py | 8 +++ .../modules/comp_scheduler/base_scheduler.py | 12 ++-- .../modules/comp_scheduler/dask_scheduler.py | 14 ++--- .../modules/db/repositories/comp_runs.py | 1 + .../modules/db/repositories/projects.py | 7 +++ .../db/repositories/projects_metadata.py | 23 ++++++++ .../simcore_service_director_v2/utils/dask.py | 2 +- .../utils/rabbitmq.py | 2 + .../test_modules_db_repositories_projects.py | 24 ++++++-- 12 files changed, 195 insertions(+), 29 deletions(-) create mode 100644 services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_metadata.py diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index b1b9a6022f0..56ed1b2dce3 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -108,11 +108,10 @@ async def add( raise ProjectNodesProjectNotFound(msg) from exc except UniqueViolation as exc: # this happens if the node already exists on creation - raise ProjectNodesDuplicateNode( - f"Project node already exists: {exc}" - ) from exc + msg = f"Project node already exists: {exc}" + raise ProjectNodesDuplicateNode(msg) from exc - async def list(self, connection: SAConnection) -> list[ProjectNode]: + async def list(self, connection: SAConnection) -> list[ProjectNode]: # noqa: A003 """list the nodes in the current project NOTE: Do not use this in an asyncio.gather call as this will fail! @@ -261,3 +260,20 @@ async def connect_pricing_unit_to_project_node( }, ) await connection.execute(on_update_stmt) + + @staticmethod + async def get_project_id_from_node_id( + connection: SAConnection, *, node_id: uuid.UUID + ) -> uuid.UUID: + """ + Raises: + ProjectNodesNodeNotFound: + """ + get_stmt = sqlalchemy.select(projects_nodes.c.project_uuid).where( + projects_nodes.c.node_id == f"{node_id}" + ) + project_id = await connection.scalar(get_stmt) + if project_id is None: + msg = f"No project found containing {node_id=}" + raise ProjectNodesNodeNotFound(msg) + return uuid.UUID(project_id) diff --git a/packages/postgres-database/tests/test_utils_projects_nodes.py b/packages/postgres-database/tests/test_utils_projects_nodes.py index 8452ed51ec6..85a14630d69 100644 --- a/packages/postgres-database/tests/test_utils_projects_nodes.py +++ b/packages/postgres-database/tests/test_utils_projects_nodes.py @@ -5,8 +5,9 @@ import asyncio import random import uuid +from collections.abc import Awaitable, Callable from random import randint -from typing import Any, Awaitable, Callable +from typing import Any import pytest import sqlalchemy @@ -152,7 +153,9 @@ async def test_list_project_nodes( created_nodes = await projects_nodes_repo.add( connection, - nodes=[create_fake_projects_node() for _ in range(randint(3, 12))], + nodes=[ + create_fake_projects_node() for _ in range(randint(3, 12)) # noqa: S311 + ], ) nodes = await projects_nodes_repo.list(connection) @@ -336,7 +339,7 @@ async def _workflow() -> None: assert list_nodes assert len(list_nodes) == NUM_NODES await projects_nodes_repo.delete( - connection, node_id=random.choice(list_nodes).node_id + connection, node_id=random.choice(list_nodes).node_id # noqa: S311 ) list_nodes = await projects_nodes_repo.list(connection) assert list_nodes @@ -344,3 +347,53 @@ async def _workflow() -> None: await _delete_project(connection, project_uuid=project.uuid) await asyncio.gather(*(_workflow() for _ in range(num_concurrent_workflows))) + + +async def test_get_project_id_from_node_id( + pg_engine: Engine, + connection: SAConnection, + projects_nodes_repo: ProjectNodesRepo, + registered_user: RowProxy, + create_fake_project: Callable[..., Awaitable[RowProxy]], + create_fake_projects_node: Callable[..., ProjectNodeCreate], +): + NUM_NODES = 11 + + async def _workflow() -> dict[uuid.UUID, list[uuid.UUID]]: + async with pg_engine.acquire() as connection: + project = await create_fake_project(connection, registered_user) + projects_nodes_repo = ProjectNodesRepo(project_uuid=project.uuid) + + list_of_nodes = await projects_nodes_repo.add( + connection, + nodes=[create_fake_projects_node() for _ in range(NUM_NODES)], + ) + + return {uuid.UUID(project["uuid"]): [node.node_id for node in list_of_nodes]} + + # create some projects + list_of_project_id_node_ids_map = await asyncio.gather( + *(_workflow() for _ in range(10)) + ) + + for project_id_to_node_ids_map in list_of_project_id_node_ids_map: + project_id = next(iter(project_id_to_node_ids_map)) + random_node_id = random.choice( + project_id_to_node_ids_map[project_id] + ) # noqa: S311 + received_project_id = await ProjectNodesRepo.get_project_id_from_node_id( + connection, node_id=random_node_id + ) + assert received_project_id == next(iter(project_id_to_node_ids_map)) + + +async def test_get_project_id_from_node_id_raises_for_invalid_node_id( + pg_engine: Engine, + connection: SAConnection, + projects_nodes_repo: ProjectNodesRepo, + faker: Faker, +): + with pytest.raises(ProjectNodesNodeNotFound): + await ProjectNodesRepo.get_project_id_from_node_id( + connection, node_id=faker.uuid4(cast_to=None) + ) diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py index a39c31e6596..29652d02488 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py @@ -29,7 +29,7 @@ ) from models_library.clusters import DEFAULT_CLUSTER_ID from models_library.projects import ProjectAtDB, ProjectID -from models_library.projects_nodes_io import NodeID +from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.services import ServiceKeyVersion from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder @@ -55,7 +55,7 @@ SchedulerError, ) from ...models.comp_pipelines import CompPipelineAtDB -from ...models.comp_runs import CompRunsAtDB, RunMetadataDict +from ...models.comp_runs import CompRunsAtDB, ProjectMetadataDict, RunMetadataDict from ...models.comp_tasks import CompTaskAtDB from ...modules.catalog import CatalogClient from ...modules.comp_scheduler.base_scheduler import BaseCompScheduler @@ -64,6 +64,7 @@ from ...modules.db.repositories.comp_runs import CompRunsRepository from ...modules.db.repositories.comp_tasks import CompTasksRepository from ...modules.db.repositories.projects import ProjectsRepository +from ...modules.db.repositories.projects_metadata import ProjectsMetadataRepository from ...modules.db.repositories.users import UsersRepository from ...modules.director_v0 import DirectorV0Client from ...modules.resource_usage_tracker_client import ResourceUsageTrackerClient @@ -147,14 +148,44 @@ async def _check_pipeline_startable( ) from exc +async def _get_project_metadata( + project_repo: ProjectsRepository, + projects_metadata_repo: ProjectsMetadataRepository, + computation: ComputationCreate, +) -> ProjectMetadataDict: + current_project_metadata = await projects_metadata_repo.get_metadata( + computation.project_id + ) + + if not current_project_metadata: + return {} + if "node_id" not in current_project_metadata: + return {} + + parent_node_id = NodeID(current_project_metadata["node_id"]) + parent_node_idstr = NodeIDStr(f"{parent_node_id}") + parent_project_id = await project_repo.get_project_id_from_node(parent_node_id) + parent_project = await project_repo.get_project(parent_project_id) + assert parent_node_idstr in parent_project.workbench + + return ProjectMetadataDict( + parent_node_id=parent_node_id, + parent_node_name=parent_project.workbench[parent_node_idstr].label, + parent_project_id=parent_project_id, + parent_project_name=parent_project.name, + ) + + async def _try_start_pipeline( *, + project_repo: ProjectsRepository, computation: ComputationCreate, complete_dag: nx.DiGraph, minimal_dag: nx.DiGraph, scheduler: BaseCompScheduler, project: ProjectAtDB, users_repo: UsersRepository, + projects_metadata_repo: ProjectsMetadataRepository, ) -> None: if not minimal_dag.nodes(): # 2 options here: either we have cycles in the graph or it's really done @@ -191,7 +222,11 @@ async def _try_start_pipeline( user_email=await users_repo.get_user_email(computation.user_id), wallet_id=wallet_id, wallet_name=wallet_name, - ), + project_metadata=await _get_project_metadata( + project_repo, projects_metadata_repo, computation + ), + ) + or {}, use_on_demand_clusters=computation.use_on_demand_clusters, ) @@ -222,10 +257,13 @@ async def create_computation( # noqa: PLR0913 clusters_repo: Annotated[ ClustersRepository, Depends(get_repository(ClustersRepository)) ], + users_repo: Annotated[UsersRepository, Depends(get_repository(UsersRepository))], + projects_metadata_repo: Annotated[ + ProjectsMetadataRepository, Depends(get_repository(ProjectsMetadataRepository)) + ], director_client: Annotated[DirectorV0Client, Depends(get_director_v0_client)], scheduler: Annotated[BaseCompScheduler, Depends(get_scheduler)], catalog_client: Annotated[CatalogClient, Depends(get_catalog_client)], - users_repo: Annotated[UsersRepository, Depends(get_repository(UsersRepository))], rut_client: Annotated[ResourceUsageTrackerClient, Depends(get_rut_client)], rpc_client: Annotated[RabbitMQRPCClient, Depends(rabbitmq_rpc_client)], ) -> ComputationGet: @@ -281,12 +319,14 @@ async def create_computation( # noqa: PLR0913 if computation.start_pipeline: await _try_start_pipeline( + project_repo=project_repo, computation=computation, complete_dag=complete_dag, minimal_dag=minimal_computational_dag, scheduler=scheduler, project=project, users_repo=users_repo, + projects_metadata_repo=projects_metadata_repo, ) # filter the tasks by the effective pipeline diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py index 03265d56744..b8177bdb2a2 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_runs.py @@ -13,6 +13,13 @@ from ..utils.db import DB_TO_RUNNING_STATE +class ProjectMetadataDict(TypedDict, total=False): + parent_node_id: NodeID + parent_node_name: str + parent_project_id: ProjectID + parent_project_name: str + + class RunMetadataDict(TypedDict, total=False): node_id_names_map: dict[NodeID, str] project_name: str @@ -21,6 +28,7 @@ class RunMetadataDict(TypedDict, total=False): user_email: str wallet_id: int | None wallet_name: str | None + project_metadata: ProjectMetadataDict class CompRunsAtDB(BaseModel): diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/base_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/base_scheduler.py index 6322ba7f1b2..5b6346d7fb7 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/base_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/base_scheduler.py @@ -398,12 +398,16 @@ async def _process_started_tasks( user_id=user_id, user_email=run_metadata.get("user_email", UNDEFINED_STR_METADATA), project_id=t.project_id, - project_name=run_metadata.get( - "project_name", UNDEFINED_STR_METADATA + project_name=run_metadata.get("project_metadata", {}).get( + "parent_project_name", + run_metadata.get("project_name", UNDEFINED_STR_METADATA), ), node_id=t.node_id, - node_name=run_metadata.get("node_id_names_map", {}).get( - t.node_id, UNDEFINED_STR_METADATA + node_name=run_metadata.get("project_metadata", {}).get( + "parent_node_name", + run_metadata.get("node_id_names_map", {}).get( + t.node_id, UNDEFINED_STR_METADATA + ), ), service_key=ServiceKey(t.image.name), service_version=ServiceVersion(t.image.tag), diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py index 6f25d450d95..14a4b0d42c3 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py @@ -21,9 +21,6 @@ from models_library.users import UserID from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from servicelib.logging_utils import log_catch -from simcore_service_director_v2.modules.db.repositories.comp_runs import ( - CompRunsRepository, -) from ...core.errors import ( ComputationalBackendOnDemandNotReadyError, @@ -35,6 +32,7 @@ from ...modules.dask_client import DaskClient from ...modules.dask_clients_pool import DaskClientsPool from ...modules.db.repositories.clusters import ClustersRepository +from ...modules.db.repositories.comp_runs import CompRunsRepository from ...utils.comp_scheduler import Iteration, get_resource_tracking_run_id from ...utils.dask import ( clean_task_output_and_log_files_if_invalid, @@ -360,9 +358,9 @@ async def _task_log_change_handler(self, event: str) -> None: *_, user_id, project_id, node_id = parse_dask_job_id(task_log_event.job_id) await publish_service_log( self.rabbitmq_client, - user_id, - project_id, - node_id, - task_log_event.log, - task_log_event.log_level, + user_id=user_id, + project_id=project_id, + node_id=node_id, + log=task_log_event.log, + log_level=task_log_event.log_level, ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py index 5332f997d39..f52607e4bc0 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py @@ -137,6 +137,7 @@ async def update( async def set_run_result( self, + *, user_id: UserID, project_id: ProjectID, iteration: PositiveInt, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py index 43734c7c3b2..856c0ec3650 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py @@ -4,6 +4,7 @@ from aiopg.sa.result import RowProxy from models_library.projects import ProjectAtDB, ProjectID from models_library.projects_nodes_io import NodeID +from simcore_postgres_database.utils_projects_nodes import ProjectNodesRepo from ....core.errors import ProjectNotFoundError from ..tables import projects @@ -32,3 +33,9 @@ async def is_node_present_in_workbench( return f"{node_uuid}" in project.workbench except ProjectNotFoundError: return False + + async def get_project_id_from_node(self, node_id: NodeID) -> ProjectID: + async with self.db_engine.acquire() as conn: + return await ProjectNodesRepo.get_project_id_from_node_id( + conn, node_id=node_id + ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_metadata.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_metadata.py new file mode 100644 index 00000000000..6d364e9536d --- /dev/null +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects_metadata.py @@ -0,0 +1,23 @@ +from typing import Any + +from models_library.projects import ProjectID +from simcore_postgres_database.utils_projects_metadata import ProjectMetadata +from simcore_postgres_database.utils_projects_metadata import ( + get as projects_metadata_get, +) + +from ._base import BaseRepository + + +class ProjectsMetadataRepository(BaseRepository): + async def get_metadata(self, project_id: ProjectID) -> dict[str, Any] | None: + """ + Raises: + DBProjectNotFoundError + """ + async with self.db_engine.acquire() as conn: + project_custom_metadata: ProjectMetadata = await projects_metadata_get( + conn, project_id + ) + custom_metadata: dict[str, Any] | None = project_custom_metadata.custom + return custom_metadata diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index 86e25488eff..de82bcdeecb 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -341,7 +341,7 @@ async def compute_task_envs( if task_envs: vendor_substituted_envs = await substitute_vendor_secrets_in_specs( app, - node_image.envs, + cast(dict[str, Any], node_image.envs), service_key=ServiceKey(node_image.name), service_version=ServiceVersion(node_image.tag), product_name=product_name, diff --git a/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py b/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py index f888115c17a..feea24b83cb 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/rabbitmq.py @@ -135,6 +135,7 @@ async def publish_service_resource_tracking_heartbeat( async def publish_service_log( rabbitmq_client: RabbitMQClient, + *, user_id: UserID, project_id: ProjectID, node_id: NodeID, @@ -154,6 +155,7 @@ async def publish_service_log( async def publish_service_progress( rabbitmq_client: RabbitMQClient, + *, user_id: UserID, project_id: ProjectID, node_id: NodeID, diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_db_repositories_projects.py b/services/director-v2/tests/unit/with_dbs/test_modules_db_repositories_projects.py index bf7dfcc36d1..c535efeaefb 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_db_repositories_projects.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_db_repositories_projects.py @@ -1,7 +1,8 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument -from typing import Any, Awaitable, Callable, cast +from collections.abc import Awaitable, Callable +from typing import Any import pytest import sqlalchemy as sa @@ -12,6 +13,7 @@ from pytest import MonkeyPatch from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from simcore_postgres_database.utils_projects_nodes import ProjectNodesNodeNotFound from simcore_service_director_v2.modules.db.repositories.projects import ( ProjectsRepository, ) @@ -79,10 +81,7 @@ async def project( async def test_is_node_present_in_workbench( initialized_app: FastAPI, project: ProjectAtDB, faker: Faker ): - project_repository = cast( - ProjectsRepository, - get_repository(initialized_app, ProjectsRepository), - ) + project_repository = get_repository(initialized_app, ProjectsRepository) for node_uuid in project.workbench: assert ( @@ -109,3 +108,18 @@ async def test_is_node_present_in_workbench( ) is False ) + + +async def test_get_project_id_from_node( + initialized_app: FastAPI, project: ProjectAtDB, faker: Faker +): + project_repository = get_repository(initialized_app, ProjectsRepository) + for node_uuid in project.workbench: + assert ( + await project_repository.get_project_id_from_node(NodeID(node_uuid)) + == project.uuid + ) + + not_existing_node_id = faker.uuid4(cast_to=None) + with pytest.raises(ProjectNodesNodeNotFound): + await project_repository.get_project_id_from_node(not_existing_node_id) From 5035fe5cc7a7d7ee9fcab15fd7f8a116a4934cd6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Mon, 20 Nov 2023 10:47:36 +0000 Subject: [PATCH 19/24] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20director-v2=20create?= =?UTF-8?q?d=20api-keys=20are=20removed=20if=20not=20used=20by=20any=20ser?= =?UTF-8?q?vice=20=E2=9A=A0=EF=B8=8F=20(#5043)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../service-library/src/servicelib/redis.py | 3 - .../src/settings_library/redis.py | 1 + .../tests/manual/docker-compose.yml | 10 +- services/clusters-keeper/docker-compose.yml | 10 +- .../modules/api_keys_manager.py | 136 +++++--- .../dynamic_sidecar/docker_compose_specs.py | 4 +- .../scheduler/_core/_events_user_services.py | 1 + .../scheduler/_core/_events_utils.py | 5 +- .../modules/osparc_variables_substitutions.py | 19 +- .../utils/base_distributed_identifier.py | 281 ++++++++++++++++ .../utils/distributed_identifer.py | 69 ---- .../unit/test_modules_api_keys_manager.py | 123 +++++-- .../unit/test_utils_distributed_identifier.py | 311 +++++++++++++++--- services/docker-compose-ops.yml | 10 +- services/docker-compose.yml | 2 +- 15 files changed, 794 insertions(+), 191 deletions(-) create mode 100644 services/director-v2/src/simcore_service_director_v2/utils/base_distributed_identifier.py delete mode 100644 services/director-v2/src/simcore_service_director_v2/utils/distributed_identifer.py diff --git a/packages/service-library/src/servicelib/redis.py b/packages/service-library/src/servicelib/redis.py index 32b87149c5b..2cb393e0dff 100644 --- a/packages/service-library/src/servicelib/redis.py +++ b/packages/service-library/src/servicelib/redis.py @@ -21,9 +21,6 @@ from .logging_utils import log_catch, log_context _DEFAULT_LOCK_TTL: Final[datetime.timedelta] = datetime.timedelta(seconds=10) -_CANCEL_TASK_TIMEOUT: Final[datetime.timedelta] = datetime.timedelta(seconds=0.1) -_MINUTE: Final[NonNegativeFloat] = 60 -_WAIT_SECS: Final[NonNegativeFloat] = 1 logger = logging.getLogger(__name__) diff --git a/packages/settings-library/src/settings_library/redis.py b/packages/settings-library/src/settings_library/redis.py index e973630cb02..30a95ce098f 100644 --- a/packages/settings-library/src/settings_library/redis.py +++ b/packages/settings-library/src/settings_library/redis.py @@ -15,6 +15,7 @@ class RedisDatabase(int, Enum): SCHEDULED_MAINTENANCE = 3 USER_NOTIFICATIONS = 4 ANNOUNCEMENTS = 5 + DISTRIBUTED_IDENTIFIERS = 6 class RedisSettings(BaseCustomSettings): diff --git a/services/autoscaling/tests/manual/docker-compose.yml b/services/autoscaling/tests/manual/docker-compose.yml index e70eb62ca3c..7dbcf5bd4cc 100644 --- a/services/autoscaling/tests/manual/docker-compose.yml +++ b/services/autoscaling/tests/manual/docker-compose.yml @@ -40,7 +40,15 @@ services: ports: - "18081:8081" environment: - - REDIS_HOSTS=resources:${REDIS_HOST}:${REDIS_PORT}:0,locks:${REDIS_HOST}:${REDIS_PORT}:1,validation_codes:${REDIS_HOST}:${REDIS_PORT}:2,scheduled_maintenance:${REDIS_HOST}:${REDIS_PORT}:3,user_notifications:${REDIS_HOST}:${REDIS_PORT}:4,announcements:${REDIS_HOST}:${REDIS_PORT}:5 + - >- + REDIS_HOSTS= + resources:${REDIS_HOST}:${REDIS_PORT}:0, + locks:${REDIS_HOST}:${REDIS_PORT}:1, + validation_codes:${REDIS_HOST}:${REDIS_PORT}:2, + scheduled_maintenance:${REDIS_HOST}:${REDIS_PORT}:3, + user_notifications:${REDIS_HOST}:${REDIS_PORT}:4, + announcements:${REDIS_HOST}:${REDIS_PORT}:5, + distributed_identifiers:${REDIS_HOST}:${REDIS_PORT}:6 # If you add/remove a db, do not forget to update the --databases entry in the docker-compose.yml autoscaling: diff --git a/services/clusters-keeper/docker-compose.yml b/services/clusters-keeper/docker-compose.yml index 7ab1543d273..1d8d5126194 100644 --- a/services/clusters-keeper/docker-compose.yml +++ b/services/clusters-keeper/docker-compose.yml @@ -35,7 +35,15 @@ services: ports: - "18081:8081" environment: - - REDIS_HOSTS=resources:${REDIS_HOST}:${REDIS_PORT}:0,locks:${REDIS_HOST}:${REDIS_PORT}:1,validation_codes:${REDIS_HOST}:${REDIS_PORT}:2,scheduled_maintenance:${REDIS_HOST}:${REDIS_PORT}:3,user_notifications:${REDIS_HOST}:${REDIS_PORT}:4,announcements:${REDIS_HOST}:${REDIS_PORT}:5 + - >- + REDIS_HOSTS= + resources:${REDIS_HOST}:${REDIS_PORT}:0, + locks:${REDIS_HOST}:${REDIS_PORT}:1, + validation_codes:${REDIS_HOST}:${REDIS_PORT}:2, + scheduled_maintenance:${REDIS_HOST}:${REDIS_PORT}:3, + user_notifications:${REDIS_HOST}:${REDIS_PORT}:4, + announcements:${REDIS_HOST}:${REDIS_PORT}:5, + distributed_identifiers:${REDIS_HOST}:${REDIS_PORT}:6 # If you add/remove a db, do not forget to update the --databases entry in the docker-compose.yml clusters-keeper: diff --git a/services/director-v2/src/simcore_service_director_v2/modules/api_keys_manager.py b/services/director-v2/src/simcore_service_director_v2/modules/api_keys_manager.py index 0fa6f457fe8..618458446f7 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/api_keys_manager.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/api_keys_manager.py @@ -1,3 +1,5 @@ +from datetime import timedelta +from typing import Any from uuid import uuid5 from fastapi import FastAPI @@ -6,38 +8,62 @@ from models_library.products import ProductName from models_library.projects_nodes_io import NodeID from models_library.rabbitmq_basic_types import RPCMethodName +from models_library.services import RunID from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import BaseModel, StrBytes, parse_obj_as from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.redis import RedisClientSDK +from settings_library.redis import RedisDatabase -from ..utils.distributed_identifer import BaseDistributedIdentifierManager +from ..core.settings import AppSettings +from ..utils.base_distributed_identifier import BaseDistributedIdentifierManager from .rabbitmq import get_rabbitmq_rpc_client +_CLEANUP_INTERVAL = timedelta(minutes=5) -class APIKeysManager(BaseDistributedIdentifierManager[str, ApiKeyGet]): - def __init__(self, app: FastAPI) -> None: - self.GET_OR_CREATE_INJECTS_IDENTIFIER = True + +class CleanupContext(BaseModel): + # used for checking if used + node_id: NodeID + + # used for removing + product_name: ProductName + user_id: UserID + + +class APIKeysManager(BaseDistributedIdentifierManager[str, ApiKeyGet, CleanupContext]): + def __init__(self, app: FastAPI, redis_client_sdk: RedisClientSDK) -> None: + super().__init__(redis_client_sdk, cleanup_interval=_CLEANUP_INTERVAL) self.app = app @property def rpc_client(self) -> RabbitMQRPCClient: return get_rabbitmq_rpc_client(self.app) - # pylint:disable=arguments-differ + @classmethod + def _deserialize_identifier(cls, raw: str) -> str: + return raw - async def get( # type:ignore [override] - self, identifier: str, product_name: ProductName, user_id: UserID - ) -> ApiKeyGet | None: - result = await self.rpc_client.request( - WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "api_key_get"), - product_name=product_name, - user_id=user_id, - name=identifier, + @classmethod + def _serialize_identifier(cls, identifier: str) -> str: + return identifier + + @classmethod + def _deserialize_cleanup_context(cls, raw: StrBytes) -> CleanupContext: + return CleanupContext.parse_raw(raw) + + @classmethod + def _serialize_cleanup_context(cls, cleanup_context: CleanupContext) -> str: + return cleanup_context.json() + + async def is_used(self, identifier: str, cleanup_context: CleanupContext) -> bool: + _ = identifier + scheduler: "DynamicSidecarsScheduler" = ( # type:ignore [name-defined] # noqa: F821 + self.app.state.dynamic_sidecar_scheduler ) - return parse_obj_as(ApiKeyGet | None, result) + return bool(scheduler.is_service_tracked(cleanup_context.node_id)) - async def create( # type:ignore [override] + async def _create( # type:ignore [override] # pylint:disable=arguments-differ self, identifier: str, product_name: ProductName, user_id: UserID ) -> tuple[str, ApiKeyGet]: result = await self.rpc_client.request( @@ -49,49 +75,68 @@ async def create( # type:ignore [override] ) return identifier, ApiKeyGet.parse_obj(result) - async def destroy( # type:ignore [override] + async def get( # type:ignore [override] # pylint:disable=arguments-differ self, identifier: str, product_name: ProductName, user_id: UserID - ) -> None: - await self.rpc_client.request( + ) -> ApiKeyGet | None: + result: Any | None = await self.rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "delete_api_keys"), + parse_obj_as(RPCMethodName, "api_key_get"), product_name=product_name, user_id=user_id, name=identifier, ) + return parse_obj_as(ApiKeyGet | None, result) + + async def _destroy(self, identifier: str, cleanup_context: CleanupContext) -> None: + await self.rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "delete_api_keys"), + product_name=cleanup_context.product_name, + user_id=cleanup_context.user_id, + name=identifier, + ) async def get_or_create_api_key( - app: FastAPI, *, product_name: ProductName, user_id: UserID, node_id: NodeID + app: FastAPI, + *, + product_name: ProductName, + user_id: UserID, + node_id: NodeID, + run_id: RunID, ) -> ApiKeyGet: api_keys_manager = _get_api_keys_manager(app) - display_name = _get_api_key_name(node_id) + display_name = _get_api_key_name(node_id, run_id) - key_data: ApiKeyGet | None = await api_keys_manager.get( + api_key: ApiKeyGet | None = await api_keys_manager.get( identifier=display_name, product_name=product_name, user_id=user_id ) - if key_data is None: - _, key_data = await api_keys_manager.create( - identifier=display_name, product_name=product_name, user_id=user_id + if api_key is None: + _, api_key = await api_keys_manager.create( + cleanup_context=CleanupContext( + node_id=node_id, product_name=product_name, user_id=user_id + ), + identifier=display_name, + product_name=product_name, + user_id=user_id, ) - return key_data + return api_key -async def safe_remove( - app: FastAPI, *, node_id: NodeID, product_name: ProductName, user_id: UserID -) -> None: +async def safe_remove(app: FastAPI, *, node_id: NodeID, run_id: RunID) -> None: api_keys_manager = _get_api_keys_manager(app) - display_name = _get_api_key_name(node_id) + display_name = _get_api_key_name(node_id, run_id) - await api_keys_manager.remove( - identifier=display_name, product_name=product_name, user_id=user_id - ) + await api_keys_manager.remove(identifier=display_name) -def _get_api_key_name(node_id: NodeID) -> str: - obfuscated_node_id = uuid5(node_id, f"{node_id}") - return f"_auto_{obfuscated_node_id}" +def _get_api_key_name(node_id: NodeID, run_id: RunID) -> str: + # Generates a new unique key name for each service run + # This avoids race conditions if the service is starting and + # an the cleanup job is removing the key from an old run which + # was wrongly shut down + return f"_auto_{uuid5(node_id, run_id)}" def _get_api_keys_manager(app: FastAPI) -> APIKeysManager: @@ -101,6 +146,19 @@ def _get_api_keys_manager(app: FastAPI) -> APIKeysManager: def setup(app: FastAPI) -> None: async def on_startup() -> None: - app.state.api_keys_manager = APIKeysManager(app) + settings: AppSettings = app.state.settings + + redis_dsn = settings.REDIS.build_redis_dsn( + RedisDatabase.DISTRIBUTED_IDENTIFIERS + ) + redis_client_sdk = RedisClientSDK(redis_dsn) + app.state.api_keys_manager = manager = APIKeysManager(app, redis_client_sdk) + + await manager.setup() + + async def on_shutdown() -> None: + manager: APIKeysManager = app.state.api_keys_manager + await manager.shutdown() app.add_event_handler("startup", on_startup) + app.add_event_handler("shutdown", on_shutdown) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py index e67c3c45f64..b8215e54ec1 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py @@ -12,7 +12,7 @@ PathMappingsLabel, SimcoreServiceLabels, ) -from models_library.services import ServiceKey, ServiceVersion +from models_library.services import RunID, ServiceKey, ServiceVersion from models_library.services_resources import ( DEFAULT_SINGLE_SERVICE_NAME, ResourcesDict, @@ -275,6 +275,7 @@ async def assemble_spec( # pylint: disable=too-many-arguments # noqa: PLR0913 node_id: NodeID, simcore_user_agent: str, swarm_stack_name: str, + run_id: RunID, ) -> str: """ returns a docker-compose spec used by @@ -391,6 +392,7 @@ async def assemble_spec( # pylint: disable=too-many-arguments # noqa: PLR0913 product_name=product_name, user_id=user_id, node_id=node_id, + run_id=run_id, ) stringified_service_spec: str = replace_env_vars_in_compose_spec( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py index e63bffe2828..bfcec1548e3 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_user_services.py @@ -98,6 +98,7 @@ async def create_user_services(app: FastAPI, scheduler_data: SchedulerData): node_id=scheduler_data.node_uuid, simcore_user_agent=scheduler_data.request_simcore_user_agent, swarm_stack_name=dynamic_services_scheduler_settings.SWARM_STACK_NAME, + run_id=scheduler_data.run_id, ) _logger.debug( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index 0722c8a6faf..99d13983d70 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -310,10 +310,7 @@ async def attempt_pod_removal_and_data_saving( ) await safe_remove( - app, - node_id=scheduler_data.node_uuid, - product_name=scheduler_data.product_name, - user_id=scheduler_data.user_id, + app, node_id=scheduler_data.node_uuid, run_id=scheduler_data.run_id ) # remove sidecar's api client diff --git a/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py index 078c6f9eae2..e1825d8eea3 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py @@ -14,7 +14,7 @@ from models_library.products import ProductName from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID -from models_library.services import ServiceKey, ServiceVersion +from models_library.services import RunID, ServiceKey, ServiceVersion from models_library.users import UserID from models_library.utils.specs_substitution import SpecsSubstitutionsResolver from pydantic import BaseModel, EmailStr @@ -187,6 +187,7 @@ async def resolve_and_substitute_service_lifetime_variables_in_specs( product_name: ProductName, user_id: UserID, node_id: NodeID, + run_id: RunID, safe: bool = True, ) -> dict[str, Any]: registry: OsparcVariablesTable = app.state.lifespan_osparc_variables_table @@ -204,6 +205,7 @@ async def resolve_and_substitute_service_lifetime_variables_in_specs( product_name=product_name, user_id=user_id, node_id=node_id, + run_id=run_id, ), # NOTE: the api key and secret cannot be resolved in parallel # due to race conditions @@ -218,25 +220,35 @@ async def resolve_and_substitute_service_lifetime_variables_in_specs( async def _get_or_create_api_key( - app: FastAPI, product_name: ProductName, user_id: UserID, node_id: NodeID + app: FastAPI, + product_name: ProductName, + user_id: UserID, + node_id: NodeID, + run_id: RunID, ) -> str: key_data = await get_or_create_api_key( app, product_name=product_name, user_id=user_id, node_id=node_id, + run_id=run_id, ) return key_data.api_key # type:ignore [no-any-return] async def _get_or_create_api_secret( - app: FastAPI, product_name: ProductName, user_id: UserID, node_id: NodeID + app: FastAPI, + product_name: ProductName, + user_id: UserID, + node_id: NodeID, + run_id: RunID, ) -> str: key_data = await get_or_create_api_key( app, product_name=product_name, user_id=user_id, node_id=node_id, + run_id=run_id, ) return key_data.api_secret # type:ignore [no-any-return] @@ -271,7 +283,6 @@ def _setup_session_osparc_variables(app: FastAPI): ("OSPARC_VARIABLE_PRODUCT_NAME", "product_name"), ("OSPARC_VARIABLE_STUDY_UUID", "project_id"), ("OSPARC_VARIABLE_NODE_ID", "node_id"), - # ANE -> PC: why not register the user_id as well at this point? ]: table.register_from_context(name, context_name) diff --git a/services/director-v2/src/simcore_service_director_v2/utils/base_distributed_identifier.py b/services/director-v2/src/simcore_service_director_v2/utils/base_distributed_identifier.py new file mode 100644 index 00000000000..b52db358631 --- /dev/null +++ b/services/director-v2/src/simcore_service_director_v2/utils/base_distributed_identifier.py @@ -0,0 +1,281 @@ +import logging +from abc import ABC, abstractmethod +from asyncio import Task +from datetime import timedelta +from typing import Final, Generic, TypeVar + +from pydantic import NonNegativeInt +from servicelib.background_task import start_periodic_task, stop_periodic_task +from servicelib.logging_utils import log_catch, log_context +from servicelib.redis import RedisClientSDK, RedisDatabase +from servicelib.utils import logged_gather + +_logger = logging.getLogger(__name__) + +_REDIS_MAX_CONCURRENCY: Final[NonNegativeInt] = 10 +_DEFAULT_CLEANUP_INTERVAL: Final[timedelta] = timedelta(minutes=1) + +Identifier = TypeVar("Identifier") +ResourceObject = TypeVar("ResourceObject") +CleanupContext = TypeVar("CleanupContext") + + +class BaseDistributedIdentifierManager( + ABC, Generic[Identifier, ResourceObject, CleanupContext] +): + """Used to implement managers for resources that require book keeping + in a distributed system. + + NOTE: that ``Identifier`` and ``CleanupContext`` are serialized and deserialized + to and from Redis. + + Generics: + Identifier -- a user defined object: used to uniquely identify the resource + ResourceObject -- a user defined object: referring to an existing resource + CleanupContext -- a user defined object: contains all necessary + arguments used for removal and cleanup. + """ + + def __init__( + self, + redis_client_sdk: RedisClientSDK, + *, + cleanup_interval: timedelta = _DEFAULT_CLEANUP_INTERVAL, + ) -> None: + """ + Arguments: + redis_client_sdk -- client connecting to Redis + + Keyword Arguments: + cleanup_interval -- interval at which cleanup for unused + resources runs (default: {_DEFAULT_CLEANUP_INTERVAL}) + """ + + if not redis_client_sdk.redis_dsn.endswith( + f"{RedisDatabase.DISTRIBUTED_IDENTIFIERS}" + ): + msg = ( + f"Redis endpoint {redis_client_sdk.redis_dsn} contains the wrong database." + f"Expected {RedisDatabase.DISTRIBUTED_IDENTIFIERS}" + ) + raise TypeError(msg) + + self._redis_client_sdk = redis_client_sdk + self.cleanup_interval = cleanup_interval + + self._cleanup_task: Task | None = None + + async def setup(self) -> None: + self._cleanup_task = start_periodic_task( + self._cleanup_unused_identifiers, + interval=self.cleanup_interval, + task_name="cleanup_unused_identifiers_task", + ) + + async def shutdown(self) -> None: + if self._cleanup_task: + await stop_periodic_task(self._cleanup_task, timeout=5) + + @classmethod + def class_path(cls) -> str: + return f"{cls.__module__}.{cls.__name__}" + + @classmethod + def _redis_key_prefix(cls) -> str: + return f"{cls.class_path()}:" + + @classmethod + def _to_redis_key(cls, identifier: Identifier) -> str: + return f"{cls._redis_key_prefix()}{cls._serialize_identifier(identifier)}" + + @classmethod + def _from_redis_key(cls, redis_key: str) -> Identifier: + sad = redis_key.removeprefix(cls._redis_key_prefix()) + return cls._deserialize_identifier(sad) + + async def _get_identifier_context( + self, identifier: Identifier + ) -> CleanupContext | None: + raw: str | None = await self._redis_client_sdk.redis.get( + self._to_redis_key(identifier) + ) + return self._deserialize_cleanup_context(raw) if raw else None + + async def _get_tracked(self) -> dict[Identifier, CleanupContext]: + identifiers: list[Identifier] = [ + self._from_redis_key(redis_key) + for redis_key in await self._redis_client_sdk.redis.keys( + f"{self._redis_key_prefix()}*" + ) + ] + + cleanup_contexts: list[CleanupContext | None] = await logged_gather( + *(self._get_identifier_context(identifier) for identifier in identifiers), + max_concurrency=_REDIS_MAX_CONCURRENCY, + ) + + return { + identifier: cleanup_context + for identifier, cleanup_context in zip( + identifiers, cleanup_contexts, strict=True + ) + # NOTE: cleanup_context will be None if the key was removed before + # recovering all the cleanup_contexts + if cleanup_context is not None + } + + async def _cleanup_unused_identifiers(self) -> None: + # removes no longer used identifiers + tracked_data: dict[Identifier, CleanupContext] = await self._get_tracked() + _logger.info("Will remove unused %s", list(tracked_data.keys())) + + for identifier, cleanup_context in tracked_data.items(): + if await self.is_used(identifier, cleanup_context): + continue + + await self.remove(identifier) + + async def create( + self, *, cleanup_context: CleanupContext, **extra_kwargs + ) -> tuple[Identifier, ResourceObject]: + """Used for creating the resources + + Arguments: + cleanup_context -- user defined CleanupContext object + **extra_kwargs -- can be overloaded by the user + + Returns: + tuple[identifier for the resource, resource object] + """ + identifier, result = await self._create(**extra_kwargs) + await self._redis_client_sdk.redis.set( + self._to_redis_key(identifier), + self._serialize_cleanup_context(cleanup_context), + ) + return identifier, result + + async def remove(self, identifier: Identifier, *, reraise: bool = False) -> None: + """Attempts to remove the resource, if an error occurs it is logged. + + Arguments: + identifier -- user chosen identifier for the resource + reraise -- when True raises any exception raised by ``destroy`` (default: {False}) + """ + + cleanup_context = await self._get_identifier_context(identifier) + if cleanup_context is None: + _logger.warning( + "Something went wrong, did not find any context for %s", identifier + ) + return + + with log_context( + _logger, logging.DEBUG, f"{self.__class__}: removing {identifier}" + ), log_catch(_logger, reraise=reraise): + await self._destroy(identifier, cleanup_context) + + await self._redis_client_sdk.redis.delete(self._to_redis_key(identifier)) + + @classmethod + @abstractmethod + def _deserialize_identifier(cls, raw: str) -> Identifier: + """User provided deserialization for the identifier + + Arguments: + raw -- stream to be deserialized + + Returns: + an identifier object + """ + + @classmethod + @abstractmethod + def _serialize_identifier(cls, identifier: Identifier) -> str: + """User provided serialization for the identifier + + Arguments: + cleanup_context -- user defined identifier object + + Returns: + object encoded as string + """ + + @classmethod + @abstractmethod + def _deserialize_cleanup_context(cls, raw: str) -> CleanupContext: + """User provided deserialization for the context + + Arguments: + raw -- stream to be deserialized + + Returns: + an object of the type chosen by the user + """ + + @classmethod + @abstractmethod + def _serialize_cleanup_context(cls, cleanup_context: CleanupContext) -> str: + """User provided serialization for the context + + Arguments: + cleanup_context -- user defined cleanup context object + + Returns: + object encoded as string + """ + + @abstractmethod + async def is_used( + self, identifier: Identifier, cleanup_context: CleanupContext + ) -> bool: + """Check if the resource associated to the ``identifier`` is + still being used. + # NOTE: a resource can be created but not in use. + + Arguments: + identifier -- user chosen identifier for the resource + cleanup_context -- user defined CleanupContext object + + Returns: + True if ``identifier`` is still being used + """ + + @abstractmethod + async def _create(self, **extra_kwargs) -> tuple[Identifier, ResourceObject]: + """Used INTERNALLY for creating the resources. + # NOTE: should not be used directly, use the public + version ``create`` instead. + + Arguments: + **extra_kwargs -- can be overloaded by the user + + Returns: + tuple[identifier for the resource, resource object] + """ + + @abstractmethod + async def get( + self, identifier: Identifier, **extra_kwargs + ) -> ResourceObject | None: + """If exists, returns the resource. + + Arguments: + identifier -- user chosen identifier for the resource + **extra_kwargs -- can be overloaded by the user + + Returns: + None if the resource does not exit + """ + + @abstractmethod + async def _destroy( + self, identifier: Identifier, cleanup_context: CleanupContext + ) -> None: + """Used to destroy an existing resource + # NOTE: should not be used directly, use the public + version ``remove`` instead. + + Arguments: + identifier -- user chosen identifier for the resource + cleanup_context -- user defined CleanupContext object + """ diff --git a/services/director-v2/src/simcore_service_director_v2/utils/distributed_identifer.py b/services/director-v2/src/simcore_service_director_v2/utils/distributed_identifer.py deleted file mode 100644 index 05d84a72f08..00000000000 --- a/services/director-v2/src/simcore_service_director_v2/utils/distributed_identifer.py +++ /dev/null @@ -1,69 +0,0 @@ -import logging -from abc import ABC, abstractmethod -from typing import Generic, TypeVar - -from servicelib.logging_utils import log_catch, log_context - -_logger = logging.getLogger(__name__) - -Ident = TypeVar("Ident") -Res = TypeVar("Res") - - -class BaseDistributedIdentifierManager(ABC, Generic[Ident, Res]): - """Common interface used to manage the lifecycle of osparc resources. - - An osparc resource can be anything that needs to be created and then removed - during the runtime of the platform. - - Safe remove the resource. - - For usage check ``packages/service-library/tests/test_osparc_generic_resource.py`` - """ - - @abstractmethod - async def get(self, identifier: Ident, **extra_kwargs) -> Res | None: - """Returns a resource if exists. - - Arguments: - identifier -- user chosen identifier for the resource - **extra_kwargs -- can be overloaded by the user - - Returns: - None if the resource does not exit - """ - - @abstractmethod - async def create(self, **extra_kwargs) -> tuple[Ident, Res]: - """Used for creating the resources - - Arguments: - **extra_kwargs -- can be overloaded by the user - - Returns: - tuple[identifier for the resource, resource object] - """ - - @abstractmethod - async def destroy(self, identifier: Ident, **extra_kwargs) -> None: - """Used to destroy an existing resource - - Arguments: - identifier -- user chosen identifier for the resource - **extra_kwargs -- can be overloaded by the user - """ - - async def remove( - self, identifier: Ident, *, reraise: bool = False, **extra_kwargs - ) -> None: - """Attempts to remove the resource, if an error occurs it is logged. - - Arguments: - identifier -- user chosen identifier for the resource - reraise -- when True raises any exception raised by ``destroy`` (default: {False}) - **extra_kwargs -- can be overloaded by the user - """ - with log_context( - _logger, logging.DEBUG, f"{self.__class__}: removing {identifier}" - ), log_catch(_logger, reraise=reraise): - await self.destroy(identifier, **extra_kwargs) diff --git a/services/director-v2/tests/unit/test_modules_api_keys_manager.py b/services/director-v2/tests/unit/test_modules_api_keys_manager.py index 3c56a05936c..2e9da785228 100644 --- a/services/director-v2/tests/unit/test_modules_api_keys_manager.py +++ b/services/director-v2/tests/unit/test_modules_api_keys_manager.py @@ -1,7 +1,9 @@ +# pylint:disable=protected-access # pylint:disable=redefined-outer-name # pylint:disable=unused-argument from collections.abc import Awaitable, Callable +from unittest.mock import AsyncMock import pytest from faker import Faker @@ -10,16 +12,22 @@ from models_library.api_schemas_webserver.auth import ApiKeyCreate, ApiKeyGet from models_library.products import ProductName from models_library.projects_nodes_io import NodeID +from models_library.services import RunID from models_library.users import UserID from pytest_mock import MockerFixture from servicelib.rabbitmq import RabbitMQRPCClient, RPCRouter +from servicelib.redis import RedisClientSDK +from settings_library.redis import RedisDatabase, RedisSettings from simcore_service_director_v2.modules.api_keys_manager import ( APIKeysManager, _get_api_key_name, + get_or_create_api_key, + safe_remove, ) pytest_simcore_core_services_selection = [ "rabbit", + "redis", ] @@ -28,15 +36,38 @@ def node_id(faker: Faker) -> NodeID: return faker.uuid4(cast_to=None) -def test_get_api_key_name_is_not_randomly_generated(node_id: NodeID): - api_key_names = {_get_api_key_name(node_id) for x in range(1000)} +@pytest.fixture +def run_id(faker: Faker) -> RunID: + return RunID(faker.pystr()) + + +@pytest.fixture +def product_name(faker: Faker) -> ProductName: + return faker.pystr() + + +@pytest.fixture +def user_id(faker: Faker) -> UserID: + return faker.pyint() + + +def test_get_api_key_name_is_not_randomly_generated(node_id: NodeID, run_id: RunID): + api_key_names = {_get_api_key_name(node_id, run_id) for _ in range(1000)} assert len(api_key_names) == 1 +@pytest.fixture +def redis_client_sdk(redis_service: RedisSettings) -> RedisClientSDK: + return RedisClientSDK( + redis_service.build_redis_dsn(RedisDatabase.DISTRIBUTED_IDENTIFIERS) + ) + + @pytest.fixture async def mock_rpc_server( rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], mocker: MockerFixture, + faker: Faker, ) -> RabbitMQRPCClient: rpc_client = await rabbitmq_rpc_client("client") rpc_server = await rabbitmq_rpc_client("mock_server") @@ -45,17 +76,25 @@ async def mock_rpc_server( # mocks the interface defined in the webserver + _storage: dict[str, ApiKeyGet] = {} + @router.expose() async def api_key_get( product_name: ProductName, user_id: UserID, name: str - ) -> ApiKeyGet: - return ApiKeyGet.parse_obj(ApiKeyGet.Config.schema_extra["examples"][0]) + ) -> ApiKeyGet | None: + return _storage.get(f"{product_name}{user_id}", None) @router.expose() async def create_api_keys( product_name: ProductName, user_id: UserID, new: ApiKeyCreate ) -> ApiKeyGet: - return ApiKeyGet.parse_obj(ApiKeyGet.Config.schema_extra["examples"][0]) + api_key = ApiKeyGet( + display_name=new.display_name, + api_key=faker.pystr(), + api_secret=faker.pystr(), + ) + _storage[f"{product_name}{user_id}"] = api_key + return api_key @router.expose() async def delete_api_keys( @@ -74,27 +113,67 @@ async def delete_api_keys( return rpc_client -async def test_rpc_endpoints( - mock_rpc_server: RabbitMQRPCClient, - faker: Faker, -): - manager = APIKeysManager(FastAPI()) +@pytest.fixture +def app() -> FastAPI: + return FastAPI() + + +@pytest.fixture +def mock_dynamic_sidecars_scheduler(app: FastAPI, is_used: bool) -> None: + scheduler_mock = AsyncMock() + scheduler_mock.is_service_tracked = lambda _: is_used + app.state.dynamic_sidecar_scheduler = scheduler_mock - identifier = faker.pystr() - product_name = faker.pystr() - user_id = faker.pyint() - api_key = await manager.get( - identifier=identifier, product_name=product_name, user_id=user_id +@pytest.fixture +def api_keys_manager( + mock_rpc_server: RabbitMQRPCClient, + mock_dynamic_sidecars_scheduler: None, + redis_client_sdk: RedisClientSDK, + app: FastAPI, +) -> APIKeysManager: + app.state.api_keys_manager = manager = APIKeysManager(app, redis_client_sdk) + return manager + + +async def _get_resource_count(api_keys_manager: APIKeysManager) -> int: + return len(await api_keys_manager._get_tracked()) # noqa: SLF001 + + +@pytest.mark.parametrize("is_used", [True]) +async def test_api_keys_workflow( + api_keys_manager: APIKeysManager, + app: FastAPI, + node_id: NodeID, + run_id: RunID, + product_name: ProductName, + user_id: UserID, +): + api_key = await get_or_create_api_key( + app, product_name=product_name, user_id=user_id, node_id=node_id, run_id=run_id ) assert isinstance(api_key, ApiKeyGet) - - identifier, api_key = await manager.create( - identifier=identifier, product_name=product_name, user_id=user_id + assert await _get_resource_count(api_keys_manager) == 1 + + await safe_remove(app, node_id=node_id, run_id=run_id) + assert await _get_resource_count(api_keys_manager) == 0 + + +@pytest.mark.parametrize("is_used", [False, True]) +async def test_background_cleanup( + api_keys_manager: APIKeysManager, + app: FastAPI, + node_id: NodeID, + run_id: RunID, + product_name: ProductName, + user_id: UserID, + is_used: bool, +) -> None: + api_key = await get_or_create_api_key( + app, product_name=product_name, user_id=user_id, node_id=node_id, run_id=run_id ) - assert isinstance(identifier, str) assert isinstance(api_key, ApiKeyGet) + assert await _get_resource_count(api_keys_manager) == 1 - await manager.destroy( - identifier=identifier, product_name=product_name, user_id=user_id - ) + await api_keys_manager._cleanup_unused_identifiers() # noqa: SLF001 + assert await _get_resource_count(api_keys_manager) == (1 if is_used else 0) diff --git a/services/director-v2/tests/unit/test_utils_distributed_identifier.py b/services/director-v2/tests/unit/test_utils_distributed_identifier.py index 9d645e7b1da..ce200feef97 100644 --- a/services/director-v2/tests/unit/test_utils_distributed_identifier.py +++ b/services/director-v2/tests/unit/test_utils_distributed_identifier.py @@ -1,105 +1,229 @@ +# pylint:disable=protected-access # pylint:disable=redefined-outer-name -import random +import asyncio import string -from typing import Any +from collections.abc import AsyncIterable, AsyncIterator +from dataclasses import dataclass +from secrets import choice +from typing import Final from uuid import UUID, uuid4 import pytest +from pydantic import BaseModel, NonNegativeInt, StrBytes from pytest_mock import MockerFixture -from simcore_service_director_v2.utils.distributed_identifer import ( +from servicelib.redis import RedisClientSDK +from servicelib.utils import logged_gather +from settings_library.redis import RedisDatabase, RedisSettings +from simcore_service_director_v2.utils.base_distributed_identifier import ( BaseDistributedIdentifierManager, ) +pytest_simcore_core_services_selection = [ + "redis", +] + +pytest_simcore_ops_services_selection = [ + # "redis-commander", +] + +# if this goes too high, max open file limit is reached +_MAX_REDIS_CONCURRENCY: Final[NonNegativeInt] = 1000 + -# define a custom type of ID for the API class UserDefinedID: + # define a custom type of ID for the API + # by choice it is hard to serialize/deserialize + def __init__(self, uuid: UUID | None = None) -> None: self._id = uuid if uuid else uuid4() def __eq__(self, other: "UserDefinedID") -> bool: return self._id == other._id + # only necessary for nice looking IDs in the logs + def __repr__(self) -> str: + return f"" + + # only necessary for RandomTextAPI def __hash__(self): return hash(str(self._id)) -# mocked api interface -class RandomTextAPI: - def __init__(self) -> None: - self._created: dict[UserDefinedID, Any] = {} +class RandomTextEntry(BaseModel): + text: str - @staticmethod - def _random_string(length: int) -> str: + @classmethod + def create(cls, length: int) -> "RandomTextEntry": letters_and_digits = string.ascii_letters + string.digits - return "".join( - random.choice(letters_and_digits) for _ in range(length) # noqa: S311 - ) + text = "".join(choice(letters_and_digits) for _ in range(length)) + return cls(text=text) - def create(self, length: int) -> tuple[UserDefinedID, Any]: + +class RandomTextAPI: + # Emulates an external API + # used to create resources + + def __init__(self) -> None: + self._created: dict[UserDefinedID, RandomTextEntry] = {} + + def create(self, length: int) -> tuple[UserDefinedID, RandomTextEntry]: identifier = UserDefinedID(uuid4()) - self._created[identifier] = self._random_string(length) + self._created[identifier] = RandomTextEntry.create(length) return identifier, self._created[identifier] def delete(self, identifier: UserDefinedID) -> None: del self._created[identifier] - def get(self, identifier: UserDefinedID) -> Any | None: + def get(self, identifier: UserDefinedID) -> RandomTextEntry | None: return self._created.get(identifier, None) -# define a custom manager using the custom user defined identifiers -# NOTE: note that the generic uses `[UserDefinedID, Any]` -# which enforces typing constraints on the overloaded abstract methods -class RandomTextResourcesManager(BaseDistributedIdentifierManager[UserDefinedID, Any]): - # pylint:disable=arguments-differ +@dataclass +class ComponentUsingRandomText: + # Emulates another component in the system + # using the created resources - def __init__(self) -> None: - self.api = RandomTextAPI() + _in_use: bool = True - async def get(self, identifier: UserDefinedID, **_) -> Any | None: - return self.api.get(identifier) + def is_used(self, an_id: UserDefinedID) -> bool: + _ = an_id + return self._in_use + + def toggle_usage(self, in_use: bool) -> None: + self._in_use = in_use + + +class AnEmptyTextCleanupContext(BaseModel): + # nothing is required during cleanup, so the context + # is an empty object. + # A ``pydantic.BaseModel`` is used for convenience + # this could have inherited from ``object`` + ... - async def create(self, length: int) -> tuple[UserDefinedID, Any]: + +class RandomTextResourcesManager( + BaseDistributedIdentifierManager[ + UserDefinedID, RandomTextEntry, AnEmptyTextCleanupContext + ] +): + # Implements a resource manager for handling the lifecycle of + # resources created by a service. + # It also comes in with automatic cleanup in case the service owing + # the resources failed to removed them in the past. + + def __init__( + self, + redis_client_sdk: RedisClientSDK, + component_using_random_text: ComponentUsingRandomText, + ) -> None: + # THESE two systems would normally come stored in the `app` context + self.api = RandomTextAPI() + self.component_using_random_text = component_using_random_text + + super().__init__(redis_client_sdk) + + @classmethod + def _deserialize_identifier(cls, raw: str) -> UserDefinedID: + return UserDefinedID(UUID(raw)) + + @classmethod + def _serialize_identifier(cls, identifier: UserDefinedID) -> str: + return f"{identifier._id}" # noqa: SLF001 + + @classmethod + def _deserialize_cleanup_context(cls, raw: StrBytes) -> AnEmptyTextCleanupContext: + return AnEmptyTextCleanupContext.parse_raw(raw) + + @classmethod + def _serialize_cleanup_context( + cls, cleanup_context: AnEmptyTextCleanupContext + ) -> str: + return cleanup_context.json() + + async def is_used( + self, identifier: UserDefinedID, cleanup_context: AnEmptyTextCleanupContext + ) -> bool: + _ = cleanup_context + return self.component_using_random_text.is_used(identifier) + + # NOTE: it is intended for the user to overwrite the **kwargs with custom names + # to provide a cleaner interface, tooling will complain slightly + async def _create( # pylint:disable=arguments-differ # type:ignore [override] + self, length: int + ) -> tuple[UserDefinedID, RandomTextEntry]: return self.api.create(length) - async def destroy(self, identifier: UserDefinedID) -> None: + async def get(self, identifier: UserDefinedID, **_) -> RandomTextEntry | None: + return self.api.get(identifier) + + async def _destroy( + self, identifier: UserDefinedID, _: AnEmptyTextCleanupContext + ) -> None: self.api.delete(identifier) @pytest.fixture -def manager() -> RandomTextResourcesManager: - return RandomTextResourcesManager() +async def redis_client_sdk( + redis_service: RedisSettings, +) -> AsyncIterator[RedisClientSDK]: + redis_resources_dns = redis_service.build_redis_dsn( + RedisDatabase.DISTRIBUTED_IDENTIFIERS + ) + + client = RedisClientSDK(redis_resources_dns) + assert client + assert client.redis_dsn == redis_resources_dns + await client.setup() + # cleanup, previous run's leftovers + await client.redis.flushall() + + yield client + # cleanup, properly close the clients + await client.redis.flushall() + await client.shutdown() -async def test_resource_is_missing(manager: RandomTextResourcesManager): - missing_identifier = UserDefinedID() - assert await manager.get(missing_identifier) is None +@pytest.fixture +def component_using_random_text() -> ComponentUsingRandomText: + return ComponentUsingRandomText() -async def test_manual_workflow(manager: RandomTextResourcesManager): - # creation - identifier, _ = await manager.create(length=1) - assert await manager.get(identifier) is not None +@pytest.fixture +async def manager_with_no_cleanup_task( + redis_client_sdk: RedisClientSDK, + component_using_random_text: ComponentUsingRandomText, +) -> RandomTextResourcesManager: + return RandomTextResourcesManager(redis_client_sdk, component_using_random_text) - # removal - await manager.destroy(identifier) - # resource no longer exists - assert await manager.get(identifier) is None +@pytest.fixture +async def manager( + manager_with_no_cleanup_task: RandomTextResourcesManager, +) -> AsyncIterable[RandomTextResourcesManager]: + await manager_with_no_cleanup_task.setup() + yield manager_with_no_cleanup_task + await manager_with_no_cleanup_task.shutdown() + + +async def test_resource_is_missing(manager: RandomTextResourcesManager): + missing_identifier = UserDefinedID() + assert await manager.get(missing_identifier) is None @pytest.mark.parametrize("delete_before_removal", [True, False]) -async def test_automatic_cleanup_workflow( +async def test_full_workflow( manager: RandomTextResourcesManager, delete_before_removal: bool ): # creation - identifier, _ = await manager.create(length=1) + identifier, _ = await manager.create( + cleanup_context=AnEmptyTextCleanupContext(), length=1 + ) assert await manager.get(identifier) is not None # optional removal if delete_before_removal: - await manager.destroy(identifier) + await manager.remove(identifier) is_still_present = not delete_before_removal assert (await manager.get(identifier) is not None) is is_still_present @@ -121,10 +245,12 @@ async def test_remove_raises_error( caplog.clear() error_message = "mock error during resource destroy" - mocker.patch.object(manager, "destroy", side_effect=RuntimeError(error_message)) + mocker.patch.object(manager, "_destroy", side_effect=RuntimeError(error_message)) # after creation object is present - identifier, _ = await manager.create(length=1) + identifier, _ = await manager.create( + cleanup_context=AnEmptyTextCleanupContext(), length=1 + ) assert await manager.get(identifier) is not None if reraise: @@ -135,3 +261,98 @@ async def test_remove_raises_error( # check logs in case of error assert "Unhandled exception:" in caplog.text assert error_message in caplog.text + + +async def _create_resources( + manager: RandomTextResourcesManager, count: int +) -> list[UserDefinedID]: + creation_results: list[tuple[UserDefinedID, RandomTextEntry]] = await logged_gather( + *[ + manager.create(cleanup_context=AnEmptyTextCleanupContext(), length=1) + for _ in range(count) + ], + max_concurrency=_MAX_REDIS_CONCURRENCY, + ) + return [x[0] for x in creation_results] + + +async def _assert_all_resources( + manager: RandomTextResourcesManager, + identifiers: list[UserDefinedID], + *, + exist: bool, +) -> None: + get_results: list[RandomTextEntry | None] = await logged_gather( + *[manager.get(identifier) for identifier in identifiers], + max_concurrency=_MAX_REDIS_CONCURRENCY, + ) + if exist: + assert all(x is not None for x in get_results) + else: + assert all(x is None for x in get_results) + + +@pytest.mark.parametrize("count", [1000]) +async def test_parallel_create_remove(manager: RandomTextResourcesManager, count: int): + # create resources + identifiers: list[UserDefinedID] = await _create_resources(manager, count) + await _assert_all_resources(manager, identifiers, exist=True) + + # safe remove the resources, they do not exist any longer + await asyncio.gather(*[manager.remove(identifier) for identifier in identifiers]) + await _assert_all_resources(manager, identifiers, exist=False) + + +async def test_background_removal_of_unused_resources( + manager_with_no_cleanup_task: RandomTextResourcesManager, + component_using_random_text: ComponentUsingRandomText, +): + # create resources + identifiers: list[UserDefinedID] = await _create_resources( + manager_with_no_cleanup_task, 10_000 + ) + await _assert_all_resources(manager_with_no_cleanup_task, identifiers, exist=True) + + # call cleanup, all resources still exist + await manager_with_no_cleanup_task._cleanup_unused_identifiers() # noqa: SLF001 + await _assert_all_resources(manager_with_no_cleanup_task, identifiers, exist=True) + + # make resources unused in external system + component_using_random_text.toggle_usage(in_use=False) + await manager_with_no_cleanup_task._cleanup_unused_identifiers() # noqa: SLF001 + await _assert_all_resources(manager_with_no_cleanup_task, identifiers, exist=False) + + +async def test_no_redis_key_overlap_when_inheriting( + redis_client_sdk: RedisClientSDK, + component_using_random_text: ComponentUsingRandomText, +): + class ChildRandomTextResourcesManager(RandomTextResourcesManager): + ... + + parent_manager = RandomTextResourcesManager( + redis_client_sdk, component_using_random_text + ) + child_manager = ChildRandomTextResourcesManager( + redis_client_sdk, component_using_random_text + ) + + # create an entry in the child and one in the parent + + parent_identifier, _ = await parent_manager.create( + cleanup_context=AnEmptyTextCleanupContext(), length=1 + ) + child_identifier, _ = await child_manager.create( + cleanup_context=AnEmptyTextCleanupContext(), length=1 + ) + assert parent_identifier != child_identifier + + keys = await redis_client_sdk.redis.keys("*") + assert len(keys) == 2 + + # check keys contain the correct prefixes + key_prefixes: set[str] = {k.split(":")[0] for k in keys} + assert key_prefixes == { + RandomTextResourcesManager.class_path(), + ChildRandomTextResourcesManager.class_path(), + } diff --git a/services/docker-compose-ops.yml b/services/docker-compose-ops.yml index f2651e92a63..3dc6450d649 100644 --- a/services/docker-compose-ops.yml +++ b/services/docker-compose-ops.yml @@ -86,7 +86,15 @@ services: image: rediscommander/redis-commander:latest init: true environment: - - REDIS_HOSTS=resources:${REDIS_HOST}:${REDIS_PORT}:0,locks:${REDIS_HOST}:${REDIS_PORT}:1,validation_codes:${REDIS_HOST}:${REDIS_PORT}:2,scheduled_maintenance:${REDIS_HOST}:${REDIS_PORT}:3,user_notifications:${REDIS_HOST}:${REDIS_PORT}:4,announcements:${REDIS_HOST}:${REDIS_PORT}:5 + - >- + REDIS_HOSTS= + resources:${REDIS_HOST}:${REDIS_PORT}:0, + locks:${REDIS_HOST}:${REDIS_PORT}:1, + validation_codes:${REDIS_HOST}:${REDIS_PORT}:2, + scheduled_maintenance:${REDIS_HOST}:${REDIS_PORT}:3, + user_notifications:${REDIS_HOST}:${REDIS_PORT}:4, + announcements:${REDIS_HOST}:${REDIS_PORT}:5, + distributed_identifiers:${REDIS_HOST}:${REDIS_PORT}:6 # If you add/remove a db, do not forget to update the --databases entry in the docker-compose.yml ports: - "18081:8081" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 35f8529b826..644a53215c1 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -956,7 +956,7 @@ services: "--loglevel", "verbose", "--databases", - "6", + "7", "--appendonly", "yes" ] From c12b6c7cb5ba252b876d23ecc88dadd7475b914f Mon Sep 17 00:00:00 2001 From: Julian Querido Date: Mon, 20 Nov 2023 12:51:36 +0100 Subject: [PATCH 20/24] =?UTF-8?q?=F0=9F=90=9B=20Read-only=20user=20cannot?= =?UTF-8?q?=20open=20the=20iframes=20(#5048)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/source/class/osparc/study/StudyOptions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/static-webserver/client/source/class/osparc/study/StudyOptions.js b/services/static-webserver/client/source/class/osparc/study/StudyOptions.js index 55dc441edf8..7b565ccde18 100644 --- a/services/static-webserver/client/source/class/osparc/study/StudyOptions.js +++ b/services/static-webserver/client/source/class/osparc/study/StudyOptions.js @@ -259,7 +259,7 @@ qx.Class.define("osparc.study.StudyOptions", { selectWallet(selection[0].walletId); }); const preferredWallet = store.getPreferredWallet(); - if (this.__projectWalletId) { + if (wallets.find(wallet => wallet.getWalletId() === parseInt(this.__projectWalletId))) { selectWallet(this.__projectWalletId); } else if (preferredWallet) { selectWallet(preferredWallet.getWalletId()); From 3e3a25cb6af395d9dc4e24e12021c3bef42e43ad Mon Sep 17 00:00:00 2001 From: Julian Querido Date: Mon, 20 Nov 2023 18:43:50 +0100 Subject: [PATCH 21/24] Only show total user's credit in the current wallet (#5050) --- .../osparc/navigation/CreditsMenuButton.js | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/navigation/CreditsMenuButton.js b/services/static-webserver/client/source/class/osparc/navigation/CreditsMenuButton.js index 547ba46d4b3..2b5a4b8f744 100644 --- a/services/static-webserver/client/source/class/osparc/navigation/CreditsMenuButton.js +++ b/services/static-webserver/client/source/class/osparc/navigation/CreditsMenuButton.js @@ -84,7 +84,6 @@ qx.Class.define("osparc.navigation.CreditsMenuButton", { if (currentUsage) { currentUsage.addListener("changeUsedCredits", () => { this.__updateCredits(); - // this.__animate(); }); } }, @@ -97,35 +96,18 @@ qx.Class.define("osparc.navigation.CreditsMenuButton", { wallet.addListener("changeCreditsAvailable", () => this.__updateCredits()); } }, - // Note - Sometimes it doesn't work and it bring the credits indicator into a failed state - // __animate: function() { - // const label = this.getChildControl("label"); - // osparc.utils.Utils.animateUsage(label.getContentElement().getDomElement()); - // }, __updateCredits: function() { const store = osparc.store.Store.getInstance(); const wallet = store.getContextWallet(); if (wallet) { let text = "-"; - const currentUsage = this.getCurrentUsage(); - let used = null; - if (currentUsage) { - used = currentUsage.getUsedCredits(); - } const creditsLeft = wallet.getCreditsAvailable(); if (creditsLeft !== null) { text = "CREDITS
"; let nCreditsText = ""; - if (used !== null) { - nCreditsText += osparc.desktop.credits.Utils.creditsToFixed(used) + " / "; - } nCreditsText += osparc.desktop.credits.Utils.creditsToFixed(creditsLeft); text += `${nCreditsText}`; - this.set({ - minWidth: used ? 90 : null, - width: used ? 90 : null - }); } this.set({ label: text, From d410cf723d52879594403536bbed9123fa4530cf Mon Sep 17 00:00:00 2001 From: Andrei Neagu <5694077+GitHK@users.noreply.github.com> Date: Tue, 21 Nov 2023 09:20:13 +0000 Subject: [PATCH 22/24] =?UTF-8?q?=F0=9F=90=9B=20getters=20for=20user=5Frol?= =?UTF-8?q?e=20and=20email=20fixed=20(#5041)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Neagu --- .../simcore_postgres_database/utils_users.py | 21 ++- .../db/repositories/services_environments.py | 16 +- .../modules/db/repositories/users.py | 12 +- .../modules/osparc_variables_substitutions.py | 27 ++-- .../tests/unit/test_utils_osparc_variables.py | 141 +++++++++++++++++- 5 files changed, 176 insertions(+), 41 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_users.py b/packages/postgres-database/src/simcore_postgres_database/utils_users.py index b8621b5099e..c172aa34cbc 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_users.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_users.py @@ -2,6 +2,7 @@ i.e. models.users main table and all its relations """ + import sqlalchemy as sa from aiopg.sa.connection import SAConnection @@ -19,16 +20,22 @@ class UserNotFoundInRepoError(BaseUserRepoError): class UsersRepo: @staticmethod async def get_role(conn: SAConnection, user_id: int) -> UserRole: - if value := await conn.scalar( + value: UserRole | None = await conn.scalar( sa.select(users.c.role).where(users.c.id == user_id) - ): + ) + if value: + assert isinstance(value, UserRole) # nosec return UserRole(value) - raise UserNotFoundInRepoError() + + raise UserNotFoundInRepoError @staticmethod async def get_email(conn: SAConnection, user_id: int) -> str: - if value := await conn.scalar( + value: str | None = await conn.scalar( sa.select(users.c.email).where(users.c.id == user_id) - ): - return f"{value}" - raise UserNotFoundInRepoError() + ) + if value: + assert isinstance(value, str) # nosec + return value + + raise UserNotFoundInRepoError diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/services_environments.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/services_environments.py index cbaf2598e88..a0ba70a85c0 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/services_environments.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/services_environments.py @@ -2,14 +2,11 @@ from models_library.products import ProductName from models_library.services import ServiceKey, ServiceVersion -from models_library.users import UserID -from pydantic import EmailStr, parse_obj_as from simcore_postgres_database.utils_services_environments import ( VENDOR_SECRET_PREFIX, VendorSecret, get_vendor_secrets, ) -from simcore_postgres_database.utils_users import UsersRepo from ._base import BaseRepository @@ -34,23 +31,12 @@ async def get_vendor_secrets( vendor_service_version=service_version, normalize_names=True, ) - - # NOTE: normalize_names = True assert all( # nosec self.is_vendor_secret_identifier(key) for key in vendor_secrets - ) # nosec + ) return vendor_secrets @classmethod def is_vendor_secret_identifier(cls, identifier: str) -> bool: return identifier.startswith(VENDOR_SECRET_PREFIX) - - async def get_user_role(self, user_id: UserID): - async with self.db_engine.acquire() as conn: - return UsersRepo().get_role(conn, user_id=user_id) - - async def get_user_email(self, user_id: UserID) -> EmailStr: - async with self.db_engine.acquire() as conn: - email = UsersRepo().get_email(conn, user_id=user_id) - return parse_obj_as(EmailStr, email) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/users.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/users.py index 0cd198b7347..94f17b90295 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/users.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/users.py @@ -1,11 +1,17 @@ from models_library.users import UserID +from pydantic import EmailStr, parse_obj_as +from simcore_postgres_database.models.users import UserRole from simcore_postgres_database.utils_users import UsersRepo from ._base import BaseRepository class UsersRepository(BaseRepository): - async def get_user_email(self, user_id: UserID) -> str: + async def get_user_email(self, user_id: UserID) -> EmailStr: async with self.db_engine.acquire() as conn: - email: str = await UsersRepo.get_email(conn, user_id) - return email + email = await UsersRepo.get_email(conn, user_id) + return parse_obj_as(EmailStr, email) + + async def get_user_role(self, user_id: UserID) -> UserRole: + async with self.db_engine.acquire() as conn: + return await UsersRepo().get_role(conn, user_id=user_id) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py index e1825d8eea3..8aa4401d712 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/osparc_variables_substitutions.py @@ -17,8 +17,9 @@ from models_library.services import RunID, ServiceKey, ServiceVersion from models_library.users import UserID from models_library.utils.specs_substitution import SpecsSubstitutionsResolver -from pydantic import BaseModel, EmailStr +from pydantic import BaseModel from servicelib.logging_utils import log_context +from simcore_postgres_database.models.users import UserRole from ..utils.db import get_repository from ..utils.osparc_variables import ( @@ -28,6 +29,7 @@ ) from .api_keys_manager import get_or_create_api_key from .db.repositories.services_environments import ServicesEnvironmentsRepository +from .db.repositories.users import UsersRepository _logger = logging.getLogger(__name__) @@ -190,7 +192,7 @@ async def resolve_and_substitute_service_lifetime_variables_in_specs( run_id: RunID, safe: bool = True, ) -> dict[str, Any]: - registry: OsparcVariablesTable = app.state.lifespan_osparc_variables_table + registry: OsparcVariablesTable = app.state.service_lifespan_osparc_variables_table resolver = SpecsSubstitutionsResolver(specs, upgrade=False) @@ -253,25 +255,27 @@ async def _get_or_create_api_secret( return key_data.api_secret # type:ignore [no-any-return] -def _setup_lifespan_osparc_variables_table(app: FastAPI): - app.state.lifespan_osparc_variables_table = table = OsparcVariablesTable() +def _setup_service_lifespan_osparc_variables_table(app: FastAPI): + app.state.service_lifespan_osparc_variables_table = table = OsparcVariablesTable() table.register_from_handler("OSPARC_VARIABLE_API_KEY")(_get_or_create_api_key) table.register_from_handler("OSPARC_VARIABLE_API_SECRET")(_get_or_create_api_secret) _logger.debug( - "Registered lifespan_osparc_variables_table=%s", sorted(table.variables_names()) + "Registered service_lifespan_osparc_variables_table=%s", + sorted(table.variables_names()), ) -async def _request_user_email(app: FastAPI, user_id: UserID) -> EmailStr: - repo = get_repository(app, ServicesEnvironmentsRepository) +async def _request_user_email(app: FastAPI, user_id: UserID) -> str: + repo = get_repository(app, UsersRepository) return await repo.get_user_email(user_id=user_id) -async def _request_user_role(app: FastAPI, user_id: UserID): - repo = get_repository(app, ServicesEnvironmentsRepository) - return await repo.get_user_role(user_id=user_id) +async def _request_user_role(app: FastAPI, user_id: UserID) -> str: + repo = get_repository(app, UsersRepository) + user_role: UserRole = await repo.get_user_role(user_id=user_id) + return f"{user_role.value}" def _setup_session_osparc_variables(app: FastAPI): @@ -283,6 +287,7 @@ def _setup_session_osparc_variables(app: FastAPI): ("OSPARC_VARIABLE_PRODUCT_NAME", "product_name"), ("OSPARC_VARIABLE_STUDY_UUID", "project_id"), ("OSPARC_VARIABLE_NODE_ID", "node_id"), + ("OSPARC_VARIABLE_USER_ID", "user_id"), ]: table.register_from_context(name, context_name) @@ -304,6 +309,6 @@ def setup(app: FastAPI): def on_startup() -> None: _setup_session_osparc_variables(app) - _setup_lifespan_osparc_variables_table(app) + _setup_service_lifespan_osparc_variables_table(app) app.add_event_handler("startup", on_startup) diff --git a/services/director-v2/tests/unit/test_utils_osparc_variables.py b/services/director-v2/tests/unit/test_utils_osparc_variables.py index 17c3b4d9899..e2045294ea2 100644 --- a/services/director-v2/tests/unit/test_utils_osparc_variables.py +++ b/services/director-v2/tests/unit/test_utils_osparc_variables.py @@ -7,16 +7,26 @@ import asyncio import json +from collections.abc import AsyncIterable +from contextlib import asynccontextmanager +from unittest.mock import AsyncMock import pytest +from asgi_lifespan import LifespanManager from faker import Faker from fastapi import FastAPI -from models_library.services import ServiceKey, ServiceVersion +from models_library.api_schemas_webserver.auth import ApiKeyGet +from models_library.services import RunID, ServiceKey, ServiceVersion from models_library.users import UserID from models_library.utils.specs_substitution import SubstitutionValue +from models_library.utils.string_substitution import OSPARC_IDENTIFIER_PREFIX from pydantic import parse_obj_as +from pytest_mock import MockerFixture from pytest_simcore.helpers.faker_compose_specs import generate_fake_docker_compose +from simcore_postgres_database.models.services_environments import VENDOR_SECRET_PREFIX from simcore_postgres_database.models.users import UserRole +from simcore_service_director_v2.api.dependencies.database import RepoType +from simcore_service_director_v2.modules import osparc_variables_substitutions from simcore_service_director_v2.modules.osparc_variables_substitutions import ( resolve_and_substitute_service_lifetime_variables_in_specs, resolve_and_substitute_session_variables_in_specs, @@ -47,10 +57,6 @@ def session_context(faker: Faker) -> ContextDict: @pytest.mark.acceptance_test() async def test_resolve_session_environs(faker: Faker, session_context: ContextDict): - assert resolve_and_substitute_session_variables_in_specs - assert substitute_vendor_secrets_in_specs - assert resolve_and_substitute_service_lifetime_variables_in_specs - async def _request_user_role(app: FastAPI, user_id: UserID) -> SubstitutionValue: print(app, user_id) await asyncio.sleep(1) @@ -103,3 +109,128 @@ async def request_user_email(app: FastAPI, user_id: UserID) -> SubstitutionValue assert environs[osparc_variable_name] == session_context[context_name] print(json.dumps(environs, indent=1)) + + +@pytest.fixture +def mock_repo_db_engine(mocker: MockerFixture) -> None: + @asynccontextmanager + async def _acquire(): + yield + + mocked_engine = AsyncMock() + mocked_engine.acquire = _acquire + + def _get_repository(app: FastAPI, repo_type: type[RepoType]) -> RepoType: + return repo_type(db_engine=mocked_engine) + + mocker.patch( + "simcore_service_director_v2.modules.osparc_variables_substitutions.get_repository", + side_effect=_get_repository, + ) + + +@pytest.fixture +def mock_user_repo(mocker: MockerFixture, mock_repo_db_engine: None) -> None: + base = "simcore_service_director_v2.modules.db.repositories.users" + mocker.patch(f"{base}.UsersRepo.get_role", return_value=UserRole("USER")) + mocker.patch(f"{base}.UsersRepo.get_email", return_value="e@ma.il") + + +@pytest.fixture +async def variables_app() -> AsyncIterable[FastAPI]: + app = FastAPI() + app.state.engine = AsyncMock() + + osparc_variables_substitutions.setup(app) + + async with LifespanManager(app): + yield app + + +async def test_resolve_and_substitute_session_variables_in_specs( + mock_user_repo: None, variables_app: FastAPI, faker: Faker +): + specs = { + "product_name": "${OSPARC_VARIABLE_PRODUCT_NAME}", + "study_uuid": "${OSPARC_VARIABLE_STUDY_UUID}", + "node_id": "${OSPARC_VARIABLE_NODE_ID}", + "user_id": "${OSPARC_VARIABLE_USER_ID}", + "email": "${OSPARC_VARIABLE_USER_EMAIL}", + "user_role": "${OSPARC_VARIABLE_USER_ROLE}", + } + print("SPECS\n", specs) + + replaced_specs = await resolve_and_substitute_session_variables_in_specs( + variables_app, + specs=specs, + user_id=1, + product_name="a_product", + project_id=faker.uuid4(cast_to=None), + node_id=faker.uuid4(cast_to=None), + ) + print("REPLACED SPECS\n", replaced_specs) + + assert OSPARC_IDENTIFIER_PREFIX not in f"{replaced_specs}" + + +@pytest.fixture +def mock_api_key_manager(mocker: MockerFixture) -> None: + mocker.patch( + "simcore_service_director_v2.modules.osparc_variables_substitutions.get_or_create_api_key", + return_value=ApiKeyGet.parse_obj(ApiKeyGet.Config.schema_extra["examples"][0]), + ) + + +async def test_resolve_and_substitute_service_lifetime_variables_in_specs( + mock_api_key_manager: None, variables_app: FastAPI, faker: Faker +): + specs = { + "api_key": "${OSPARC_VARIABLE_API_KEY}", + "api_secret": "${OSPARC_VARIABLE_API_SECRET}", + } + print("SPECS\n", specs) + + replaced_specs = await resolve_and_substitute_service_lifetime_variables_in_specs( + variables_app, + specs=specs, + user_id=1, + product_name="a_product", + node_id=faker.uuid4(cast_to=None), + run_id=RunID.create(), + ) + print("REPLACED SPECS\n", replaced_specs) + + assert OSPARC_IDENTIFIER_PREFIX not in f"{replaced_specs}" + + +@pytest.fixture +def mock_get_vendor_secrets(mocker: MockerFixture, mock_repo_db_engine: None) -> None: + base = "simcore_service_director_v2.modules.db.repositories.services_environments" + mocker.patch( + f"{base}.get_vendor_secrets", + return_value={ + "OSPARC_VARIABLE_VENDOR_SECRET_ONE": 1, + "OSPARC_VARIABLE_VENDOR_SECRET_TWO": "two", + }, + ) + + +async def test_substitute_vendor_secrets_in_specs( + mock_get_vendor_secrets: None, variables_app: FastAPI, faker: Faker +): + specs = { + "api_key": "${OSPARC_VARIABLE_VENDOR_SECRET_ONE}", + "api_secret": "${OSPARC_VARIABLE_VENDOR_SECRET_TWO}", + } + print("SPECS\n", specs) + + replaced_specs = await substitute_vendor_secrets_in_specs( + variables_app, + specs=specs, + product_name="a_product", + service_key=ServiceKey("simcore/services/dynamic/fake"), + service_version=ServiceVersion("0.0.1"), + ) + print("REPLACED SPECS\n", replaced_specs) + + assert VENDOR_SECRET_PREFIX not in f"{replaced_specs}" From 3f42bed5b0a2341c2d5ff6d4a6d965e5885b72f2 Mon Sep 17 00:00:00 2001 From: Dustin Kaiser <8209087+mrnicegyu11@users.noreply.github.com> Date: Tue, 21 Nov 2023 14:03:54 +0100 Subject: [PATCH 23/24] =?UTF-8?q?=E2=9C=A8=20Add=20common=20simcore-servic?= =?UTF-8?q?e=20executable=20alias=20(#5051)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: kaiser --- packages/postgres-database/setup.py | 3 ++- packages/service-integration/setup.py | 4 +++- .../src/service_integration/commands/metadata.py | 2 +- packages/service-integration/tests/conftest.py | 2 +- .../tests/test__usecase_jupytermath.py | 2 +- .../service-integration/tests/test_command_compose.py | 2 +- .../service-integration/tests/test_command_metadata.py | 8 ++++---- .../service-integration/tests/test_command_run_creator.py | 2 +- services/agent/setup.py | 2 +- services/api-server/setup.py | 5 ++--- services/autoscaling/setup.py | 1 + services/catalog/setup.py | 1 + services/clusters-keeper/setup.py | 1 + services/dask-sidecar/setup.py | 1 + services/datcore-adapter/setup.py | 1 + services/director-v2/setup.py | 4 ++-- services/director/setup.py | 1 + services/dynamic-scheduler/setup.py | 1 + services/dynamic-sidecar/setup.py | 1 + services/invitations/setup.py | 1 + services/payments/setup.py | 1 + services/resource-usage-tracker/setup.py | 1 + services/storage/setup.py | 4 ++-- services/web/server/setup.py | 1 + 24 files changed, 33 insertions(+), 19 deletions(-) mode change 100644 => 100755 services/agent/setup.py mode change 100644 => 100755 services/api-server/setup.py mode change 100644 => 100755 services/autoscaling/setup.py diff --git a/packages/postgres-database/setup.py b/packages/postgres-database/setup.py index cfea1b0ec4e..08b8ce1bc7b 100644 --- a/packages/postgres-database/setup.py +++ b/packages/postgres-database/setup.py @@ -62,8 +62,9 @@ def read_reqs(reqs_path: Path) -> set[str]: }, entry_points={ "console_scripts": [ - "simcore-postgres-database=simcore_postgres_database.cli:main", + "simcore-service-postgres-database=simcore_postgres_database.cli:main", "sc-pg=simcore_postgres_database.cli:main", + "simcore-service=simcore_postgres_database.cli:main", ] }, zip_safe=False, diff --git a/packages/service-integration/setup.py b/packages/service-integration/setup.py index ca294ddc323..579bae96f75 100644 --- a/packages/service-integration/setup.py +++ b/packages/service-integration/setup.py @@ -82,8 +82,10 @@ def read_reqs(reqs_path: Path) -> set[str]: zip_safe=False, entry_points={ "console_scripts": [ - "osparc-service-integrator=service_integration.cli:app", "ooil=service_integration.cli:app", + "osparc-service-integrator=service_integration.cli:app", + "simcore-service=service_integration.cli:app", + "simcore-service-integrator=service_integration.cli:app", ], "pytest11": ["simcore_service_integration=service_integration.pytest_plugin"], }, diff --git a/packages/service-integration/src/service_integration/commands/metadata.py b/packages/service-integration/src/service_integration/commands/metadata.py index e7436487d62..3e26c455c57 100644 --- a/packages/service-integration/src/service_integration/commands/metadata.py +++ b/packages/service-integration/src/service_integration/commands/metadata.py @@ -71,5 +71,5 @@ def get_version( # MUST have no new line so that we can produce a VERSION file with no extra new-line # VERSION: $(METADATA) - # @osparc-service-integrator get-version --metadata-file $< > $@ + # @simcore-service-integrator get-version --metadata-file $< > $@ rich.print(current_version, end="") diff --git a/packages/service-integration/tests/conftest.py b/packages/service-integration/tests/conftest.py index 40eb8166894..2c210da61e6 100644 --- a/packages/service-integration/tests/conftest.py +++ b/packages/service-integration/tests/conftest.py @@ -58,7 +58,7 @@ def run_program_with_args() -> Callable: runner = CliRunner() def _invoke(*cmd): - print("RUNNING", "osparc-service-integrator", cmd) + print("RUNNING", "simcore-service-integrator", cmd) print(runner.make_env()) return runner.invoke(cli.app, list(cmd)) diff --git a/packages/service-integration/tests/test__usecase_jupytermath.py b/packages/service-integration/tests/test__usecase_jupytermath.py index c64c06d89d6..874e0e01120 100644 --- a/packages/service-integration/tests/test__usecase_jupytermath.py +++ b/packages/service-integration/tests/test__usecase_jupytermath.py @@ -75,7 +75,7 @@ def run_program_in_repo(tmp_path: Path, jupytermath_repo: Path) -> Iterable[Call def _invoke(*cmd) -> tuple[Path, Result]: print( "RUNNING", - "osparc-service-integrator", + "simcore-service-integrator", " ".join(cmd), f"at {workdir=}", ) diff --git a/packages/service-integration/tests/test_command_compose.py b/packages/service-integration/tests/test_command_compose.py index 0f4979c4685..562ae7c3708 100644 --- a/packages/service-integration/tests/test_command_compose.py +++ b/packages/service-integration/tests/test_command_compose.py @@ -32,7 +32,7 @@ def test_make_docker_compose_meta( """ docker-compose-build.yml: $(metatada) # Injects metadata from $< as labels - osparc-service-integrator compose --metadata $< --to-spec-file $@ + simcore-service-integrator compose --metadata $< --to-spec-file $@ """ result = run_program_with_args( diff --git a/packages/service-integration/tests/test_command_metadata.py b/packages/service-integration/tests/test_command_metadata.py index 7c89a2cce1c..97376a5e633 100644 --- a/packages/service-integration/tests/test_command_metadata.py +++ b/packages/service-integration/tests/test_command_metadata.py @@ -50,7 +50,7 @@ def test_make_version( As Makefile recipe: version-service-patch version-service-minor version-service-major: $(metatada) ## kernel/service versioning as patch - osparc-service-integrator bump-version --metadata-file $< --upgrade $(subst version-service-,,$@) + simcore-service-integrator bump-version --metadata-file $< --upgrade $(subst version-service-,,$@) """ # ensures current_metadata fixture worked as expected assert current_metadata[target_version] == current_version @@ -81,15 +81,15 @@ def test_make_version( "cmd,expected_output", [ ( - "osparc-service-integrator get-version --metadata-file tests/data/metadata.yml", + "simcore-service-integrator get-version --metadata-file tests/data/metadata.yml", "1.1.0", ), ( - "osparc-service-integrator get-version --metadata-file tests/data/metadata.yml integration-version", + "simcore-service-integrator get-version --metadata-file tests/data/metadata.yml integration-version", "1.0.0", ), ( - "osparc-service-integrator get-version --metadata-file tests/data/metadata.yml version", + "simcore-service-integrator get-version --metadata-file tests/data/metadata.yml version", "1.1.0", ), ], diff --git a/packages/service-integration/tests/test_command_run_creator.py b/packages/service-integration/tests/test_command_run_creator.py index 40dece4fd48..cc7445d876b 100644 --- a/packages/service-integration/tests/test_command_run_creator.py +++ b/packages/service-integration/tests/test_command_run_creator.py @@ -20,7 +20,7 @@ def test_make_service_cli_run(run_program_with_args, metadata_file_path: Path): """ service.cli/run: $(metatada) # Updates adapter script from metadata in $< - osparc-service-integrator run-creator --metadata $< --runscript $@ + simcore-service-integrator run-creator --metadata $< --runscript $@ """ run_script_path: Path = metadata_file_path.parent / "run" diff --git a/services/agent/setup.py b/services/agent/setup.py old mode 100644 new mode 100755 index ae8b5787860..69038620294 --- a/services/agent/setup.py +++ b/services/agent/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- import re import sys @@ -59,6 +58,7 @@ def read_reqs(reqs_path: Path) -> set[str]: entry_points={ "console_scripts": [ "simcore-service-agent = simcore_service_agent.cli:main", + "simcore-service = simcore_service_agent.cli:main", ], }, ) diff --git a/services/api-server/setup.py b/services/api-server/setup.py old mode 100644 new mode 100755 index 275a76c5ae1..666a008866c --- a/services/api-server/setup.py +++ b/services/api-server/setup.py @@ -1,14 +1,12 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- import re import sys from pathlib import Path -from typing import Set from setuptools import find_packages, setup -def read_reqs(reqs_path: Path) -> Set[str]: +def read_reqs(reqs_path: Path) -> set[str]: return { r for r in re.findall( @@ -61,6 +59,7 @@ def read_reqs(reqs_path: Path) -> Set[str]: entry_points={ "console_scripts": [ "simcore-service-api-server = simcore_service_api_server.cli:main", + "simcore-service = simcore_service_api_server.cli:main", ], }, ) diff --git a/services/autoscaling/setup.py b/services/autoscaling/setup.py old mode 100644 new mode 100755 index 2c198fc6a3f..3d3d1786928 --- a/services/autoscaling/setup.py +++ b/services/autoscaling/setup.py @@ -62,6 +62,7 @@ def read_reqs(reqs_path: Path) -> set[str]: entry_points={ "console_scripts": [ "simcore-service-autoscaling = simcore_service_autoscaling.cli:main", + "simcore-service = simcore_service_autoscaling.cli:main", ], }, ) diff --git a/services/catalog/setup.py b/services/catalog/setup.py index 01853621f14..2ea0254ba1e 100644 --- a/services/catalog/setup.py +++ b/services/catalog/setup.py @@ -56,6 +56,7 @@ def read_reqs(reqs_path: Path) -> set[str]: entry_points={ "console_scripts": [ "simcore-service-catalog=simcore_service_catalog.cli:main", + "simcore-service=simcore_service_catalog.cli:main", ], }, ) diff --git a/services/clusters-keeper/setup.py b/services/clusters-keeper/setup.py index 8a67e39e66d..ae5b36f9b7d 100755 --- a/services/clusters-keeper/setup.py +++ b/services/clusters-keeper/setup.py @@ -60,6 +60,7 @@ def read_reqs(reqs_path: Path) -> set[str]: entry_points={ "console_scripts": [ "simcore-service-clusters-keeper = simcore_service_clusters_keeper.cli:main", + "simcore-service = simcore_service_clusters_keeper.cli:main", ], }, ) diff --git a/services/dask-sidecar/setup.py b/services/dask-sidecar/setup.py index 32ed6cabab7..864a2de4804 100644 --- a/services/dask-sidecar/setup.py +++ b/services/dask-sidecar/setup.py @@ -48,6 +48,7 @@ def read_reqs(reqs_path: Path) -> set[str]: entry_points={ "console_scripts": [ "simcore-service-dask-sidecar = simcore_service_dask_sidecar.cli:main", + "simcore-service = simcore_service_dask_sidecar.cli:main", ], }, ) diff --git a/services/datcore-adapter/setup.py b/services/datcore-adapter/setup.py index 9e8ae2ffcf1..264fc8e1add 100644 --- a/services/datcore-adapter/setup.py +++ b/services/datcore-adapter/setup.py @@ -49,6 +49,7 @@ def read_reqs(reqs_path: Path) -> set[str]: entry_points={ "console_scripts": [ "simcore-service-datcore-adapter=simcore_service_datcore_adapter.cli:main", + "simcore-service=simcore_service_datcore_adapter.cli:main", ], }, ) diff --git a/services/director-v2/setup.py b/services/director-v2/setup.py index 2f76446213d..92f9e8f1bfc 100644 --- a/services/director-v2/setup.py +++ b/services/director-v2/setup.py @@ -1,12 +1,11 @@ import re import sys from pathlib import Path -from typing import Set from setuptools import find_packages, setup -def read_reqs(reqs_path: Path) -> Set[str]: +def read_reqs(reqs_path: Path) -> set[str]: return { r for r in re.findall( @@ -60,6 +59,7 @@ def read_reqs(reqs_path: Path) -> Set[str]: entry_points={ "console_scripts": [ "simcore-service-director-v2=simcore_service_director_v2.cli:main", + "simcore-service=simcore_service_director_v2.cli:main", ], }, ) diff --git a/services/director/setup.py b/services/director/setup.py index b9fefde11b3..8c12d36f5cb 100644 --- a/services/director/setup.py +++ b/services/director/setup.py @@ -50,6 +50,7 @@ def read_reqs(reqs_path: Path): entry_points={ "console_scripts": [ "simcore-service-director = simcore_service_director.__main__:main", + "simcore-service = simcore_service_director.__main__:main", ], }, ) diff --git a/services/dynamic-scheduler/setup.py b/services/dynamic-scheduler/setup.py index 2ae87cac077..5ca9677bd20 100755 --- a/services/dynamic-scheduler/setup.py +++ b/services/dynamic-scheduler/setup.py @@ -58,6 +58,7 @@ def read_reqs(reqs_path: Path) -> set[str]: "entry_points": { "console_scripts": [ "simcore-service-dynamic-scheduler = simcore_service_dynamic_scheduler.cli:main", + "simcore-service = simcore_service_dynamic_scheduler.cli:main", ], }, } diff --git a/services/dynamic-sidecar/setup.py b/services/dynamic-sidecar/setup.py index c7a8a90b430..6efe636e0ad 100644 --- a/services/dynamic-sidecar/setup.py +++ b/services/dynamic-sidecar/setup.py @@ -56,6 +56,7 @@ def read_reqs(reqs_path: Path) -> set[str]: entry_points={ "console_scripts": [ "simcore-service-dynamic-sidecar=simcore_service_dynamic_sidecar.cli:main", + "simcore-service=simcore_service_dynamic_sidecar.cli:main", ], }, ) diff --git a/services/invitations/setup.py b/services/invitations/setup.py index 08a970dc5fb..4f053c504ac 100755 --- a/services/invitations/setup.py +++ b/services/invitations/setup.py @@ -57,6 +57,7 @@ def read_reqs(reqs_path: Path) -> set[str]: "entry_points": { "console_scripts": [ "simcore-service-invitations = simcore_service_invitations.cli:main", + "simcore-service = simcore_service_invitations.cli:main", ], }, } diff --git a/services/payments/setup.py b/services/payments/setup.py index 38d343110d0..569209d1bf6 100755 --- a/services/payments/setup.py +++ b/services/payments/setup.py @@ -58,6 +58,7 @@ def read_reqs(reqs_path: Path) -> set[str]: "entry_points": { "console_scripts": [ "simcore-service-payments = simcore_service_payments.cli:main", + "simcore-service = simcore_service_payments.cli:main", ], }, } diff --git a/services/resource-usage-tracker/setup.py b/services/resource-usage-tracker/setup.py index 3fa916277c9..ce2abba82a1 100755 --- a/services/resource-usage-tracker/setup.py +++ b/services/resource-usage-tracker/setup.py @@ -58,6 +58,7 @@ def read_reqs(reqs_path: Path) -> set[str]: "entry_points": { "console_scripts": [ "simcore-service-resource-usage-tracker = simcore_service_resource_usage_tracker.cli:app", + "simcore-service = simcore_service_resource_usage_tracker.cli:app", ], }, } diff --git a/services/storage/setup.py b/services/storage/setup.py index 496bd8f92f2..c4d008b678c 100644 --- a/services/storage/setup.py +++ b/services/storage/setup.py @@ -1,12 +1,11 @@ import re import sys from pathlib import Path -from typing import Set from setuptools import find_packages, setup -def read_reqs(reqs_path: Path) -> Set[str]: +def read_reqs(reqs_path: Path) -> set[str]: return { r for r in re.findall( @@ -54,6 +53,7 @@ def read_reqs(reqs_path: Path) -> Set[str]: entry_points={ "console_scripts": [ "simcore-service-storage = simcore_service_storage.cli:main", + "simcore-service = simcore_service_storage.cli:main", ], }, ) diff --git a/services/web/server/setup.py b/services/web/server/setup.py index 00269972393..6ec7537b4c3 100644 --- a/services/web/server/setup.py +++ b/services/web/server/setup.py @@ -57,6 +57,7 @@ def read_reqs(reqs_path: Path) -> set[str]: entry_points={ "console_scripts": [ "simcore-service-webserver=simcore_service_webserver.__main__:main", + "simcore-service=simcore_service_webserver.__main__:main", ] }, python_requires="~=3.10", From 56fe30c39ceb3aff16dbf4522928be98a2728eee Mon Sep 17 00:00:00 2001 From: Mads Bisgaard <126242332+bisgaard-itis@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:39:03 +0100 Subject: [PATCH 24/24] =?UTF-8?q?=F0=9F=8E=A8=20Job=20log=20streaming:=20c?= =?UTF-8?q?onnect=20endpoint=20with=20rabbitMQ=20(#5045)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> --- .../src/pytest_simcore/helpers/typing_env.py | 1 + .../src/pytest_simcore/helpers/utils_envs.py | 9 +- .../src/pytest_simcore/rabbit_service.py | 11 +- .../src/servicelib/fastapi/errors.py | 9 + .../src/servicelib/fastapi/rabbitmq.py | 87 ++++++ .../tests/fastapi/test_openapi.py | 12 +- .../tests/fastapi/test_rabbitmq.py | 147 +++++++++ .../tests/rabbitmq/test_rabbitmq.py | 76 +++-- .../rabbitmq/test_rabbitmq_connection.py | 14 +- services/api-server/requirements/_test.in | 1 + services/api-server/requirements/_test.txt | 38 ++- services/api-server/requirements/prod.txt | 2 +- .../api/dependencies/rabbitmq.py | 86 ++++++ .../api/routes/solvers_jobs_getters.py | 36 +-- .../core/application.py | 4 + .../core/settings.py | 6 +- .../services/director_v2.py | 11 + .../services/rabbitmq.py | 27 ++ services/api-server/tests/conftest.py | 4 + .../tests/unit/api_solvers/conftest.py | 32 +- .../test_api_routers_solvers_jobs_logs.py | 57 +++- services/api-server/tests/unit/conftest.py | 31 +- .../api-server/tests/unit/test__fastapi.py | 4 +- .../tests/unit/test_services_rabbitmq.py | 287 ++++++++++++++++++ .../tests/unit/test_utils_client_base.py | 1 - .../tests/unit/test_modules_rabbitmq.py | 4 +- .../tests/unit/test_utils_rabbitmq.py | 8 +- .../tests/unit/test_modules_rabbitmq.py | 4 +- ...t_modules_comp_scheduler_dask_scheduler.py | 8 +- .../unit/with_dbs/test_utils_rabbitmq.py | 30 +- .../test_services_auto_recharge_listener.py | 8 +- ...pi_resource_tracker_credit_transactions.py | 2 +- .../unit/with_dbs/test_background_task.py | 2 +- .../with_dbs/test_process_rabbitmq_message.py | 4 +- ...rabbitmq_message_triggered_by_listening.py | 4 +- ...age_triggered_by_listening_with_billing.py | 4 +- ...t_process_rabbitmq_message_with_billing.py | 6 +- .../notifications/_rabbitmq_consumers.py | 37 ++- .../notifications/test_rabbitmq_consumers.py | 4 +- 39 files changed, 938 insertions(+), 180 deletions(-) create mode 100644 packages/service-library/src/servicelib/fastapi/errors.py create mode 100644 packages/service-library/src/servicelib/fastapi/rabbitmq.py create mode 100644 packages/service-library/tests/fastapi/test_rabbitmq.py create mode 100644 services/api-server/src/simcore_service_api_server/api/dependencies/rabbitmq.py create mode 100644 services/api-server/src/simcore_service_api_server/services/rabbitmq.py create mode 100644 services/api-server/tests/unit/test_services_rabbitmq.py diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/typing_env.py b/packages/pytest-simcore/src/pytest_simcore/helpers/typing_env.py index 390c0ea9874..142dca2f5a2 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/typing_env.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/typing_env.py @@ -1,6 +1,7 @@ from typing import TypeAlias EnvVarsDict: TypeAlias = dict[str, str] +EnvVarsList: TypeAlias = list[str] # SEE packages/pytest-simcore/tests/test_helpers_utils_envs.py diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py index 8ea387abd90..9b187de2348 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py @@ -10,13 +10,20 @@ import dotenv import pytest -from .typing_env import EnvVarsDict +from .typing_env import EnvVarsDict, EnvVarsList # # monkeypatch using dict # +def delenvs_from_dict(monkeypatch: pytest.MonkeyPatch, envs: EnvVarsList): + for var in envs: + assert isinstance(var, str) + assert var is not None # None keys cannot be is defined w/o value + monkeypatch.delenv(var) + + def setenvs_from_dict( monkeypatch: pytest.MonkeyPatch, envs: EnvVarsDict ) -> EnvVarsDict: diff --git a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py index d190d711ff6..8392fb32995 100644 --- a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py @@ -81,7 +81,7 @@ async def rabbit_service( @pytest.fixture -async def rabbitmq_client( +async def create_rabbitmq_client( rabbit_service: RabbitSettings, ) -> AsyncIterator[Callable[[str], RabbitMQClient]]: created_clients = [] @@ -101,6 +101,7 @@ def _creator(client_name: str, *, heartbeat: int = 60) -> RabbitMQClient: return client yield _creator + # cleanup, properly close the clients await asyncio.gather(*(client.close() for client in created_clients)) @@ -126,3 +127,11 @@ async def _creator(client_name: str, *, heartbeat: int = 60) -> RabbitMQRPCClien yield _creator # cleanup, properly close the clients await asyncio.gather(*(client.close() for client in created_clients)) + + + +async def rabbitmq_client(create_rabbitmq_client): + # NOTE: Legacy fixture + # Use create_rabbitmq_client instead of rabbitmq_client + # SEE docs/coding-conventions.md::CC4 + return create_rabbitmq_client diff --git a/packages/service-library/src/servicelib/fastapi/errors.py b/packages/service-library/src/servicelib/fastapi/errors.py new file mode 100644 index 00000000000..9eebef84637 --- /dev/null +++ b/packages/service-library/src/servicelib/fastapi/errors.py @@ -0,0 +1,9 @@ +from pydantic.errors import PydanticErrorMixin + + +class ApplicationRuntimeError(PydanticErrorMixin, RuntimeError): + pass + + +class ApplicationStateError(ApplicationRuntimeError): + msg_template: str = "Invalid app.state.{state}: {msg}" diff --git a/packages/service-library/src/servicelib/fastapi/rabbitmq.py b/packages/service-library/src/servicelib/fastapi/rabbitmq.py new file mode 100644 index 00000000000..a71634858df --- /dev/null +++ b/packages/service-library/src/servicelib/fastapi/rabbitmq.py @@ -0,0 +1,87 @@ +import logging + +from fastapi import FastAPI +from models_library.rabbitmq_messages import RabbitMessageBase +from settings_library.rabbit import RabbitSettings + +from ..rabbitmq import RabbitMQClient +from ..rabbitmq._utils import wait_till_rabbitmq_responsive +from .errors import ApplicationStateError + +_logger = logging.getLogger(__name__) + + +def _create_client(app: FastAPI): + app.state.rabbitmq_client = RabbitMQClient( + client_name=app.state.rabbitmq_client_name, + settings=app.state.rabbitmq_settings, + ) + + +async def _remove_client(app: FastAPI): + await app.state.rabbitmq_client.close() + app.state.rabbitmq_client = None + + +async def connect(app: FastAPI): + assert app.state.rabbitmq_settings # nosec + await wait_till_rabbitmq_responsive(app.state.rabbitmq_settings.dsn) + _create_client(app) + + +async def disconnect(app: FastAPI): + if app.state.rabbitmq_client: + await _remove_client(app) + + +async def reconnect(app: FastAPI): + await disconnect(app) + await connect(app) + + +def setup_rabbit( + app: FastAPI, + *, + settings: RabbitSettings, + name: str, +) -> None: + """Sets up rabbit in a given app + + - Inits app.states for rabbitmq + - Creates a client to communicate with rabbitmq + + Arguments: + app -- fastapi app + settings -- Rabbit settings or if None, the connection to rabbit is not done upon startup + name -- name for the rmq client name + """ + app.state.rabbitmq_client = None # RabbitMQClient | None + app.state.rabbitmq_client_name = name + app.state.rabbitmq_settings = settings + + if settings is None: + return + + async def on_startup() -> None: + await connect(app) + + app.add_event_handler("startup", on_startup) + + async def on_shutdown() -> None: + await disconnect(app) + + app.add_event_handler("shutdown", on_shutdown) + + +def get_rabbitmq_client(app: FastAPI) -> RabbitMQClient: + if not app.state.rabbitmq_client: + raise ApplicationStateError( + state="rabbitmq_client", + msg="Rabbitmq service unavailable. Check app settings", + ) + assert isinstance(rabbitmq_client := app.state.rabbitmq_client, RabbitMQClient) + return rabbitmq_client + + +async def post_message(app: FastAPI, message: RabbitMessageBase) -> None: + await get_rabbitmq_client(app).publish(message.channel_name, message) diff --git a/packages/service-library/tests/fastapi/test_openapi.py b/packages/service-library/tests/fastapi/test_openapi.py index 661f77b43f3..ab5e6de960b 100644 --- a/packages/service-library/tests/fastapi/test_openapi.py +++ b/packages/service-library/tests/fastapi/test_openapi.py @@ -1,8 +1,14 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + import pytest import starlette.routing from fastapi.applications import FastAPI from fastapi.routing import APIRouter -from openapi_spec_validator import openapi_v31_spec_validator, validate_spec +from openapi_spec_validator import validate_spec from openapi_spec_validator.exceptions import OpenAPISpecValidatorError from servicelib.fastapi.openapi import ( override_fastapi_openapi_method, @@ -31,7 +37,7 @@ def test_exclusive_min_openapi_issue(app: FastAPI): # NOTE: With the latest update of openapi_spec_validator, now passes validation 3.1 but # does not seem resolved. It was moved to https://github.com/tiangolo/fastapi/discussions/9140 with pytest.raises(OpenAPISpecValidatorError): - validate_spec(app.openapi(), validator=openapi_v31_spec_validator) + validate_spec(app.openapi()) def test_overriding_openapi_method(app: FastAPI): @@ -48,7 +54,7 @@ def test_overriding_openapi_method(app: FastAPI): assert openapi assert isinstance(openapi, dict) - validate_spec(openapi, validator=openapi_v31_spec_validator) + validate_spec(openapi) # NOTE: https://github.com/tiangolo/fastapi/issues/240 now passes validation 3.1 but # does not seem resolved. It was moved to https://github.com/tiangolo/fastapi/discussions/9140 diff --git a/packages/service-library/tests/fastapi/test_rabbitmq.py b/packages/service-library/tests/fastapi/test_rabbitmq.py new file mode 100644 index 00000000000..59fd37fb147 --- /dev/null +++ b/packages/service-library/tests/fastapi/test_rabbitmq.py @@ -0,0 +1,147 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + + +import pytest +from faker import Faker +from models_library.rabbitmq_messages import LoggerRabbitMessage, RabbitMessageBase +from tenacity.retry import retry_if_exception_type +from tenacity.stop import stop_after_delay +from tenacity.wait import wait_fixed + +_TENACITY_RETRY_PARAMS = { + "reraise": True, + "retry": retry_if_exception_type(AssertionError), + "stop": stop_after_delay(30), + "wait": wait_fixed(0.1), +} + +# Selection of core and tool services started in this swarm fixture (integration) +pytest_simcore_core_services_selection = [ + "rabbit", +] + +pytest_simcore_ops_services_selection = [] + + +@pytest.fixture +def rabbit_log_message(faker: Faker) -> LoggerRabbitMessage: + return LoggerRabbitMessage( + user_id=faker.pyint(min_value=1), + project_id=faker.uuid4(), + node_id=faker.uuid4(), + messages=faker.pylist(allowed_types=(str,)), + ) + + +@pytest.fixture(params=["rabbit_log_message"]) +def rabbit_message( + request: pytest.FixtureRequest, + rabbit_log_message: LoggerRabbitMessage, +) -> RabbitMessageBase: + return { + "rabbit_log_message": rabbit_log_message, + }[request.param] + + +# # https://github.com/ITISFoundation/osparc-simcore/issues/5059 +# def test_rabbitmq_does_not_initialize_if_deactivated( +# disabled_rabbitmq: None, +# initialized_app: FastAPI, +# ): +# assert hasattr(initialized_app.state, "rabbitmq_client") +# assert initialized_app.state.rabbitmq_client is None +# with pytest.raises(InvalidConfig): +# get_rabbitmq_client(initialized_app) + +# def test_rabbitmq_initializes( +# enabled_rabbitmq: RabbitSettings, +# initialized_app: FastAPI, +# ): +# assert hasattr(initialized_app.state, "rabbitmq_client") +# assert initialized_app.state.rabbitmq_client is not None +# assert ( +# get_rabbitmq_client(initialized_app) +# == initialized_app.state.rabbitmq_client +# ) + +# async def test_post_message( +# enabled_rabbitmq: RabbitSettings, +# initialized_app: FastAPI, +# rabbit_message: RabbitMessageBase, +# create_rabbitmq_client: Callable[[str], RabbitMQClient], +# mocker: MockerFixture, +# ): +# mocked_message_handler = mocker.AsyncMock(return_value=True) +# consumer_rmq = create_rabbitmq_client("pytest_consumer") +# await consumer_rmq.subscribe( +# rabbit_message.channel_name, +# mocked_message_handler, +# topics=[BIND_TO_ALL_TOPICS] if rabbit_message.routing_key() else None, +# ) + +# producer_rmq = get_rabbitmq_client(initialized_app) +# assert producer_rmq is not None +# await producer_rmq.publish(rabbit_message.channel_name, rabbit_message) + +# async for attempt in AsyncRetrying(**_TENACITY_RETRY_PARAMS): +# with attempt: +# print( +# f"--> checking for message in rabbit exchange {rabbit_message.channel_name}, {attempt.retry_state.retry_object.statistics}" +# ) +# mocked_message_handler.assert_called_once_with( +# rabbit_message.json().encode() +# ) +# print("... message received") + +# async def test_post_message_with_disabled_rabbit_does_not_raise( +# disabled_rabbitmq: None, +# disabled_ec2: None, +# mocked_redis_server: None, +# initialized_app: FastAPI, +# rabbit_message: RabbitMessageBase, +# ): +# await post_message(initialized_app, message=rabbit_message) + +# async def _switch_off_rabbit_mq_instance( +# async_docker_client: aiodocker.Docker, +# ) -> None: +# # remove the rabbit MQ instance +# rabbit_services = [ +# s +# for s in await async_docker_client.services.list() +# if "rabbit" in s["Spec"]["Name"] +# ] +# await asyncio.gather( +# *(async_docker_client.services.delete(s["ID"]) for s in rabbit_services) +# ) + +# @retry(**_TENACITY_RETRY_PARAMS) +# async def _check_service_task_gone(service: Mapping[str, Any]) -> None: +# print( +# f"--> checking if service {service['ID']}:{service['Spec']['Name']} is really gone..." +# ) +# list_of_tasks = await async_docker_client.containers.list( +# all=True, +# filters={ +# "label": [f"com.docker.swarm.service.id={service['ID']}"], +# }, +# ) +# assert not list_of_tasks +# print(f"<-- service {service['ID']}:{service['Spec']['Name']} is gone.") + +# await asyncio.gather(*(_check_service_task_gone(s) for s in rabbit_services)) + +# async def test_post_message_when_rabbit_disconnected( +# enabled_rabbitmq: RabbitSettings, +# disabled_ec2: None, +# mocked_redis_server: None, +# initialized_app: FastAPI, +# rabbit_log_message: LoggerRabbitMessage, +# async_docker_client: aiodocker.Docker, +# ): +# await _switch_off_rabbit_mq_instance(async_docker_client) + +# # now posting should not raise out +# await post_message(initialized_app, message=rabbit_log_message) diff --git a/packages/service-library/tests/rabbitmq/test_rabbitmq.py b/packages/service-library/tests/rabbitmq/test_rabbitmq.py index af93023ea5f..b4465cd7569 100644 --- a/packages/service-library/tests/rabbitmq/test_rabbitmq.py +++ b/packages/service-library/tests/rabbitmq/test_rabbitmq.py @@ -111,13 +111,14 @@ async def _assert_message_received( async def test_rabbit_client_pub_sub_message_is_lost_if_no_consumer_present( - rabbitmq_client: Callable[[str], RabbitMQClient], + cleanup_check_rabbitmq_server_has_no_errors: None, + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocked_message_parser: mock.AsyncMock, random_rabbit_message: Callable[..., PytestRabbitMessage], ): - consumer = rabbitmq_client("consumer") - publisher = rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") message = random_rabbit_message() exchange_name = random_exchange_name() @@ -128,13 +129,14 @@ async def test_rabbit_client_pub_sub_message_is_lost_if_no_consumer_present( async def test_rabbit_client_pub_sub( - rabbitmq_client: Callable[[str], RabbitMQClient], + cleanup_check_rabbitmq_server_has_no_errors: None, + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocked_message_parser: mock.AsyncMock, random_rabbit_message: Callable[..., PytestRabbitMessage], ): - consumer = rabbitmq_client("consumer") - publisher = rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") message = random_rabbit_message() exchange_name = random_exchange_name() @@ -145,18 +147,19 @@ async def test_rabbit_client_pub_sub( @pytest.mark.parametrize("num_subs", [10]) async def test_rabbit_client_pub_many_subs( - rabbitmq_client: Callable[[str], RabbitMQClient], + cleanup_check_rabbitmq_server_has_no_errors: None, + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocker: MockerFixture, random_rabbit_message: Callable[..., PytestRabbitMessage], num_subs: int, ): - consumers = (rabbitmq_client(f"consumer_{n}") for n in range(num_subs)) + consumers = (create_rabbitmq_client(f"consumer_{n}") for n in range(num_subs)) mocked_message_parsers = [ mocker.AsyncMock(return_value=True) for _ in range(num_subs) ] - publisher = rabbitmq_client("publisher") + publisher = create_rabbitmq_client("publisher") message = random_rabbit_message() exchange_name = random_exchange_name() await asyncio.gather( @@ -176,13 +179,14 @@ async def test_rabbit_client_pub_many_subs( async def test_rabbit_client_pub_sub_republishes_if_exception_raised( - rabbitmq_client: Callable[[str], RabbitMQClient], + cleanup_check_rabbitmq_server_has_no_errors: None, + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocked_message_parser: mock.AsyncMock, random_rabbit_message: Callable[..., PytestRabbitMessage], ): - publisher = rabbitmq_client("publisher") - consumer = rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") message = random_rabbit_message() @@ -206,18 +210,19 @@ def _raise_once_then_true(*args, **kwargs): @pytest.mark.parametrize("num_subs", [10]) async def test_pub_sub_with_non_exclusive_queue( - rabbitmq_client: Callable[[str], RabbitMQClient], + cleanup_check_rabbitmq_server_has_no_errors: None, + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocker: MockerFixture, random_rabbit_message: Callable[..., PytestRabbitMessage], num_subs: int, ): - consumers = (rabbitmq_client(f"consumer_{n}") for n in range(num_subs)) + consumers = (create_rabbitmq_client(f"consumer_{n}") for n in range(num_subs)) mocked_message_parsers = [ mocker.AsyncMock(return_value=True) for _ in range(num_subs) ] - publisher = rabbitmq_client("publisher") + publisher = create_rabbitmq_client("publisher") message = random_rabbit_message() exchange_name = random_exchange_name() await asyncio.gather( @@ -244,13 +249,13 @@ async def test_pub_sub_with_non_exclusive_queue( def test_rabbit_pub_sub_performance( benchmark, - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocked_message_parser: mock.AsyncMock, random_rabbit_message: Callable[..., PytestRabbitMessage], ): - consumer = rabbitmq_client("consumer") - publisher = rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") message = random_rabbit_message() exchange_name = random_exchange_name() @@ -270,7 +275,8 @@ def run_test_async(): async def test_rabbit_pub_sub_with_topic( - rabbitmq_client: Callable[[str], RabbitMQClient], + cleanup_check_rabbitmq_server_has_no_errors: None, + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocker: MockerFixture, random_rabbit_message: Callable[..., PytestRabbitMessage], @@ -278,21 +284,23 @@ async def test_rabbit_pub_sub_with_topic( exchange_name = f"{random_exchange_name()}_topic" critical_message = random_rabbit_message(topic="pytest.red.critical") debug_message = random_rabbit_message(topic="pytest.orange.debug") - publisher = rabbitmq_client("publisher") + publisher = create_rabbitmq_client("publisher") - all_receiving_consumer = rabbitmq_client("all_receiving_consumer") + all_receiving_consumer = create_rabbitmq_client("all_receiving_consumer") all_receiving_mocked_message_parser = mocker.AsyncMock(return_value=True) await all_receiving_consumer.subscribe( exchange_name, all_receiving_mocked_message_parser, topics=[BIND_TO_ALL_TOPICS] ) - only_critical_consumer = rabbitmq_client("only_critical_consumer") + only_critical_consumer = create_rabbitmq_client("only_critical_consumer") only_critical_mocked_message_parser = mocker.AsyncMock(return_value=True) await only_critical_consumer.subscribe( exchange_name, only_critical_mocked_message_parser, topics=["*.*.critical"] ) - orange_and_critical_consumer = rabbitmq_client("orange_and_critical_consumer") + orange_and_critical_consumer = create_rabbitmq_client( + "orange_and_critical_consumer" + ) orange_and_critical_mocked_message_parser = mocker.AsyncMock(return_value=True) await orange_and_critical_consumer.subscribe( exchange_name, @@ -322,14 +330,15 @@ async def test_rabbit_pub_sub_with_topic( async def test_rabbit_pub_sub_bind_and_unbind_topics( - rabbitmq_client: Callable[[str], RabbitMQClient], + cleanup_check_rabbitmq_server_has_no_errors: None, + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocked_message_parser: mock.AsyncMock, random_rabbit_message: Callable[..., PytestRabbitMessage], ): exchange_name = f"{random_exchange_name()}_topic" - publisher = rabbitmq_client("publisher") - consumer = rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") severities = ["debug", "info", "warning", "critical"] messages = {sev: random_rabbit_message(topic=f"pytest.{sev}") for sev in severities} @@ -393,15 +402,16 @@ async def test_rabbit_pub_sub_bind_and_unbind_topics( @pytest.mark.no_cleanup_check_rabbitmq_server_has_no_errors() async def test_rabbit_adding_topics_to_a_fanout_exchange( - rabbitmq_client: Callable[[str], RabbitMQClient], + cleanup_check_rabbitmq_server_has_no_errors: None, + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocked_message_parser: mock.AsyncMock, random_rabbit_message: Callable[..., PytestRabbitMessage], ): exchange_name = f"{random_exchange_name()}_fanout" message = random_rabbit_message() - publisher = rabbitmq_client("publisher") - consumer = rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") queue_name = await consumer.subscribe(exchange_name, mocked_message_parser) await publisher.publish(exchange_name, message) await _assert_message_received(mocked_message_parser, 1, message) @@ -424,12 +434,12 @@ async def test_rabbit_adding_topics_to_a_fanout_exchange( @pytest.mark.no_cleanup_check_rabbitmq_server_has_no_errors() async def test_rabbit_not_using_the_same_exchange_type_raises( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocked_message_parser: mock.AsyncMock, ): exchange_name = f"{random_exchange_name()}_fanout" - client = rabbitmq_client("consumer") + client = create_rabbitmq_client("consumer") # this will create a FANOUT exchange await client.subscribe(exchange_name, mocked_message_parser) # now do a second subscribtion wiht topics, will create a TOPICS exchange @@ -439,12 +449,12 @@ async def test_rabbit_not_using_the_same_exchange_type_raises( @pytest.mark.no_cleanup_check_rabbitmq_server_has_no_errors() async def test_unsubscribe_consumer( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_exchange_name: Callable[[], str], mocked_message_parser: mock.AsyncMock, ): exchange_name = f"{random_exchange_name()}" - client = rabbitmq_client("consumer") + client = create_rabbitmq_client("consumer") await client.subscribe(exchange_name, mocked_message_parser, exclusive_queue=False) # Unsubsribe just a consumer, the queue will be still there await client.unsubscribe_consumer(exchange_name) diff --git a/packages/service-library/tests/rabbitmq/test_rabbitmq_connection.py b/packages/service-library/tests/rabbitmq/test_rabbitmq_connection.py index 16ed91356b9..e7d6c6dfea8 100644 --- a/packages/service-library/tests/rabbitmq/test_rabbitmq_connection.py +++ b/packages/service-library/tests/rabbitmq/test_rabbitmq_connection.py @@ -58,10 +58,12 @@ async def async_docker_client() -> AsyncIterator[aiodocker.Docker]: async def test_rabbit_client_lose_connection( - rabbitmq_client: Callable[[str], RabbitMQClient], async_docker_client: aiodocker.Docker, + cleanup_check_rabbitmq_server_has_no_errors: None, + create_rabbitmq_client: Callable[[str], RabbitMQClient], + docker_client: docker.client.DockerClient, ): - rabbit_client = rabbitmq_client("pinger") + rabbit_client = create_rabbitmq_client("pinger") assert await rabbit_client.ping() is True async with paused_container(async_docker_client, "rabbit"): # check that connection was lost @@ -102,10 +104,10 @@ def _creator(**kwargs: dict[str, Any]) -> PytestRabbitMessage: async def test_rabbit_client_with_paused_container( random_exchange_name: Callable[[], str], random_rabbit_message: Callable[..., PytestRabbitMessage], - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], async_docker_client: aiodocker.Docker, ): - rabbit_client = rabbitmq_client("pinger") + rabbit_client = create_rabbitmq_client("pinger") assert await rabbit_client.ping() is True exchange_name = random_exchange_name() message = random_rabbit_message() @@ -192,11 +194,11 @@ async def _assert_rabbit_client_state( @pytest.mark.no_cleanup_check_rabbitmq_server_has_no_errors() async def test_rabbit_server_closes_connection( rabbit_service: RabbitSettings, - rabbitmq_client: Callable[[str, int], RabbitMQClient], + create_rabbitmq_client: Callable[[str, int], RabbitMQClient], docker_client: docker.client.DockerClient, ): _assert_rabbitmq_has_connections(rabbit_service, 0) - rabbit_client = rabbitmq_client("tester", heartbeat=2) + rabbit_client = create_rabbitmq_client("tester", heartbeat=2) message = PytestRabbitMessage(message="blahblah", topic="topic") await rabbit_client.publish("test", message) await asyncio.sleep(5) diff --git a/services/api-server/requirements/_test.in b/services/api-server/requirements/_test.in index a2876347204..6462a30af3a 100644 --- a/services/api-server/requirements/_test.in +++ b/services/api-server/requirements/_test.in @@ -29,3 +29,4 @@ pytest-runner respx sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html types-boto3 +async_asgi_testclient # only added to test streaming endpoints (see https://github.com/tiangolo/fastapi/discussions/9126). diff --git a/services/api-server/requirements/_test.txt b/services/api-server/requirements/_test.txt index 1258a3c6102..12ef9891b5e 100644 --- a/services/api-server/requirements/_test.txt +++ b/services/api-server/requirements/_test.txt @@ -9,7 +9,7 @@ aiohttp==3.8.5 # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aioresponses -aioresponses==0.7.4 +aioresponses==0.7.5 # via -r requirements/_test.in aiosignal==1.3.1 # via @@ -25,6 +25,8 @@ anyio==3.7.0 # httpcore asgi-lifespan==2.1.0 # via -r requirements/_test.in +async-asgi-testclient==1.4.11 + # via -r requirements/_test.in async-timeout==4.0.2 # via # -c requirements/_base.txt @@ -43,19 +45,19 @@ aws-sam-translator==1.55.0 # cfn-lint aws-xray-sdk==2.12.1 # via moto -boto3==1.28.76 +boto3==1.29.2 # via # aws-sam-translator # moto -boto3-stubs==1.28.76 +boto3-stubs==1.29.2 # via types-boto3 -botocore==1.31.76 +botocore==1.32.2 # via # aws-xray-sdk # boto3 # moto # s3transfer -botocore-stubs==1.31.76 +botocore-stubs==1.32.2 # via boto3-stubs certifi==2023.7.22 # via @@ -83,7 +85,9 @@ click==8.1.3 # -r requirements/_test.in # flask coverage==7.3.2 - # via pytest-cov + # via + # coverage + # pytest-cov cryptography==41.0.3 # via # -c requirements/../../../requirements/constraints.txt @@ -105,7 +109,7 @@ exceptiongroup==1.1.1 # -c requirements/_base.txt # anyio # pytest -faker==19.13.0 +faker==20.0.3 # via -r requirements/_test.in flask==2.1.3 # via @@ -202,8 +206,9 @@ multidict==6.0.4 # via # -c requirements/_base.txt # aiohttp + # async-asgi-testclient # yarl -mypy==1.6.1 +mypy==1.7.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy @@ -220,7 +225,7 @@ packaging==23.1 # -c requirements/_base.txt # docker # pytest -pbr==5.11.1 +pbr==6.0.0 # via # jschema-to-python # sarif-om @@ -268,7 +273,9 @@ python-dateutil==2.8.2 # faker # moto python-jose==3.3.0 - # via moto + # via + # moto + # python-jose pytz==2023.3.post1 # via moto pyyaml==6.0.1 @@ -281,10 +288,11 @@ pyyaml==6.0.1 # responses requests==2.31.0 # via + # async-asgi-testclient # docker # moto # responses -responses==0.23.3 +responses==0.24.1 # via moto respx==0.20.2 # via -r requirements/_test.in @@ -316,7 +324,7 @@ sqlalchemy==1.4.48 # -c requirements/_base.txt # -r requirements/_test.in # alembic -sqlalchemy2-stubs==0.0.2a36 +sqlalchemy2-stubs==0.0.2a37 # via sqlalchemy sshpubkeys==3.3.1 # via moto @@ -325,12 +333,10 @@ tomli==2.0.1 # coverage # mypy # pytest -types-awscrt==0.19.8 +types-awscrt==0.19.12 # via botocore-stubs types-boto3==1.0.2 # via -r requirements/_test.in -types-pyyaml==6.0.12.12 - # via responses types-s3transfer==0.7.0 # via boto3-stubs typing-extensions==4.6.3 @@ -353,7 +359,7 @@ werkzeug==2.1.2 # via # flask # moto -wrapt==1.15.0 +wrapt==1.16.0 # via aws-xray-sdk xmltodict==0.13.0 # via moto diff --git a/services/api-server/requirements/prod.txt b/services/api-server/requirements/prod.txt index 2ca6529a5ce..960f670e55b 100644 --- a/services/api-server/requirements/prod.txt +++ b/services/api-server/requirements/prod.txt @@ -12,7 +12,7 @@ # installs this repo's packages ../../packages/models-library ../../packages/postgres-database/ -../../packages/service-library[fastapi] +../../packages/service-library ../../packages/settings-library/ ../../packages/simcore-sdk # installs current package diff --git a/services/api-server/src/simcore_service_api_server/api/dependencies/rabbitmq.py b/services/api-server/src/simcore_service_api_server/api/dependencies/rabbitmq.py new file mode 100644 index 00000000000..e1202913cea --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/api/dependencies/rabbitmq.py @@ -0,0 +1,86 @@ +import asyncio +from asyncio.queues import Queue +from typing import Annotated, AsyncIterable, Final, cast +from uuid import UUID + +from fastapi import Depends, FastAPI +from models_library.projects import ProjectID +from models_library.rabbitmq_messages import LoggerRabbitMessage +from pydantic import PositiveInt +from servicelib.fastapi.dependencies import get_app +from servicelib.rabbitmq import RabbitMQClient + +from ...models.schemas.jobs import JobLog +from ...services.director_v2 import DirectorV2Api +from ..dependencies.authentication import get_current_user_id +from ..dependencies.services import get_api_client + +_NEW_LINE: Final[str] = "\n" + + +def get_rabbitmq_client(app: Annotated[FastAPI, Depends(get_app)]) -> RabbitMQClient: + assert app.state.rabbitmq_client # nosec + return cast(RabbitMQClient, app.state.rabbitmq_client) + + +class LogListener: + _queue: Queue[JobLog] + _queue_name: str + _rabbit_consumer: RabbitMQClient + _project_id: ProjectID + _user_id: PositiveInt + _director2_api: DirectorV2Api + + def __init__( + self, + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + rabbit_consumer: Annotated[RabbitMQClient, Depends(get_rabbitmq_client)], + director2_api: Annotated[DirectorV2Api, Depends(get_api_client(DirectorV2Api))], + ): + + self._rabbit_consumer = rabbit_consumer + self._user_id = user_id + self._director2_api = director2_api + self._queue: Queue[JobLog] = Queue(50) + + async def listen( + self, + project_id: UUID, + ): + self._project_id = project_id + + self._queue_name = await self._rabbit_consumer.subscribe( + LoggerRabbitMessage.get_channel_name(), + self._add_logs_to_queue, + exclusive_queue=True, + topics=[f"{self._project_id}.*"], + ) + + async def stop_listening(self): + await self._rabbit_consumer.unsubscribe(self._queue_name) + + async def _add_logs_to_queue(self, data: bytes): + got = LoggerRabbitMessage.parse_raw(data) + item = JobLog( + job_id=got.project_id, + node_id=got.node_id, + log_level=got.log_level, + messages=got.messages, + ) + await self._queue.put(item) + return True + + async def _project_done(self) -> bool: + task = await self._director2_api.get_computation( + self._project_id, self._user_id + ) + return not task.stopped is None + + async def log_generator(self) -> AsyncIterable[str]: + while True: + while self._queue.empty(): + if await self._project_done(): + return + await asyncio.sleep(5) + log: JobLog = await self._queue.get() + yield log.json() + _NEW_LINE diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index b4a86a9a5bf..4a6a00d269b 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -13,21 +13,15 @@ from models_library.api_schemas_webserver.projects import ProjectGet from models_library.api_schemas_webserver.resource_usage import PricingUnitGet from models_library.api_schemas_webserver.wallets import WalletGetWithAvailableCredits -from models_library.projects_nodes_io import BaseFileLink, NodeID, NodeIDStr +from models_library.projects_nodes_io import BaseFileLink from pydantic.types import PositiveInt from servicelib.logging_utils import log_context +from starlette.background import BackgroundTask from ...models.basic_types import VersionStr from ...models.pagination import Page, PaginationParams from ...models.schemas.files import File -from ...models.schemas.jobs import ( - ArgumentTypes, - Job, - JobID, - JobLog, - JobMetadata, - JobOutputs, -) +from ...models.schemas.jobs import ArgumentTypes, Job, JobID, JobMetadata, JobOutputs from ...models.schemas.solvers import SolverKeyId from ...services.catalog import CatalogApi from ...services.director_v2 import DirectorV2Api, DownloadLink, NodeName @@ -38,6 +32,7 @@ from ..dependencies.application import get_reverse_url_mapper from ..dependencies.authentication import get_current_user_id, get_product_name from ..dependencies.database import Engine, get_db_engine +from ..dependencies.rabbitmq import LogListener from ..dependencies.services import get_api_client from ..dependencies.webserver import AuthSession, get_webserver_session from ..errors.http_error import create_error_json_response @@ -372,28 +367,17 @@ async def get_log_stream( solver_key: SolverKeyId, version: VersionStr, job_id: JobID, + log_listener: Annotated[LogListener, Depends(LogListener)], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], ): - async def _fake_generator(node_id: NodeIDStr): - for ii in range(100): - job_log: JobLog = JobLog( - job_id=job_id, - node_id=NodeID(node_id), - log_level=logging.DEBUG, - messages=[f"Hi Manuel. Gruss from the API-server {ii/100}"], - ) - yield job_log.json() + "\n" - job_name = _compose_job_resource_name(solver_key, version, job_id) - - with log_context(_logger, logging.DEBUG, f"Stream logs for {job_name=}"): + with log_context(_logger, logging.DEBUG, "Begin streaming logs"): + _logger.debug("job: %s", job_name) project: ProjectGet = await webserver_api.get_project(project_id=job_id) _raise_if_job_not_associated_with_solver(solver_key, version, project) - - node_ids = list(project.workbench.keys()) - assert len(node_ids) == 1 # nosec - + await log_listener.listen(job_id) return StreamingResponse( - _fake_generator(node_id=node_ids[0]), + log_listener.log_generator(), media_type="application/x-ndjson", + background=BackgroundTask(log_listener.stop_listening), ) diff --git a/services/api-server/src/simcore_service_api_server/core/application.py b/services/api-server/src/simcore_service_api_server/core/application.py index 8160e38d648..3c6e35f6bbb 100644 --- a/services/api-server/src/simcore_service_api_server/core/application.py +++ b/services/api-server/src/simcore_service_api_server/core/application.py @@ -19,6 +19,7 @@ from ..api.root import create_router from ..api.routes.health import router as health_router from ..services import catalog, director_v2, remote_debug, storage, webserver +from ..services.rabbitmq import setup_rabbitmq from .events import create_start_app_handler, create_stop_app_handler from .openapi import override_openapi_method, use_route_names_as_operation_ids from .settings import ApplicationSettings @@ -76,6 +77,9 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI: if settings.SC_BOOT_MODE == BootModeEnum.DEBUG: remote_debug.setup(app) + if settings.API_SERVER_RABBITMQ: + setup_rabbitmq(app) + if settings.API_SERVER_WEBSERVER: webserver.setup(app, settings.API_SERVER_WEBSERVER) diff --git a/services/api-server/src/simcore_service_api_server/core/settings.py b/services/api-server/src/simcore_service_api_server/core/settings.py index 2bf1dc531f3..c90547a6f11 100644 --- a/services/api-server/src/simcore_service_api_server/core/settings.py +++ b/services/api-server/src/simcore_service_api_server/core/settings.py @@ -8,6 +8,7 @@ from settings_library.basic_types import PortInt, VersionTag from settings_library.catalog import CatalogSettings from settings_library.postgres import PostgresSettings +from settings_library.rabbit import RabbitSettings from settings_library.storage import StorageSettings from settings_library.utils_logging import MixinLoggingSettings from settings_library.utils_service import ( @@ -119,9 +120,12 @@ class ApplicationSettings(BasicSettings): # DOCKER BOOT SC_BOOT_MODE: BootModeEnum | None - # POSTGRES API_SERVER_POSTGRES: PostgresSettings | None = Field(auto_default_from_env=True) + API_SERVER_RABBITMQ: RabbitSettings | None = Field( + auto_default_from_env=True, description="settings for service/rabbitmq" + ) + # SERVICES with http API API_SERVER_WEBSERVER: WebServerSettings | None = Field(auto_default_from_env=True) API_SERVER_CATALOG: CatalogSettings | None = Field(auto_default_from_env=True) diff --git a/services/api-server/src/simcore_service_api_server/services/director_v2.py b/services/api-server/src/simcore_service_api_server/services/director_v2.py index 4906aee600b..dbb18f12569 100644 --- a/services/api-server/src/simcore_service_api_server/services/director_v2.py +++ b/services/api-server/src/simcore_service_api_server/services/director_v2.py @@ -1,5 +1,6 @@ import logging from contextlib import contextmanager +from typing import Any, ClassVar from uuid import UUID from fastapi import FastAPI @@ -40,6 +41,16 @@ def guess_progress(self) -> PercentageInt: return PercentageInt(100) return PercentageInt(0) + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + { + **ComputationTask.Config.schema_extra["examples"][0], + "url": "https://link-to-stop-computation", + } + ] + } + class TaskLogFileGet(BaseModel): task_id: NodeID diff --git a/services/api-server/src/simcore_service_api_server/services/rabbitmq.py b/services/api-server/src/simcore_service_api_server/services/rabbitmq.py new file mode 100644 index 00000000000..c39390566bb --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/services/rabbitmq.py @@ -0,0 +1,27 @@ +import logging + +from fastapi import FastAPI +from servicelib.rabbitmq import RabbitMQClient, wait_till_rabbitmq_responsive +from settings_library.rabbit import RabbitSettings + +_logger = logging.getLogger(__name__) + + +def setup_rabbitmq(app: FastAPI) -> None: + settings: RabbitSettings = app.state.settings.API_SERVER_RABBITMQ + app.state.rabbitmq_client = None + app.state.rabbitmq_rpc_server = None + + async def _on_startup() -> None: + await wait_till_rabbitmq_responsive(settings.dsn) + + app.state.rabbitmq_client = RabbitMQClient( + client_name="api_server", settings=settings + ) + + async def _on_shutdown() -> None: + if app.state.rabbitmq_client: + await app.state.rabbitmq_client.close() + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/api-server/tests/conftest.py b/services/api-server/tests/conftest.py index 56f9a8611a1..ab40b2927a2 100644 --- a/services/api-server/tests/conftest.py +++ b/services/api-server/tests/conftest.py @@ -14,12 +14,16 @@ pytest_plugins = [ "pytest_simcore.cli_runner", + "pytest_simcore.docker_compose", + "pytest_simcore.docker_swarm", "pytest_simcore.httpbin_service", "pytest_simcore.pydantic_models", "pytest_simcore.pytest_global_environs", + "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", "pytest_simcore.schemas", "pytest_simcore.services_api_mocks_for_aiohttp_clients", + "pytest_simcore.tmp_path_extra", ] diff --git a/services/api-server/tests/unit/api_solvers/conftest.py b/services/api-server/tests/unit/api_solvers/conftest.py index 5a401dbfe8f..18e8bab0825 100644 --- a/services/api-server/tests/unit/api_solvers/conftest.py +++ b/services/api-server/tests/unit/api_solvers/conftest.py @@ -5,13 +5,18 @@ from collections.abc import Callable from copy import deepcopy -from typing import Any +from datetime import datetime, timedelta +from typing import Any, AsyncIterable, Final +import httpx import pytest -from fastapi import FastAPI +from fastapi import FastAPI, status +from fastapi.encoders import jsonable_encoder +from models_library.projects_state import RunningState from pytest_simcore.helpers import faker_catalog from respx import MockRouter from simcore_service_api_server.core.settings import ApplicationSettings +from simcore_service_api_server.services.director_v2 import ComputationTaskGet @pytest.fixture @@ -82,3 +87,26 @@ def mocked_catalog_service_api( ) return respx_mock + + +@pytest.fixture +async def mocked_directorv2_service( + mocked_directorv2_service_api_base, +) -> AsyncIterable[MockRouter]: + stop_time: Final[datetime] = datetime.now() + timedelta(seconds=5) + + def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response: + task = ComputationTaskGet.parse_obj( + ComputationTaskGet.Config.schema_extra["examples"][0] + ) + if datetime.now() > stop_time: + task.state = RunningState.SUCCESS + task.stopped = datetime.now() + return httpx.Response( + status_code=status.HTTP_200_OK, json=jsonable_encoder(task) + ) + + mocked_directorv2_service_api_base.get( + path__regex=r"/v2/computations/(?P[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" + ).mock(side_effect=_get_computation) + yield mocked_directorv2_service_api_base diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py index e78a39d32e0..ba1179ee8ec 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py @@ -5,24 +5,63 @@ # pylint: disable=unused-variable +import asyncio import logging -from typing import Iterable +from pprint import pprint +from typing import Final, Iterable import httpx import pytest from faker import Faker from fastapi import FastAPI from models_library.api_schemas_webserver.projects import ProjectGet +from models_library.rabbitmq_messages import LoggerRabbitMessage from pytest_mock import MockFixture from pytest_simcore.simcore_webserver_projects_rest_api import GET_PROJECT +from respx import MockRouter +from simcore_service_api_server.api.dependencies.rabbitmq import get_rabbitmq_client from simcore_service_api_server.models.schemas.jobs import JobID, JobLog _logger = logging.getLogger(__name__) +_faker = Faker() + + +@pytest.fixture +async def fake_rabbit_consumer(app: FastAPI): + class FakeRabbitConsumer: + _queue_name: Final[str] = "my_queue" + _n_logs: int = 0 + _total_n_logs: int + _produced_logs: list[str] = [] + + async def subscribe(self, channel_name, callback, exclusive_queue, topics): + async def produce_log(): + for _ in range(5): + txt = _faker.text() + self._produced_logs.append(txt) + msg = LoggerRabbitMessage( + user_id=_faker.pyint(), + project_id=_faker.uuid4(), + node_id=_faker.uuid4(), + messages=[txt], + ) + await callback(msg.json()) + await asyncio.sleep(0.2) + + asyncio.create_task(produce_log()) + return self._queue_name + + async def unsubscribe(self, queue_name): + assert queue_name == self._queue_name + + fake_rabbit_consumer = FakeRabbitConsumer() + app.dependency_overrides[get_rabbitmq_client] = lambda: fake_rabbit_consumer + yield fake_rabbit_consumer @pytest.fixture def fake_project_for_streaming( - mocker: MockFixture, faker: Faker + app: FastAPI, mocker: MockFixture, faker: Faker ) -> Iterable[ProjectGet]: assert isinstance(response_body := GET_PROJECT.response_body, dict) @@ -46,12 +85,14 @@ async def test_log_streaming( client: httpx.AsyncClient, solver_key: str, solver_version: str, + fake_rabbit_consumer, fake_project_for_streaming: ProjectGet, + mocked_directorv2_service: MockRouter, ): job_id: JobID = fake_project_for_streaming.uuid - ii: int = 0 + collected_messages: list[str] = [] async with client.stream( "GET", f"/v0/solvers/{solver_key}/releases/{solver_version}/jobs/{job_id}/logstream", @@ -60,6 +101,10 @@ async def test_log_streaming( response.raise_for_status() async for line in response.aiter_lines(): job_log = JobLog.parse_raw(line) - _logger.debug(job_log.json()) - ii += 1 - assert ii > 0 + pprint(job_log.json()) + collected_messages += job_log.messages + + assert ( + collected_messages + == fake_rabbit_consumer._produced_logs[: len(collected_messages)] + ) diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 3aa9a1d6cf1..c54f28e7a1d 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -19,7 +19,6 @@ from cryptography.fernet import Fernet from faker import Faker from fastapi import FastAPI, status -from httpx._transports.asgi import ASGITransport from models_library.api_schemas_long_running_tasks.tasks import ( TaskGet, TaskProgress, @@ -34,7 +33,6 @@ from moto.server import ThreadedMotoServer from packaging.version import Version from pydantic import HttpUrl, parse_obj_as -from pytest import MonkeyPatch # noqa: PT013 from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.utils_envs import EnvVarsDict, setenvs_from_dict from pytest_simcore.helpers.utils_host import get_localhost_ip @@ -59,7 +57,7 @@ @pytest.fixture def app_environment( - monkeypatch: MonkeyPatch, default_app_env_vars: EnvVarsDict + monkeypatch: pytest.MonkeyPatch, default_app_env_vars: EnvVarsDict ) -> EnvVarsDict: """Config that disables many plugins e.g. database or tracing""" @@ -70,6 +68,7 @@ def app_environment( "WEBSERVER_HOST": "webserver", "WEBSERVER_SESSION_SECRET_KEY": Fernet.generate_key().decode("utf-8"), "API_SERVER_POSTGRES": "null", + "API_SERVER_RABBITMQ": "null", "LOG_LEVEL": "debug", "SC_BOOT_MODE": "production", }, @@ -92,18 +91,14 @@ async def client(app: FastAPI) -> AsyncIterator[httpx.AsyncClient]: # # Prefer this client instead of fastapi.testclient.TestClient # - async with LifespanManager(app): - # needed for app to trigger start/stop event handlers - async with httpx.AsyncClient( - app=app, - base_url="http://api.testserver.io", - headers={"Content-Type": "application/json"}, - ) as client: - assert isinstance(client._transport, ASGITransport) - # rewires location test's app to client.app - client.app = client._transport.app - yield client + # LifespanManager will trigger app's startup&shutown event handlers + async with LifespanManager(app), httpx.AsyncClient( + app=app, + base_url="http://api.testserver.io", + headers={"Content-Type": "application/json"}, + ) as httpx_async_client: + yield httpx_async_client ## MOCKED Repositories -------------------------------------------------- @@ -490,7 +485,8 @@ def _generate_mock( capture_path: Path, side_effects_callbacks: list[SideEffectCallback], ) -> list[respx.MockRouter]: - assert capture_path.is_file() and capture_path.suffix == ".json" + assert capture_path.is_file() + assert capture_path.suffix == ".json" captures: list[HttpApiCallCaptureModel] = parse_obj_as( list[HttpApiCallCaptureModel], json.loads(capture_path.read_text()) ) @@ -509,9 +505,8 @@ def _get_correct_mock_router_for_capture( for router in respx_mock: if capture.host == router._bases["host"].value: return router - raise RuntimeError( - f"Missing respx.MockRouter for capture with {capture.host}" - ) + msg = f"Missing respx.MockRouter for capture with {capture.host}" + raise RuntimeError(msg) class CaptureSideEffect: def __init__( diff --git a/services/api-server/tests/unit/test__fastapi.py b/services/api-server/tests/unit/test__fastapi.py index a375c0c3e58..66303f34492 100644 --- a/services/api-server/tests/unit/test__fastapi.py +++ b/services/api-server/tests/unit/test__fastapi.py @@ -114,7 +114,7 @@ def test_fastapi_route_paths_in_paths(client: TestClient, faker: Faker): } -def test_fastapi_route_name_parsing(client: TestClient, faker: Faker): +def test_fastapi_route_name_parsing(client: TestClient, app: FastAPI, faker: Faker): # # Ensures ':' is allowed in routes # SEE https://github.com/encode/starlette/pull/1657 @@ -125,7 +125,7 @@ def test_fastapi_route_name_parsing(client: TestClient, faker: Faker): # Checks whether parse correctly ":action" suffix for action in ("start", "stop"): - expected_path = client.app.router.url_path_for( + expected_path = app.router.url_path_for( f"{action}_job", solver_key=solver_key, version=version, job_id=job_id ) resp = client.post( diff --git a/services/api-server/tests/unit/test_services_rabbitmq.py b/services/api-server/tests/unit/test_services_rabbitmq.py new file mode 100644 index 00000000000..9f4cb42649c --- /dev/null +++ b/services/api-server/tests/unit/test_services_rabbitmq.py @@ -0,0 +1,287 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +import asyncio +import logging +from collections.abc import AsyncIterable, Callable +from contextlib import asynccontextmanager +from datetime import datetime, timedelta +from typing import Final, Iterable +from unittest.mock import AsyncMock + +import httpx +import pytest +import respx +from faker import Faker +from fastapi import FastAPI, status +from fastapi.encoders import jsonable_encoder +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID +from models_library.projects_state import RunningState +from models_library.rabbitmq_messages import LoggerRabbitMessage +from models_library.users import UserID +from pydantic import parse_obj_as +from pytest_mock import MockerFixture +from pytest_simcore.helpers.utils_envs import ( + EnvVarsDict, + delenvs_from_dict, + setenvs_from_dict, +) +from servicelib.fastapi.rabbitmq import get_rabbitmq_client +from servicelib.rabbitmq import RabbitMQClient +from simcore_service_api_server.api.dependencies.rabbitmq import LogListener +from simcore_service_api_server.models.schemas.jobs import JobLog +from simcore_service_api_server.services.director_v2 import ( + ComputationTaskGet, + DirectorV2Api, +) + +pytest_simcore_core_services_selection = [ + "rabbit", +] +pytest_simcore_ops_services_selection = [] + +_logger = logging.getLogger() +_faker: Faker = Faker() + + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, + rabbit_env_vars_dict: EnvVarsDict, + mocker: MockerFixture, +) -> EnvVarsDict: + # do not init other services + mocker.patch("simcore_service_api_server.core.application.webserver.setup") + mocker.patch("simcore_service_api_server.core.application.catalog.setup") + mocker.patch("simcore_service_api_server.core.application.storage.setup") + + delenvs_from_dict(monkeypatch, ["API_SERVER_RABBITMQ"]) + return setenvs_from_dict( + monkeypatch, + { + **rabbit_env_vars_dict, + "API_SERVER_POSTGRES": "null", + }, + ) + + +@pytest.fixture +def user_id(faker: Faker) -> UserID: + return parse_obj_as(UserID, faker.pyint()) + + +@pytest.fixture +def project_id(faker: Faker) -> ProjectID: + return parse_obj_as(ProjectID, faker.uuid4()) + + +@pytest.fixture +def node_id(faker: Faker) -> NodeID: + return parse_obj_as(NodeID, faker.uuid4()) + + +async def test_subscribe_publish_receive_logs( + client: httpx.AsyncClient, + app: FastAPI, + faker: Faker, + user_id: UserID, + project_id: ProjectID, + node_id: NodeID, + create_rabbitmq_client: Callable[[str], RabbitMQClient], +): + async def _consumer_message_handler(data, **kwargs): + _consumer_message_handler.called = True + _consumer_message_handler.data = data + _ = LoggerRabbitMessage.parse_raw(data) + return True + + # create consumer & subscribe + rabbit_consumer: RabbitMQClient = get_rabbitmq_client(app) + queue_name = await rabbit_consumer.subscribe( + LoggerRabbitMessage.get_channel_name(), + _consumer_message_handler, + exclusive_queue=False, # this instance should receive the incoming messages + topics=[f"{project_id}.*"], + ) + + # log producer + rabbitmq_producer = create_rabbitmq_client("pytest_producer") + log_message = LoggerRabbitMessage( + user_id=user_id, + project_id=project_id, + node_id=node_id, + messages=[faker.text() for _ in range(10)], + ) + _consumer_message_handler.called = False + _consumer_message_handler.data = None + await rabbitmq_producer.publish(log_message.channel_name, log_message) + + # check it received + await asyncio.sleep(1) + + assert _consumer_message_handler.called + data = _consumer_message_handler.data + assert isinstance(data, bytes) + assert LoggerRabbitMessage.parse_raw(data) == log_message + + # unsuscribe + await rabbit_consumer.remove_topics(queue_name, topics=[f"{project_id}.*"]) + + +@asynccontextmanager +async def rabbit_consuming_context( + app: FastAPI, + project_id: ProjectID, +) -> AsyncIterable[AsyncMock]: + consumer_message_handler = AsyncMock(return_value=True) + + rabbit_consumer: RabbitMQClient = get_rabbitmq_client(app) + queue_name = await rabbit_consumer.subscribe( + LoggerRabbitMessage.get_channel_name(), + consumer_message_handler, + exclusive_queue=True, + topics=[f"{project_id}.*"], + ) + + yield consumer_message_handler + + await rabbit_consumer.unsubscribe(queue_name) + + +@pytest.fixture +def produce_logs( + faker: Faker, + create_rabbitmq_client: Callable[[str], RabbitMQClient], + user_id: UserID, +): + async def _go(name, project_id_=None, node_id_=None, messages_=None, level_=None): + rabbitmq_producer = create_rabbitmq_client(f"pytest_producer_{name}") + log_message = LoggerRabbitMessage( + user_id=user_id, + project_id=project_id_ or faker.uuid4(), + node_id=node_id_, + messages=messages_ or [faker.text() for _ in range(10)], + log_level=level_ or logging.INFO, + ) + await rabbitmq_producer.publish(log_message.channel_name, log_message) + + return _go + + +async def test_multiple_producers_and_single_consumer( + client: httpx.AsyncClient, + app: FastAPI, + user_id: UserID, + project_id: ProjectID, + node_id: NodeID, + produce_logs: Callable, +): + await produce_logs("lost", project_id) + + async with rabbit_consuming_context(app, project_id) as consumer_message_handler: + # multiple producers + asyncio.gather( + *[ + produce_logs("expected", project_id, node_id, ["expected message"] * 3), + *(produce_logs(f"{n}") for n in range(5)), + ] + ) + await asyncio.sleep(1) + + # check it received + assert consumer_message_handler.await_count == 1 + (data,) = consumer_message_handler.call_args[0] + assert isinstance(data, bytes) + received_message = LoggerRabbitMessage.parse_raw(data) + + assert received_message.user_id == user_id + assert received_message.project_id == project_id + assert received_message.node_id == node_id + assert received_message.messages == ["expected message"] * 3 + + +# +# -------------------- +# + + +@pytest.fixture +def computation_done() -> Iterable[Callable[[], bool]]: + stop_time: Final[datetime] = datetime.now() + timedelta(seconds=5) + + def _job_done() -> bool: + return datetime.now() >= stop_time + + yield _job_done + + +@pytest.fixture +async def log_listener( + client: httpx.AsyncClient, + app: FastAPI, + project_id: ProjectID, + user_id: UserID, + mocked_directorv2_service_api_base: respx.MockRouter, + computation_done: Callable[[], bool], +) -> AsyncIterable[LogListener]: + def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response: + task = ComputationTaskGet.parse_obj( + ComputationTaskGet.Config.schema_extra["examples"][0] + ) + if computation_done(): + task.state = RunningState.SUCCESS + task.stopped = datetime.now() + return httpx.Response( + status_code=status.HTTP_200_OK, json=jsonable_encoder(task) + ) + + mocked_directorv2_service_api_base.get(f"/v2/computations/{project_id}").mock( + side_effect=_get_computation + ) + + assert isinstance(d2_client := DirectorV2Api.get_instance(app), DirectorV2Api) + log_listener: LogListener = LogListener( + user_id=user_id, + rabbit_consumer=get_rabbitmq_client(app), + director2_api=d2_client, + ) + await log_listener.listen(project_id) + yield log_listener + + +async def test_log_listener( + client: httpx.AsyncClient, + app: FastAPI, + project_id: ProjectID, + node_id: NodeID, + produce_logs: Callable, + log_listener: LogListener, + faker: Faker, + computation_done: Callable[[], bool], +): + published_logs: list[str] = [] + + async def _log_publisher(): + while not computation_done(): + msg: str = faker.text() + await asyncio.sleep(faker.pyfloat(min_value=0.0, max_value=5.0)) + await produce_logs("expected", project_id, node_id, [msg], logging.DEBUG) + published_logs.append(msg) + + publish_task = asyncio.create_task(_log_publisher()) + + collected_messages: list[str] = [] + async for log in log_listener.log_generator(): + job_log: JobLog = JobLog.parse_raw(log) + assert len(job_log.messages) == 1 + assert job_log.job_id == project_id + collected_messages.append(job_log.messages[0]) + + publish_task.cancel() + assert len(published_logs) > 0 + assert published_logs == collected_messages diff --git a/services/api-server/tests/unit/test_utils_client_base.py b/services/api-server/tests/unit/test_utils_client_base.py index 75f527c6c62..b3aa1b5ca18 100644 --- a/services/api-server/tests/unit/test_utils_client_base.py +++ b/services/api-server/tests/unit/test_utils_client_base.py @@ -50,7 +50,6 @@ class TheClientApi(BaseServiceClientApi): # test startup/shutdown async with LifespanManager(app): - # check startup assert TheClientApi.get_instance(app) api_obj = TheClientApi.get_instance(app) diff --git a/services/autoscaling/tests/unit/test_modules_rabbitmq.py b/services/autoscaling/tests/unit/test_modules_rabbitmq.py index f1cdf140174..1ac71ae11ad 100644 --- a/services/autoscaling/tests/unit/test_modules_rabbitmq.py +++ b/services/autoscaling/tests/unit/test_modules_rabbitmq.py @@ -110,11 +110,11 @@ async def test_post_message( mocked_redis_server: None, initialized_app: FastAPI, rabbit_message: RabbitMessageBase, - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], mocker: MockerFixture, ): mocked_message_handler = mocker.AsyncMock(return_value=True) - client = rabbitmq_client("pytest_consumer") + client = create_rabbitmq_client("pytest_consumer") await client.subscribe( rabbit_message.channel_name, mocked_message_handler, diff --git a/services/autoscaling/tests/unit/test_utils_rabbitmq.py b/services/autoscaling/tests/unit/test_utils_rabbitmq.py index 3ec04e7a1db..9391d656be6 100644 --- a/services/autoscaling/tests/unit/test_utils_rabbitmq.py +++ b/services/autoscaling/tests/unit/test_utils_rabbitmq.py @@ -52,7 +52,7 @@ async def test_post_task_log_message( disabled_ec2: None, mocked_redis_server: None, initialized_app: FastAPI, - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], mocker: MockerFixture, async_docker_client: aiodocker.Docker, create_service: Callable[ @@ -63,7 +63,7 @@ async def test_post_task_log_message( faker: Faker, ): mocked_message_handler = mocker.AsyncMock(return_value=True) - client = rabbitmq_client("pytest_consumer") + client = create_rabbitmq_client("pytest_consumer") await client.subscribe( LoggerRabbitMessage.get_channel_name(), mocked_message_handler, @@ -142,7 +142,7 @@ async def test_post_task_progress_message( disabled_ec2: None, mocked_redis_server: None, initialized_app: FastAPI, - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], mocker: MockerFixture, async_docker_client: aiodocker.Docker, create_service: Callable[ @@ -153,7 +153,7 @@ async def test_post_task_progress_message( faker: Faker, ): mocked_message_handler = mocker.AsyncMock(return_value=True) - client = rabbitmq_client("pytest_consumer") + client = create_rabbitmq_client("pytest_consumer") await client.subscribe( ProgressRabbitMessageNode.get_channel_name(), mocked_message_handler, diff --git a/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py b/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py index 2dec9f7d707..f993daec927 100644 --- a/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py +++ b/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py @@ -100,11 +100,11 @@ async def test_post_message( mocked_redis_server: None, initialized_app: FastAPI, rabbit_message: RabbitMessageBase, - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], mocker: MockerFixture, ): mocked_message_handler = mocker.AsyncMock(return_value=True) - client = rabbitmq_client("pytest_consumer") + client = create_rabbitmq_client("pytest_consumer") await client.subscribe( rabbit_message.channel_name, mocked_message_handler, diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py index ac4433ed04a..164a1f5f090 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py @@ -503,9 +503,9 @@ async def _return_tasks_pending(job_ids: list[str]) -> list[DaskClientTaskState] @pytest.fixture async def instrumentation_rabbit_client_parser( - rabbitmq_client: Callable[[str], RabbitMQClient], mocker: MockerFixture + create_rabbitmq_client: Callable[[str], RabbitMQClient], mocker: MockerFixture ) -> AsyncIterator[mock.AsyncMock]: - client = rabbitmq_client("instrumentation_pytest_consumer") + client = create_rabbitmq_client("instrumentation_pytest_consumer") mock = mocker.AsyncMock(return_value=True) queue_name = await client.subscribe( InstrumentationRabbitMessage.get_channel_name(), mock @@ -516,9 +516,9 @@ async def instrumentation_rabbit_client_parser( @pytest.fixture async def resource_tracking_rabbit_client_parser( - rabbitmq_client: Callable[[str], RabbitMQClient], mocker: MockerFixture + create_rabbitmq_client: Callable[[str], RabbitMQClient], mocker: MockerFixture ) -> AsyncIterator[mock.AsyncMock]: - client = rabbitmq_client("resource_tracking_pytest_consumer") + client = create_rabbitmq_client("resource_tracking_pytest_consumer") mock = mocker.AsyncMock(return_value=True) queue_name = await client.subscribe( RabbitResourceTrackingBaseMessage.get_channel_name(), mock diff --git a/services/director-v2/tests/unit/with_dbs/test_utils_rabbitmq.py b/services/director-v2/tests/unit/with_dbs/test_utils_rabbitmq.py index 616619bfa4d..35dbbc91b75 100644 --- a/services/director-v2/tests/unit/with_dbs/test_utils_rabbitmq.py +++ b/services/director-v2/tests/unit/with_dbs/test_utils_rabbitmq.py @@ -110,14 +110,14 @@ def tasks( async def test_publish_service_started_metrics( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], user: dict[str, Any], simcore_user_agent: str, tasks: list[CompTaskAtDB], mocked_message_parser: mock.AsyncMock, ): - consumer = rabbitmq_client("consumer") - publisher = rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") await consumer.subscribe( InstrumentationRabbitMessage.get_channel_name(), mocked_message_parser @@ -134,14 +134,14 @@ async def test_publish_service_started_metrics( async def test_publish_service_stopped_metrics( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], user: dict[str, Any], simcore_user_agent: str, tasks: list[CompTaskAtDB], mocked_message_parser: mock.AsyncMock, ): - consumer = rabbitmq_client("consumer") - publisher = rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") await consumer.subscribe( InstrumentationRabbitMessage.get_channel_name(), mocked_message_parser @@ -159,7 +159,7 @@ async def test_publish_service_stopped_metrics( async def test_publish_service_resource_tracking_started( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], user: dict[str, Any], project: ProjectAtDB, simcore_user_agent: str, @@ -168,8 +168,8 @@ async def test_publish_service_resource_tracking_started( faker: Faker, osparc_product_name: str, ): - consumer = rabbitmq_client("consumer") - publisher = rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") random_task = random.choice(tasks) # noqa: S311 @@ -215,12 +215,12 @@ async def test_publish_service_resource_tracking_started( async def test_publish_service_resource_tracking_stopped( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], mocked_message_parser: mock.AsyncMock, faker: Faker, ): - consumer = rabbitmq_client("consumer") - publisher = rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") await consumer.subscribe( RabbitResourceTrackingBaseMessage.get_channel_name(), mocked_message_parser @@ -249,12 +249,12 @@ async def test_publish_service_resource_tracking_stopped( async def test_publish_service_resource_tracking_heartbeat( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], mocked_message_parser: mock.AsyncMock, faker: Faker, ): - consumer = rabbitmq_client("consumer") - publisher = rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") await consumer.subscribe( RabbitResourceTrackingBaseMessage.get_channel_name(), mocked_message_parser diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 7b3098ddf63..bc54d8297e4 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -102,9 +102,9 @@ async def mocked_message_parser(mocker: MockerFixture) -> mock.AsyncMock: async def test_process_message__called( mocked_message_parser: mock.AsyncMock, app: FastAPI, - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], ): - publisher = rabbitmq_client("publisher") + publisher = create_rabbitmq_client("publisher") msg = WalletCreditsMessage(wallet_id=1, credits=Decimal(80.5), product_name="s4l") await publisher.publish(WalletCreditsMessage.get_channel_name(), msg) @@ -222,7 +222,7 @@ async def _assert_payments_transactions_db_row(postgres_db) -> PaymentsTransacti async def test_process_message__whole_autorecharge_flow_success( app: FastAPI, - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], wallet_id: int, populate_test_db: None, mocked_pay_with_payment_method: mock.AsyncMock, @@ -231,7 +231,7 @@ async def test_process_message__whole_autorecharge_flow_success( mock_resoruce_usage_tracker_service_api: MockRouter, postgres_db: sa.engine.Engine, ): - publisher = rabbitmq_client("publisher") + publisher = create_rabbitmq_client("publisher") msg = WalletCreditsMessage( wallet_id=wallet_id, credits=Decimal(80.5), product_name="s4l" ) diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py index 107f86cd709..1e7098cecda 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py @@ -33,7 +33,7 @@ def resource_tracker_credit_transactions_db( async def test_credit_transactions_workflow( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], mocked_redis_server: None, postgres_db: sa.engine.Engine, async_client: httpx.AsyncClient, diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py index ae9554fa644..0aa020ce241 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py @@ -120,7 +120,7 @@ def resource_tracker_setup_db( async def test_process_event_functions( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], mocked_redis_server: None, postgres_db: sa.engine.Engine, resource_tracker_setup_db, diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py index e0192aa267c..f238862a1a8 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py @@ -26,7 +26,7 @@ async def test_process_event_functions( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_rabbit_message_start, mocked_redis_server: None, postgres_db: sa.engine.Engine, @@ -34,7 +34,7 @@ async def test_process_event_functions( initialized_app, ): engine = initialized_app.state.engine - publisher = rabbitmq_client("publisher") + publisher = create_rabbitmq_client("publisher") msg = random_rabbit_message_start( wallet_id=None, diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening.py index c80652d3ffe..aa5bee5ffd9 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening.py @@ -23,14 +23,14 @@ async def test_process_events_via_rabbit( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_rabbit_message_start, mocked_redis_server: None, postgres_db: sa.engine.Engine, initialized_app, resource_tracker_service_run_db, ): - publisher = rabbitmq_client("publisher") + publisher = create_rabbitmq_client("publisher") msg = random_rabbit_message_start( wallet_id=None, wallet_name=None, diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py index 7c055d5349e..295e471b8c9 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py @@ -147,7 +147,7 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato @pytest.mark.flaky(max_runs=3) async def test_process_events_via_rabbit( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_rabbit_message_start, mocked_redis_server: None, postgres_db: sa.engine.Engine, @@ -155,7 +155,7 @@ async def test_process_events_via_rabbit( resource_tracker_service_run_db, resource_tracker_pricing_tables_db, ): - publisher = rabbitmq_client("publisher") + publisher = create_rabbitmq_client("publisher") msg = random_rabbit_message_start( wallet_id=1, wallet_name="what ever", diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py index c8c71534f07..cf6df1feb83 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py @@ -165,7 +165,7 @@ async def mocked_message_parser(mocker: MockerFixture) -> mock.AsyncMock: async def test_process_event_functions( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], random_rabbit_message_start, mocked_redis_server: None, postgres_db: sa.engine.Engine, @@ -175,8 +175,8 @@ async def test_process_event_functions( mocked_message_parser, ): engine = initialized_app.state.engine - publisher = rabbitmq_client("publisher") - consumer = rabbitmq_client("consumer") + publisher = create_rabbitmq_client("publisher") + consumer = create_rabbitmq_client("consumer") await consumer.subscribe( WalletCreditsLimitReachedMessage.get_channel_name(), mocked_message_parser, diff --git a/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_consumers.py b/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_consumers.py index 69b3e99ab03..b79ee8ceeee 100644 --- a/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_consumers.py +++ b/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_consumers.py @@ -1,7 +1,7 @@ import functools import logging from collections.abc import AsyncIterator, Callable, Coroutine -from typing import Any, Final +from typing import Any, Final, NamedTuple from aiohttp import web from models_library.rabbitmq_messages import ( @@ -186,37 +186,34 @@ async def _osparc_credits_message_parser(app: web.Application, data: bytes) -> b return True -EXCHANGE_TO_PARSER_CONFIG: Final[ - tuple[ - tuple[ - str, - Callable[[web.Application, bytes], Coroutine[Any, Any, bool]], - dict[str, Any], - ], - ..., - ] -] = ( - ( +class SubcribeArgumentsTuple(NamedTuple): + exchange_name: str + parser_fct: Callable[[web.Application, bytes], Coroutine[Any, Any, bool]] + queue_kwargs: dict[str, Any] + + +_EXCHANGE_TO_PARSER_CONFIG: Final[tuple[SubcribeArgumentsTuple, ...,]] = ( + SubcribeArgumentsTuple( LoggerRabbitMessage.get_channel_name(), _log_message_parser, {"topics": []}, ), - ( + SubcribeArgumentsTuple( ProgressRabbitMessageNode.get_channel_name(), _progress_message_parser, {"topics": []}, ), - ( + SubcribeArgumentsTuple( InstrumentationRabbitMessage.get_channel_name(), _instrumentation_message_parser, {"exclusive_queue": False}, ), - ( + SubcribeArgumentsTuple( EventRabbitMessage.get_channel_name(), _events_message_parser, {}, ), - ( + SubcribeArgumentsTuple( WalletCreditsMessage.get_channel_name(), _osparc_credits_message_parser, {"topics": []}, @@ -230,16 +227,18 @@ async def _subscribe_to_rabbitmq(app) -> dict[str, str]: subscribed_queues = await logged_gather( *( rabbit_client.subscribe( - exchange_name, functools.partial(parser_fct, app), **queue_kwargs + p.exchange_name, + functools.partial(p.parser_fct, app), + **p.queue_kwargs, ) - for exchange_name, parser_fct, queue_kwargs in EXCHANGE_TO_PARSER_CONFIG + for p in _EXCHANGE_TO_PARSER_CONFIG ), reraise=False, ) return { exchange_name: queue_name for (exchange_name, *_), queue_name in zip( - EXCHANGE_TO_PARSER_CONFIG, subscribed_queues + _EXCHANGE_TO_PARSER_CONFIG, subscribed_queues, strict=True ) } diff --git a/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py b/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py index d59e45c3b81..d7a82f51138 100644 --- a/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py +++ b/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py @@ -170,9 +170,9 @@ def client( @pytest.fixture async def rabbitmq_publisher( - rabbitmq_client: Callable[[str], RabbitMQClient], + create_rabbitmq_client: Callable[[str], RabbitMQClient], ) -> RabbitMQClient: - return rabbitmq_client("pytest_publisher") + return create_rabbitmq_client("pytest_publisher") @pytest.mark.parametrize("user_role", [UserRole.GUEST], ids=str)