From db23f5eecbc377f8be2f40b2d29bf318b8bd956b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 10 Nov 2023 15:49:20 +0100 Subject: [PATCH 01/78] listen to wallet events in payment system --- .../core/application.py | 4 + .../services/auto_recharge_listener.py | 36 +++++++++ .../services/auto_recharge_process_message.py | 20 +++++ .../test_services_auto_recharge_listener.py | 75 +++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 services/payments/src/simcore_service_payments/services/auto_recharge_listener.py create mode 100644 services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py create mode 100644 services/payments/tests/unit/test_services_auto_recharge_listener.py diff --git a/services/payments/src/simcore_service_payments/core/application.py b/services/payments/src/simcore_service_payments/core/application.py index 899f2735d0f..815355b1cf9 100644 --- a/services/payments/src/simcore_service_payments/core/application.py +++ b/services/payments/src/simcore_service_payments/core/application.py @@ -11,6 +11,7 @@ ) from ..api.rest.routes import setup_rest_api from ..api.rpc.routes import setup_rpc_api_routes +from ..services.auto_recharge_listener import setup_auto_recharge_listener from ..services.payments_gateway import setup_payments_gateway from ..services.postgres import setup_postgres from ..services.rabbitmq import setup_rabbitmq @@ -51,6 +52,9 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: # APIs w/ RUT setup_resource_usage_tracker(app) + # Listening to Rabbitmq + setup_auto_recharge_listener(app) + # ERROR HANDLERS # ... add here ... diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_listener.py b/services/payments/src/simcore_service_payments/services/auto_recharge_listener.py new file mode 100644 index 00000000000..2949c847b02 --- /dev/null +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_listener.py @@ -0,0 +1,36 @@ +import functools +import logging + +from fastapi import FastAPI +from models_library.rabbitmq_messages import WalletCreditsMessage +from servicelib.logging_utils import log_context +from servicelib.rabbitmq import RabbitMQClient + +from .auto_recharge_process_message import process_message +from .rabbitmq import get_rabbitmq_client + +_logger = logging.getLogger(__name__) + + +async def _subscribe_to_rabbitmq(app) -> str: + with log_context(_logger, logging.INFO, msg="Subscribing to rabbitmq channel"): + rabbit_client: RabbitMQClient = get_rabbitmq_client(app) + subscribed_queue: str = await rabbit_client.subscribe( + WalletCreditsMessage.get_channel_name(), + message_handler=functools.partial(process_message, app), + exclusive_queue=False, + topics=["#"], + ) + return subscribed_queue + + +def setup_auto_recharge_listener(app: FastAPI) -> None: + async def _on_startup() -> None: + app.state.auto_recharge_rabbitmq_consumer = await _subscribe_to_rabbitmq(app) + + async def _on_shutdown() -> None: + # NOTE: We want to have persistent queue, therefore we will not unsubscribe + ... + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py new file mode 100644 index 00000000000..8480063b19b --- /dev/null +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -0,0 +1,20 @@ +import logging + +from fastapi import FastAPI +from models_library.rabbitmq_messages import WalletCreditsMessage +from pydantic import parse_raw_as + +_logger = logging.getLogger(__name__) + + +async def process_message(app: FastAPI, data: bytes) -> bool: + assert app # nosec + rabbit_message = parse_raw_as(WalletCreditsMessage, data) + _logger.debug("Process msg: %s", rabbit_message) + + # 1. Check if auto-recharge functionality is ON for wallet_id + # 2. Check if wallet credits are bellow the threshold + # 3. Get Payment method + # 4. Pay with payment method + + return True diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py new file mode 100644 index 00000000000..34e82ec1993 --- /dev/null +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -0,0 +1,75 @@ +from collections.abc import Callable +from decimal import Decimal +from unittest import mock + +import pytest +from fastapi import FastAPI +from models_library.rabbitmq_messages import WalletCreditsMessage +from pytest_mock.plugin import MockerFixture +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from servicelib.rabbitmq import RabbitMQClient +from tenacity._asyncio import AsyncRetrying +from tenacity.retry import retry_if_exception_type +from tenacity.stop import stop_after_delay +from tenacity.wait import wait_fixed + +pytest_simcore_core_services_selection = [ + "postgres", + "rabbit", +] +pytest_simcore_ops_services_selection = [ + "adminer", +] + + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, + rabbit_env_vars_dict: EnvVarsDict, # rabbitMQ settings from 'rabbit' service + postgres_env_vars_dict: EnvVarsDict, + wait_for_postgres_ready_and_db_migrated: None, + external_environment: EnvVarsDict, +): + # set environs + monkeypatch.delenv("PAYMENTS_RABBITMQ", raising=False) + monkeypatch.delenv("PAYMENTS_POSTGRES", raising=False) + + return setenvs_from_dict( + monkeypatch, + { + **app_environment, + **rabbit_env_vars_dict, + **postgres_env_vars_dict, + **external_environment, + "POSTGRES_CLIENT_NAME": "payments-service-pg-client", + }, + ) + + +@pytest.fixture +async def mocked_message_parser(mocker: MockerFixture) -> mock.AsyncMock: + # return mocker.AsyncMock(return_value=True) + return mocker.patch( + "simcore_service_payments.services.auto_recharge_listener.process_message" + ) + + +async def test_process_event_functions( + mocked_message_parser: mock.AsyncMock, + app: FastAPI, + rabbitmq_client: Callable[[str], RabbitMQClient], +): + publisher = rabbitmq_client("publisher") + msg = WalletCreditsMessage(wallet_id=1, credits=Decimal(120.5)) + await publisher.publish(WalletCreditsMessage.get_channel_name(), msg) + + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), + stop=stop_after_delay(5), + retry=retry_if_exception_type(AssertionError), + reraise=True, + ): + with attempt: + mocked_message_parser.assert_called_once() From bd38e85063ef8b7f01d9f987c80e99cfc43f224b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 10 Nov 2023 16:03:19 +0100 Subject: [PATCH 02/78] fix test --- .../tests/unit/test_services_auto_recharge_listener.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 34e82ec1993..78ff4097d02 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -1,3 +1,9 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + from collections.abc import Callable from decimal import Decimal from unittest import mock From caefaf185f0472b6c20f94fa168d60ec0649ddb5 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 10 Nov 2023 17:03:53 +0100 Subject: [PATCH 03/78] fix tests --- services/payments/tests/unit/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index f177e747db8..b46bc3a4673 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -57,6 +57,9 @@ def _do(): # The following services are affected if rabbitmq is not in place mocker.patch("simcore_service_payments.core.application.setup_rabbitmq") mocker.patch("simcore_service_payments.core.application.setup_rpc_api_routes") + mocker.patch( + "simcore_service_payments.core.application.setup_auto_recharge_listener" + ) return _do From 1967ea48b65c9f0bc8f0c05036935f55d5a6c48e Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Sun, 12 Nov 2023 11:03:00 +0100 Subject: [PATCH 04/78] review @sanderegg --- .../src/servicelib/rabbitmq/_client.py | 14 +++++++++++++- .../tests/rabbitmq/test_rabbitmq.py | 17 +++++++++++++++++ .../services/auto_recharge_listener.py | 13 +++++++++++-- .../test_services_auto_recharge_listener.py | 1 - 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/_client.py b/packages/service-library/src/servicelib/rabbitmq/_client.py index ae668f26737..542bd2ba2ee 100644 --- a/packages/service-library/src/servicelib/rabbitmq/_client.py +++ b/packages/service-library/src/servicelib/rabbitmq/_client.py @@ -63,6 +63,9 @@ async def _get_channel(self) -> aio_pika.abc.AbstractChannel: channel.close_callbacks.add(self._channel_close_callback) return channel + async def _get_consumer_tag(self, exchange_name) -> str: + return f"{get_rabbitmq_client_unique_name(self.client_name)}_{exchange_name}" + async def subscribe( self, exchange_name: str, @@ -139,10 +142,11 @@ async def _on_message( ) await message.nack() + _consumer_tag = await self._get_consumer_tag(exchange_name) await queue.consume( _on_message, exclusive=exclusive_queue, - consumer_tag=f"{get_rabbitmq_client_unique_name(self.client_name)}_{exchange_name}", + consumer_tag=_consumer_tag, ) output: str = queue.name return output @@ -214,3 +218,11 @@ async def publish(self, exchange_name: str, message: RabbitMessage) -> None: aio_pika.Message(message.body()), routing_key=message.routing_key() or "", ) + + async def unsubscribe_consumer(self, exchange_name: str): + assert self._channel_pool # nosec + async with self._channel_pool.acquire() as channel: + queue_name = exchange_name + queue = await channel.get_queue(queue_name) + _consumer_tag = await self._get_consumer_tag(exchange_name) + await queue.cancel(_consumer_tag) diff --git a/packages/service-library/tests/rabbitmq/test_rabbitmq.py b/packages/service-library/tests/rabbitmq/test_rabbitmq.py index bf95e267d7c..af93023ea5f 100644 --- a/packages/service-library/tests/rabbitmq/test_rabbitmq.py +++ b/packages/service-library/tests/rabbitmq/test_rabbitmq.py @@ -435,3 +435,20 @@ async def test_rabbit_not_using_the_same_exchange_type_raises( # now do a second subscribtion wiht topics, will create a TOPICS exchange with pytest.raises(aio_pika.exceptions.ChannelPreconditionFailed): await client.subscribe(exchange_name, mocked_message_parser, topics=[]) + + +@pytest.mark.no_cleanup_check_rabbitmq_server_has_no_errors() +async def test_unsubscribe_consumer( + rabbitmq_client: Callable[[str], RabbitMQClient], + random_exchange_name: Callable[[], str], + mocked_message_parser: mock.AsyncMock, +): + exchange_name = f"{random_exchange_name()}" + client = rabbitmq_client("consumer") + await client.subscribe(exchange_name, mocked_message_parser, exclusive_queue=False) + # Unsubsribe just a consumer, the queue will be still there + await client.unsubscribe_consumer(exchange_name) + # Unsubsribe the queue + await client.unsubscribe(exchange_name) + with pytest.raises(aio_pika.exceptions.ChannelNotFoundEntity): + await client.unsubscribe(exchange_name) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_listener.py b/services/payments/src/simcore_service_payments/services/auto_recharge_listener.py index 2949c847b02..a71038cafb5 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_listener.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_listener.py @@ -24,13 +24,22 @@ async def _subscribe_to_rabbitmq(app) -> str: return subscribed_queue +async def _unsubscribe_consumer(app) -> None: + with log_context(_logger, logging.INFO, msg="Unsubscribing from rabbitmq queue"): + rabbit_client: RabbitMQClient = get_rabbitmq_client(app) + await rabbit_client.unsubscribe_consumer( + WalletCreditsMessage.get_channel_name(), + ) + return None + + def setup_auto_recharge_listener(app: FastAPI) -> None: async def _on_startup() -> None: app.state.auto_recharge_rabbitmq_consumer = await _subscribe_to_rabbitmq(app) async def _on_shutdown() -> None: - # NOTE: We want to have persistent queue, therefore we will not unsubscribe - ... + # NOTE: We want to have persistent queue, therefore we will unsubscribe only consumer + app.state.auto_recharge_rabbitmq_constumer = await _unsubscribe_consumer(app) app.add_event_handler("startup", _on_startup) app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 78ff4097d02..9557dba382d 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -56,7 +56,6 @@ def app_environment( @pytest.fixture async def mocked_message_parser(mocker: MockerFixture) -> mock.AsyncMock: - # return mocker.AsyncMock(return_value=True) return mocker.patch( "simcore_service_payments.services.auto_recharge_listener.process_message" ) From cb4a3da0fcb1526b068f1247d091f6d675ebca7e Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 8 Nov 2023 21:53:53 +0100 Subject: [PATCH 05/78] draft --- .../src/servicelib/fastapi/http_client.py | 35 ++++++++ .../services/healthchecks.py | 83 +++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 services/payments/src/simcore_service_payments/services/healthchecks.py diff --git a/packages/service-library/src/servicelib/fastapi/http_client.py b/packages/service-library/src/servicelib/fastapi/http_client.py index 2eb87400eee..c8ddd0a71d8 100644 --- a/packages/service-library/src/servicelib/fastapi/http_client.py +++ b/packages/service-library/src/servicelib/fastapi/http_client.py @@ -1,11 +1,39 @@ import contextlib import logging +from dataclasses import dataclass +from datetime import timedelta +from typing import TypeAlias import httpx from fastapi import FastAPI _logger = logging.getLogger(__name__) +# +# Healthchecks: liveness probes and readiness probes +# +# SEE https://medium.com/polarsquad/how-should-i-answer-a-health-check-aa1fcf6e858e +# SEE https://docs.docker.com/engine/reference/builder/#healthcheck + + +@dataclass +class ServiceIsAlive: + elapsed: timedelta + + def __bool__(self) -> bool: + return True + + +@dataclass +class ServiceIsDead: + reason: str + + def __bool__(self) -> bool: + return False + + +LivenessResult: TypeAlias = ServiceIsAlive | ServiceIsDead + class BaseHttpApi: def __init__(self, client: httpx.AsyncClient): @@ -51,6 +79,13 @@ async def is_healhy(self) -> bool: except httpx.HTTPError: return False + async def check_liveness(self) -> LivenessResult: + try: + response = await self.client.get("/") + return ServiceIsAlive(elapsed=response.elapsed) + except httpx.RequestError as err: + return ServiceIsDead(reason=f"{err}") + class AppStateMixin: """ diff --git a/services/payments/src/simcore_service_payments/services/healthchecks.py b/services/payments/src/simcore_service_payments/services/healthchecks.py new file mode 100644 index 00000000000..97cab5d2486 --- /dev/null +++ b/services/payments/src/simcore_service_payments/services/healthchecks.py @@ -0,0 +1,83 @@ +""" + +## Types of health checks + +Based on the service types, we can categorize health checks based on the actions they take. + +- Reboot: When the target is unhealthy, the target should be restarted to recover to a working state. +Container and VM orchestration platforms typically perform reboots. +- Cut traffic: When the target is unhealthy, no traffic should be sent to the target. Service discovery +services and load balancers typically cut traffic from targets in one way or another. + +The difference between these is that rebooting attempts to actively repair the target, while cutting +traffic leaves room for the target to repair itself. + +In Kubernetes health-checks are called *probes*: +- The health check for reboots is called a *liveness probe*: "Check if the container is alive". +- The health check for cutting traffic is called a *readiness probe*: "Check if the container is ready to receive traffic". + +Taken from https://medium.com/polarsquad/how-should-i-answer-a-health-check-aa1fcf6e858e + + +## docker healthchecks: + + --interval=DURATION (default: 30s) + --timeout=DURATION (default: 30s) + --start-period=DURATION (default: 0s) + --retries=N (default: 3) + + The health check will first run *interval* seconds after the container is started, and + then again *interval* seconds after each previous check completes. + + If a single run of the check takes longer than *timeout* seconds then the check is considered to have failed (SEE HealthCheckFailed). + + It takes *retries* consecutive failures of the health check for the container to be considered **unhealthy**. + + *start period* provides initialization time for containers that need time to bootstrap. Probe failure during + that period will not be counted towards the maximum number of retries. + + However, if a health check succeeds during the *start period*, the container is considered started and all consecutive + failures will be counted towards the maximum number of retries. + +Taken from https://docs.docker.com/engine/reference/builder/#healthcheck +""" + +# can run some diagnostic tests to determine readiness and livelihood +# e.g. Can we do payments? +# +# - is the gateway ready? +# - no. log why? alert! +# - is the RUT reachable? +# - no. log why? alert! +# - is the database reachable? +# - no. log why? alert! +# + +import logging + +from sqlalchemy.ext.asyncio import AsyncEngine + +from .payments_gateway import PaymentsGatewayApi +from .resource_usage_tracker import ResourceUsageTrackerApi + +_logger = logging.getLogger(__name__) + + +async def check_payments_gateway_liveness( + payments_gateway_api: PaymentsGatewayApi, +) -> bool: + alive = await payments_gateway_api.check_liveness() + _logger.info( + "liveness check of '%s': %s", payments_gateway_api.client.base_url, alive + ) + return bool(alive) + + +async def check_resource_usage_tracker_liveness(rut: ResourceUsageTrackerApi) -> bool: + alive = await rut.check_liveness() + _logger.info("liveness check of '%s': %s", rut.client.base_url, alive) + return bool(alive) + + +async def check_postgres_liveness(engine: AsyncEngine) -> bool: + raise NotImplementedError From 1f979f14a9d0b6d5c4c259a9770b81f0832beaf0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Wed, 8 Nov 2023 22:20:18 +0100 Subject: [PATCH 06/78] fixes app fixture --- services/payments/tests/unit/conftest.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index b46bc3a4673..0184824c18e 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -126,15 +126,22 @@ def payments_clean_db(postgres_db: sa.engine.Engine) -> Iterator[None]: con.execute(payments_transactions.delete()) +# +MAX_TIME_FOR_APP_TO_STARTUP = 10 +MAX_TIME_FOR_APP_TO_SHUTDOWN = 10 + + @pytest.fixture -async def app(app_environment: EnvVarsDict) -> AsyncIterator[FastAPI]: - test_app = create_app() +async def app( + app_environment: EnvVarsDict, is_pdb_enabled: bool +) -> AsyncIterator[FastAPI]: + the_test_app = create_app() async with LifespanManager( - test_app, - startup_timeout=None, # for debugging - shutdown_timeout=10, + the_test_app, + startup_timeout=None if is_pdb_enabled else MAX_TIME_FOR_APP_TO_STARTUP, + shutdown_timeout=None if is_pdb_enabled else MAX_TIME_FOR_APP_TO_SHUTDOWN, ): - yield test_app + yield the_test_app # @@ -332,7 +339,6 @@ def mock_payments_gateway_service_or_none( mock_payments_methods_routes: Callable, external_environment: EnvVarsDict, ) -> MockRouter | None: - # EITHER tests against external payments-gateway if payments_gateway_url := external_environment.get("PAYMENTS_GATEWAY_URL"): print("🚨 EXTERNAL: these tests are running against", f"{payments_gateway_url=}") From 99071c9e48042689232160526512c7807a7893a0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 14:46:29 +0100 Subject: [PATCH 07/78] ignores openapi --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ec92564a31c..9836532c4c9 100644 --- a/.gitignore +++ b/.gitignore @@ -178,5 +178,6 @@ Untitled* # service settings.schemas.json services/**/settings-schema.json +services/payments/scripts/openapi.json tests/public-api/osparc_python_wheels/* From 2075983f7124ae2537a61597190f1cafa4a7e46a Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 15:03:44 +0100 Subject: [PATCH 08/78] deprecated and new --- .../api/rpc/_payments_methods.py | 38 +++++++++++++++++++ .../services/payments.py | 34 ++++++++++++++++- .../services/payments_gateway.py | 10 ++++- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py index 5b02867ef57..81f485a417a 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py @@ -1,4 +1,5 @@ import logging +import warnings from decimal import Decimal from fastapi import FastAPI @@ -127,6 +128,11 @@ async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=t user_email: EmailStr, comment: str | None = None, ) -> WalletPaymentInitiated: + warnings.warn( + f"{__name__}.init_payment_with_payment_method is deprecated. Use instead pay_with_payment_method", + DeprecationWarning, + stacklevel=1, + ) return await payments.init_payment_with_payment_method( gateway=PaymentsGatewayApi.get_from_app_state(app), repo_transactions=PaymentsTransactionsRepo(db_engine=app.state.engine), @@ -142,3 +148,35 @@ async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=t user_email=user_email, comment=comment, ) + + +@router.expose() +async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-arguments + app: FastAPI, + *, + payment_method_id: PaymentMethodID, + amount_dollars: Decimal, + target_credits: Decimal, + product_name: str, + wallet_id: WalletID, + wallet_name: str, + user_id: UserID, + user_name: str, + user_email: EmailStr, + comment: str | None = None, +): + return await payments.pay_with_payment_method( + gateway=PaymentsGatewayApi.get_from_app_state(app), + repo_transactions=PaymentsTransactionsRepo(db_engine=app.state.engine), + repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine), + payment_method_id=payment_method_id, + amount_dollars=amount_dollars, + target_credits=target_credits, + product_name=product_name, + wallet_id=wallet_id, + wallet_name=wallet_name, + user_id=user_id, + user_name=user_name, + user_email=user_email, + comment=comment, + ) diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index dcb0dceac17..48e757bacdf 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -6,6 +6,7 @@ # pylint: disable=too-many-arguments import logging +import warnings from decimal import Decimal import arrow @@ -91,7 +92,6 @@ async def cancel_one_time_payment( user_id: UserID, wallet_id: WalletID, ) -> None: - payment = await repo.get_payment_transaction( payment_id=payment_id, user_id=user_id, wallet_id=wallet_id ) @@ -123,7 +123,6 @@ async def acknowledge_one_time_payment( payment_id: PaymentID, ack: AckPayment, ) -> PaymentsTransactionsDB: - return await repo_transactions.update_ack_payment_transaction( payment_id=payment_id, completion_state=( @@ -189,6 +188,12 @@ async def init_payment_with_payment_method( user_email: EmailStr, comment: str | None = None, ) -> WalletPaymentInitiated: + warnings.warn( + f"{__name__}.init_payment_with_payment_method is deprecated. Use instead pay_with_payment_method", + DeprecationWarning, + stacklevel=1, + ) + initiated_at = arrow.utcnow().datetime acked = await repo_methods.get_payment_method( @@ -224,6 +229,31 @@ async def init_payment_with_payment_method( ) +async def pay_with_payment_method( + gateway: PaymentsGatewayApi, + repo_transactions: PaymentsTransactionsRepo, + repo_methods: PaymentsMethodsRepo, + *, + payment_method_id: PaymentMethodID, + amount_dollars: Decimal, + target_credits: Decimal, + product_name: str, + wallet_id: WalletID, + wallet_name: str, + user_id: UserID, + user_name: str, + user_email: EmailStr, + comment: str | None = None, +): + initiated_at = arrow.utcnow().datetime + + acked = await repo_methods.get_payment_method( + payment_method_id, user_id=user_id, wallet_id=wallet_id + ) + + raise NotImplementedError + + async def get_payments_page( repo: PaymentsTransactionsRepo, *, diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index 90d60dfe25d..446c3cf38d8 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -5,8 +5,8 @@ """ - import logging +import warnings import httpx from fastapi import FastAPI @@ -111,6 +111,11 @@ async def delete_payment_method(self, id_: PaymentMethodID) -> None: async def init_payment_with_payment_method( self, id_: PaymentMethodID, payment: InitPayment ) -> PaymentInitiated: + warnings.warn( + f"{__name__}.init_payment_with_payment_method is deprecated. Use instead pay_with_payment_method", + DeprecationWarning, + stacklevel=1, + ) response = await self.client.post( f"/payment-methods/{id_}:pay", json=jsonable_encoder(payment), @@ -118,6 +123,9 @@ async def init_payment_with_payment_method( response.raise_for_status() return PaymentInitiated.parse_obj(response.json()) + async def pay_with_payment_method(self): + raise NotImplementedError + def setup_payments_gateway(app: FastAPI): assert app.state # nosec From 908db562a4f75c06bbf98b590eba642d96fa1783 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 15:04:08 +0100 Subject: [PATCH 09/78] updates OAS --- services/payments/scripts/Makefile | 2 +- services/payments/scripts/fake_payment_gateway.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/services/payments/scripts/Makefile b/services/payments/scripts/Makefile index 78a4a19b81f..166c0763f9d 100644 --- a/services/payments/scripts/Makefile +++ b/services/payments/scripts/Makefile @@ -9,6 +9,6 @@ run-devel: ## runs fake_payment_gateway server uvicorn fake_payment_gateway:the_app --reload -openapi.json: +openapi.json: ## creates OAS @set -o allexport; source .env-secret; set +o allexport; \ python fake_payment_gateway.py openapi > $@ diff --git a/services/payments/scripts/fake_payment_gateway.py b/services/payments/scripts/fake_payment_gateway.py index 883ed6d8845..4401d7fc1e6 100644 --- a/services/payments/scripts/fake_payment_gateway.py +++ b/services/payments/scripts/fake_payment_gateway.py @@ -39,7 +39,10 @@ PaymentMethodInitiated, PaymentMethodsBatch, ) -from simcore_service_payments.models.schemas.acknowledgements import AckPayment +from simcore_service_payments.models.schemas.acknowledgements import ( + AckPayment, + AckPaymentWithPaymentMethod, +) from simcore_service_payments.models.schemas.auth import Token logging.basicConfig(level=logging.INFO) @@ -291,7 +294,7 @@ def batch_get_payment_methods( @router.get( "/{id}", - response_class=GetPaymentMethod, + response_model=GetPaymentMethod, responses=ERROR_RESPONSES, ) def get_payment_method( @@ -315,10 +318,10 @@ def delete_payment_method( @router.post( "/{id}:pay", - response_model=PaymentInitiated, + response_model=AckPaymentWithPaymentMethod, responses=ERROR_RESPONSES, ) - def init_payment_with_payment_method( + def pay_with_payment_method( id: PaymentMethodID, payment: InitPayment, auth: Annotated[int, Depends(auth_session)], From c8611f442367e13c7320763ec59506c699d521c6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 15:11:23 +0100 Subject: [PATCH 10/78] updates ack for payment method --- .../models/schemas/acknowledgements.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py b/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py index ac08098a2cd..c99a6379ed8 100644 --- a/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py +++ b/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py @@ -58,15 +58,21 @@ class SavedPaymentMethod(AckPaymentMethod): # -class AckPayment(_BaseAck): +class AckPaymentWithPaymentMethod(_BaseAck): + # NOTE: This model is equivalent to `AckPayment` below + # Nonetheless I decided to separate it for clarity in the OAS + # since in payments w/ payment-method the field `saved` will + # never be provided. invoice_url: HttpUrl | None = Field( default=None, description="Link to invoice is required when success=true" ) + + +class AckPayment(AckPaymentWithPaymentMethod): saved: SavedPaymentMethod | None = Field( default=None, - description="If the user decided to save the payment method" - "after payment it returns the payment-method acknoledgement response." - "Otherwise it defaults to None.", + description="Gets the payment-method if user opted to save it during payment." + "If used did not opt to save of payment-method was already saved, then it defaults to None", ) class Config: From 726777ad36963719c0f30926d416376187e07aa1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 15:38:04 +0100 Subject: [PATCH 11/78] cleanup --- .../models/schemas/acknowledgements.py | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py b/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py index c99a6379ed8..13a6494d9c5 100644 --- a/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py +++ b/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py @@ -22,6 +22,20 @@ class SavedPaymentMethod(AckPaymentMethod): payment_method_id: PaymentMethodID +class AckPaymentWithPaymentMethod(_BaseAck): + # NOTE: This model is equivalent to `AckPayment` below + # Nonetheless I decided to separate it for clarity in the OAS + # since in payments w/ payment-method the field `saved` will + # never be provided. + invoice_url: HttpUrl | None = Field( + default=None, description="Link to invoice is required when success=true" + ) + + +# +# ACK one-time payments +# + _ONE_TIME_SUCCESS: dict[str, Any] = { "success": True, "invoice_url": "https://invoices.com/id=12345", @@ -53,22 +67,11 @@ class SavedPaymentMethod(AckPaymentMethod): ] -# -# ACK one-time payments -# - - -class AckPaymentWithPaymentMethod(_BaseAck): - # NOTE: This model is equivalent to `AckPayment` below - # Nonetheless I decided to separate it for clarity in the OAS - # since in payments w/ payment-method the field `saved` will - # never be provided. +class AckPayment(_BaseAck): invoice_url: HttpUrl | None = Field( default=None, description="Link to invoice is required when success=true" ) - -class AckPayment(AckPaymentWithPaymentMethod): saved: SavedPaymentMethod | None = Field( default=None, description="Gets the payment-method if user opted to save it during payment." From e85ca64fe29089cd38fe67660ffcbb21bfd2d51a Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 15:48:05 +0100 Subject: [PATCH 12/78] gateway --- .../services/payments_gateway.py | 14 ++++++++++++-- services/payments/tests/unit/conftest.py | 18 ++++++++++++++---- .../unit/test_services_payments_gateway.py | 12 ++---------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index 446c3cf38d8..bc011f9309c 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -14,6 +14,9 @@ from httpx import URL from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID from servicelib.fastapi.http_client import AppStateMixin, BaseHttpApi +from simcore_service_payments.models.schemas.acknowledgements import ( + AckPaymentWithPaymentMethod, +) from ..core.settings import ApplicationSettings from ..models.payments_gateway import ( @@ -123,8 +126,15 @@ async def init_payment_with_payment_method( response.raise_for_status() return PaymentInitiated.parse_obj(response.json()) - async def pay_with_payment_method(self): - raise NotImplementedError + async def pay_with_payment_method( + self, id_: PaymentMethodID, payment: InitPayment + ) -> AckPaymentWithPaymentMethod: + response = await self.client.post( + f"/payment-methods/{id_}:pay", + json=jsonable_encoder(payment), + ) + response.raise_for_status() + return AckPaymentWithPaymentMethod.parse_obj(response.json()) def setup_payments_gateway(app: FastAPI): diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index 0184824c18e..ce38f392e72 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -37,6 +37,9 @@ PaymentMethodInitiated, PaymentMethodsBatch, ) +from simcore_service_payments.models.schemas.acknowledgements import ( + AckPaymentWithPaymentMethod, +) @pytest.fixture @@ -254,16 +257,23 @@ def _batch_get(request: httpx.Request): json=jsonable_encoder(PaymentMethodsBatch(items=items)), ) - def _init_payment(request: httpx.Request, pm_id: PaymentMethodID): + def _pay(request: httpx.Request, pm_id: PaymentMethodID): assert "*" not in request.headers["X-Init-Api-Secret"] assert InitPayment.parse_raw(request.content) is not None # checks _get(request, pm_id) + payment_id = faker.uuid4() return httpx.Response( status.HTTP_200_OK, - json=jsonable_encoder(PaymentInitiated(payment_id=faker.uuid4())), + json=jsonable_encoder( + AckPaymentWithPaymentMethod( + success=True, + message=f"Payment '{payment_id}' with payment-method '{pm_id}'", + invoice_url=faker.url(), + ) + ), ) # ------ @@ -290,8 +300,8 @@ def _init_payment(request: httpx.Request, pm_id: PaymentMethodID): mock_router.post( path__regex=r"/payment-methods/(?P[\w-]+):pay$", - name="init_payment_with_payment_method", - ).mock(side_effect=_init_payment) + name="pay_with_payment_method", + ).mock(side_effect=_pay) yield _mock diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index f29008257f9..d88606d87ce 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -92,7 +92,6 @@ async def test_one_time_payment_workflow( mock_payments_gateway_service_or_none: MockRouter | None, amount_dollars: float, ): - payment_gateway_api = PaymentsGatewayApi.get_from_app_state(app) assert payment_gateway_api @@ -131,7 +130,6 @@ async def test_payment_methods_workflow( mock_payments_gateway_service_or_none: MockRouter | None, amount_dollars: float, ): - payments_gateway_api: PaymentsGatewayApi = PaymentsGatewayApi.get_from_app_state( app ) @@ -171,8 +169,7 @@ async def test_payment_methods_workflow( assert len(items) == 1 assert items[0] == got_payment_method - # init payments with payment-method (needs to be ACK to complete) - payment_initiated = await payments_gateway_api.init_payment_with_payment_method( + payment_with_payment_method = await payments_gateway_api.pay_with_payment_method( id_=payment_method_id, payment=InitPayment( amount_dollars=amount_dollars, @@ -182,10 +179,7 @@ async def test_payment_methods_workflow( wallet_name=faker.word(), ), ) - - # cancel payment - payment_canceled = await payments_gateway_api.cancel_payment(payment_initiated) - assert payment_canceled is not None + assert payment_with_payment_method.success # delete payment-method await payments_gateway_api.delete_payment_method(payment_method_id) @@ -197,8 +191,6 @@ async def test_payment_methods_workflow( assert http_status_error.response.status_code == status.HTTP_404_NOT_FOUND if mock_payments_gateway_service_or_none: - assert mock_payments_gateway_service_or_none.routes["cancel_payment"].called - # all defined payment-methods for route in mock_payments_gateway_service_or_none.routes: if route.name and "payment_method" in route.name: From 0435700828dc74a69db33f2ed5fd5dc0d8f7c695 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 16:18:06 +0100 Subject: [PATCH 13/78] rpc pay with payment-method --- .../db/payments_transactions_repo.py | 15 +++++++ .../src/simcore_service_payments/models/db.py | 28 +++++++++++- .../services/payments.py | 45 +++++++++++++++++-- .../tests/unit/test_rpc_payments_methods.py | 18 ++++---- 4 files changed, 93 insertions(+), 13 deletions(-) diff --git a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py index 0b7e972be76..b52ccfa4497 100644 --- a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py @@ -22,6 +22,21 @@ class PaymentsTransactionsRepo(BaseRepository): + async def insert_payment_transaction( + self, + payment_id: PaymentID, + *, + price_dollars: Decimal, + osparc_credits: Decimal, + product_name: str, + user_id: UserID, + user_email: str, + wallet_id: WalletID, + comment: str | None, + initiated_at: datetime.datetime, + ): + raise NotImplementedError + async def insert_init_payment_transaction( self, payment_id: PaymentID, diff --git a/services/payments/src/simcore_service_payments/models/db.py b/services/payments/src/simcore_service_payments/models/db.py index be7f858bfe2..85fbea0c160 100644 --- a/services/payments/src/simcore_service_payments/models/db.py +++ b/services/payments/src/simcore_service_payments/models/db.py @@ -2,7 +2,11 @@ from decimal import Decimal from typing import Any, ClassVar -from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID +from models_library.api_schemas_webserver.wallets import ( + PaymentID, + PaymentMethodID, + PaymentTransaction, +) from models_library.emails import LowerCaseEmailStr from models_library.products import ProductName from models_library.users import UserID @@ -59,6 +63,28 @@ class Config: ] } + def to_api_model(self) -> PaymentTransaction: + data: dict[str, Any] = { + "payment_id": self.payment_id, + "price_dollars": self.price_dollars, + "osparc_credits": self.osparc_credits, + "wallet_id": self.wallet_id, + "created_at": self.initiated_at, + "state": self.state, + "completed_at": self.completed_at, + } + + if self.comment: + data["comment"] = self.comment + + if self.state_message: + data["state_message"] = self.state_message + + if self.invoice_url: + data["invoice_url"] = self.invoice_url + + return PaymentTransaction.parse_obj(data) + _EXAMPLE_AFTER_INIT_PAYMENT_METHOD = { "payment_method_id": "12345", diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index 48e757bacdf..cc994e9d55d 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -6,6 +6,7 @@ # pylint: disable=too-many-arguments import logging +import uuid import warnings from decimal import Decimal @@ -13,6 +14,7 @@ from models_library.api_schemas_webserver.wallets import ( PaymentID, PaymentMethodID, + PaymentTransaction, WalletPaymentInitiated, ) from models_library.users import UserID @@ -29,7 +31,7 @@ from ..db.payments_transactions_repo import PaymentsTransactionsRepo from ..models.db import PaymentsTransactionsDB from ..models.payments_gateway import InitPayment, PaymentInitiated -from ..models.schemas.acknowledgements import AckPayment +from ..models.schemas.acknowledgements import AckPayment, AckPaymentWithPaymentMethod from ..services.resource_usage_tracker import ResourceUsageTrackerApi from .payments_gateway import PaymentsGatewayApi @@ -244,14 +246,51 @@ async def pay_with_payment_method( user_name: str, user_email: EmailStr, comment: str | None = None, -): +) -> PaymentTransaction: initiated_at = arrow.utcnow().datetime + # TODO: check with Dennis whether ack_payment.payment_id + init_payment_id = f"{uuid.uuid4()}" acked = await repo_methods.get_payment_method( payment_method_id, user_id=user_id, wallet_id=wallet_id ) - raise NotImplementedError + ack: AckPaymentWithPaymentMethod = await gateway.pay_with_payment_method( + acked.payment_method_id, + payment=InitPayment( + amount_dollars=amount_dollars, + credits=target_credits, + user_name=user_name, + user_email=user_email, + wallet_name=wallet_name, + ), + ) + + # TODO: insert_payment_transaction + payment_id = await repo_transactions.insert_init_payment_transaction( + payment_id=init_payment_id, + price_dollars=amount_dollars, + osparc_credits=target_credits, + product_name=product_name, + user_id=user_id, + user_email=user_email, + wallet_id=wallet_id, + comment=comment, + initiated_at=initiated_at, + ) + + transaction = await repo_transactions.update_ack_payment_transaction( + payment_id=payment_id, + completion_state=( + PaymentTransactionState.SUCCESS + if ack.success + else PaymentTransactionState.FAILED + ), + state_message=ack.message, + invoice_url=ack.invoice_url, + ) + + return transaction.to_api_model() async def get_payments_page( diff --git a/services/payments/tests/unit/test_rpc_payments_methods.py b/services/payments/tests/unit/test_rpc_payments_methods.py index 15ce2f05724..7cce4ae6f95 100644 --- a/services/payments/tests/unit/test_rpc_payments_methods.py +++ b/services/payments/tests/unit/test_rpc_payments_methods.py @@ -10,7 +10,7 @@ from fastapi import FastAPI from models_library.api_schemas_webserver.wallets import ( PaymentMethodInitiated, - WalletPaymentInitiated, + PaymentTransaction, ) from models_library.basic_types import IDStr from models_library.products import ProductName @@ -222,9 +222,9 @@ async def test_webserver_pay_with_payment_method_workflow( ack=AckPaymentMethod(success=True, message="Faked ACK"), ) - payment_inited = await rpc_client.request( + transaction = await rpc_client.request( PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_payment_with_payment_method"), + parse_obj_as(RPCMethodName, "pay_with_payment_method"), payment_method_id=created.payment_method_id, amount_dollars=faker.pyint(), target_credits=faker.pyint(), @@ -237,14 +237,14 @@ async def test_webserver_pay_with_payment_method_workflow( comment="Payment with stored credit-card", ) - assert isinstance(payment_inited, WalletPaymentInitiated) - assert payment_inited.payment_id - assert payment_inited.payment_form_url is None + assert isinstance(transaction, PaymentTransaction) + assert transaction.payment_id + assert transaction.state == "SUCCESS" payment = await PaymentsTransactionsRepo(app.state.engine).get_payment_transaction( - payment_inited.payment_id, user_id=user_id, wallet_id=wallet_id + transaction.payment_id, user_id=user_id, wallet_id=wallet_id ) assert payment is not None - assert payment.payment_id == payment_inited.payment_id - assert payment.state == PaymentTransactionState.PENDING + assert payment.payment_id == transaction.payment_id + assert payment.state == PaymentTransactionState.SUCCESS assert payment.comment == "Payment with stored credit-card" From 32669a9197c4e92f67195e2b1a22635c6a47cfe1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 18:27:14 +0100 Subject: [PATCH 14/78] minor --- services/payments/openapi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/payments/openapi.json b/services/payments/openapi.json index 692374075ef..aeb658b7350 100644 --- a/services/payments/openapi.json +++ b/services/payments/openapi.json @@ -235,7 +235,7 @@ } ], "title": "Saved", - "description": "If the user decided to save the payment methodafter payment it returns the payment-method acknoledgement response.Otherwise it defaults to None." + "description": "Gets the payment-method if user opted to save it during payment.If used did not opt to save of payment-method was already saved, then it defaults to None" } }, "type": "object", From 5451b6b90ae4df9e07f35c78f57b7424925dd732 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 19:01:16 +0100 Subject: [PATCH 15/78] init and update --- .../db/payments_transactions_repo.py | 15 ---------- .../services/payments.py | 29 +++++++++---------- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py index b52ccfa4497..0b7e972be76 100644 --- a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py @@ -22,21 +22,6 @@ class PaymentsTransactionsRepo(BaseRepository): - async def insert_payment_transaction( - self, - payment_id: PaymentID, - *, - price_dollars: Decimal, - osparc_credits: Decimal, - product_name: str, - user_id: UserID, - user_email: str, - wallet_id: WalletID, - comment: str | None, - initiated_at: datetime.datetime, - ): - raise NotImplementedError - async def insert_init_payment_transaction( self, payment_id: PaymentID, diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index cc994e9d55d..ff588648d5d 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -248,27 +248,15 @@ async def pay_with_payment_method( comment: str | None = None, ) -> PaymentTransaction: initiated_at = arrow.utcnow().datetime - # TODO: check with Dennis whether ack_payment.payment_id - init_payment_id = f"{uuid.uuid4()}" acked = await repo_methods.get_payment_method( payment_method_id, user_id=user_id, wallet_id=wallet_id ) - ack: AckPaymentWithPaymentMethod = await gateway.pay_with_payment_method( - acked.payment_method_id, - payment=InitPayment( - amount_dollars=amount_dollars, - credits=target_credits, - user_name=user_name, - user_email=user_email, - wallet_name=wallet_name, - ), - ) - - # TODO: insert_payment_transaction + # TODO: retry if PaymentAlreadyExistsError + # TODO: async with repo_transactions.begin(): should set a scope transaction inside payment_id = await repo_transactions.insert_init_payment_transaction( - payment_id=init_payment_id, + payment_id=f"{uuid.uuid4()}", # TODO: check with Dennis whether ack_payment.payment_id price_dollars=amount_dollars, osparc_credits=target_credits, product_name=product_name, @@ -279,6 +267,17 @@ async def pay_with_payment_method( initiated_at=initiated_at, ) + ack: AckPaymentWithPaymentMethod = await gateway.pay_with_payment_method( + acked.payment_method_id, + payment=InitPayment( + amount_dollars=amount_dollars, + credits=target_credits, + user_name=user_name, + user_email=user_email, + wallet_name=wallet_name, + ), + ) + transaction = await repo_transactions.update_ack_payment_transaction( payment_id=payment_id, completion_state=( From 91292b296b41a313ad7ab2aff505b56037e7d84f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 19:21:26 +0100 Subject: [PATCH 16/78] health --- .../src/models_library/healthchecks.py | 28 ++++++++++++++++ .../src/servicelib/fastapi/http_client.py | 33 ++----------------- .../tests/fastapi/test_http_client.py | 6 ++++ 3 files changed, 37 insertions(+), 30 deletions(-) create mode 100644 packages/models-library/src/models_library/healthchecks.py diff --git a/packages/models-library/src/models_library/healthchecks.py b/packages/models-library/src/models_library/healthchecks.py new file mode 100644 index 00000000000..11e6f88162a --- /dev/null +++ b/packages/models-library/src/models_library/healthchecks.py @@ -0,0 +1,28 @@ +# +# Healthchecks: liveness probes and readiness probes +# +# SEE https://medium.com/polarsquad/how-should-i-answer-a-health-check-aa1fcf6e858e +# SEE https://docs.docker.com/engine/reference/builder/#healthcheck + + +from datetime import timedelta +from typing import TypeAlias + +from pydantic import BaseModel + + +class IsResponsive(BaseModel): + elapsed: timedelta # time elapsed to respond + + def __bool__(self) -> bool: + return True + + +class IsNonResponsive(BaseModel): + reason: str + + def __bool__(self) -> bool: + return False + + +LivenessResult: TypeAlias = IsResponsive | IsNonResponsive diff --git a/packages/service-library/src/servicelib/fastapi/http_client.py b/packages/service-library/src/servicelib/fastapi/http_client.py index c8ddd0a71d8..986973adcf2 100644 --- a/packages/service-library/src/servicelib/fastapi/http_client.py +++ b/packages/service-library/src/servicelib/fastapi/http_client.py @@ -1,39 +1,12 @@ import contextlib import logging -from dataclasses import dataclass -from datetime import timedelta -from typing import TypeAlias import httpx from fastapi import FastAPI +from models_library.healthchecks import IsNonResponsive, IsResponsive, LivenessResult _logger = logging.getLogger(__name__) -# -# Healthchecks: liveness probes and readiness probes -# -# SEE https://medium.com/polarsquad/how-should-i-answer-a-health-check-aa1fcf6e858e -# SEE https://docs.docker.com/engine/reference/builder/#healthcheck - - -@dataclass -class ServiceIsAlive: - elapsed: timedelta - - def __bool__(self) -> bool: - return True - - -@dataclass -class ServiceIsDead: - reason: str - - def __bool__(self) -> bool: - return False - - -LivenessResult: TypeAlias = ServiceIsAlive | ServiceIsDead - class BaseHttpApi: def __init__(self, client: httpx.AsyncClient): @@ -82,9 +55,9 @@ async def is_healhy(self) -> bool: async def check_liveness(self) -> LivenessResult: try: response = await self.client.get("/") - return ServiceIsAlive(elapsed=response.elapsed) + return IsResponsive(elapsed=response.elapsed) except httpx.RequestError as err: - return ServiceIsDead(reason=f"{err}") + return IsNonResponsive(reason=f"{err}") class AppStateMixin: diff --git a/packages/service-library/tests/fastapi/test_http_client.py b/packages/service-library/tests/fastapi/test_http_client.py index 483051476cf..7cbf40a726e 100644 --- a/packages/service-library/tests/fastapi/test_http_client.py +++ b/packages/service-library/tests/fastapi/test_http_client.py @@ -12,6 +12,7 @@ import respx from asgi_lifespan import LifespanManager from fastapi import FastAPI, status +from models_library.healthchecks import IsResponsive from servicelib.fastapi.http_client import AppStateMixin, BaseHttpApi @@ -98,5 +99,10 @@ class MyClientApi(BaseHttpApi, AppStateMixin): assert await api.ping() assert await api.is_healhy() + alive = await api.check_liveness() + assert bool(alive) + assert isinstance(alive, IsResponsive) + assert alive.elapsed.total_seconds() < 1 + # shutdown event assert api.client.is_closed From dbcff38473259d5f7b959ba2ce6fe1c4e41e9ceb Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 19:36:27 +0100 Subject: [PATCH 17/78] health --- .../api/rest/_dependencies.py | 3 +- .../api/rpc/_health.py | 26 ++++++++++++++++ .../services/healthchecks.py | 31 +++++++++---------- .../services/postgres.py | 21 +++++++++++++ 4 files changed, 64 insertions(+), 17 deletions(-) create mode 100644 services/payments/src/simcore_service_payments/api/rpc/_health.py diff --git a/services/payments/src/simcore_service_payments/api/rest/_dependencies.py b/services/payments/src/simcore_service_payments/api/rest/_dependencies.py index 8388242d820..773f215cbba 100644 --- a/services/payments/src/simcore_service_payments/api/rest/_dependencies.py +++ b/services/payments/src/simcore_service_payments/api/rest/_dependencies.py @@ -12,6 +12,7 @@ from ...db.base import BaseRepository from ...models.auth import SessionData from ...services.auth import get_session_data +from ...services.postgres import get_engine from ...services.resource_usage_tracker import ResourceUsageTrackerApi _logger = logging.getLogger(__name__) @@ -43,7 +44,7 @@ def get_rut_api(request: Request) -> ResourceUsageTrackerApi: def get_db_engine(request: Request) -> AsyncEngine: - engine: AsyncEngine = request.app.state.engine + engine: AsyncEngine = get_engine(request.app) assert engine # nosec return engine diff --git a/services/payments/src/simcore_service_payments/api/rpc/_health.py b/services/payments/src/simcore_service_payments/api/rpc/_health.py new file mode 100644 index 00000000000..b5323c6d801 --- /dev/null +++ b/services/payments/src/simcore_service_payments/api/rpc/_health.py @@ -0,0 +1,26 @@ +import logging + +from fastapi import FastAPI +from models_library.healthchecks import LivenessResult +from servicelib.rabbitmq import RPCRouter + +from ...services.healthchecks import create_health_report +from ...services.payments_gateway import PaymentsGatewayApi +from ...services.postgres import get_engine +from ...services.resource_usage_tracker import ResourceUsageTrackerApi + +_logger = logging.getLogger(__name__) + + +router = RPCRouter() + + +@router.expose() +async def check_health( + app: FastAPI, +) -> dict[str, LivenessResult]: + return await create_health_report( + gateway=PaymentsGatewayApi.get_from_app_state(app), + rut=ResourceUsageTrackerApi.get_from_app_state(app), + engine=get_engine(app), + ) diff --git a/services/payments/src/simcore_service_payments/services/healthchecks.py b/services/payments/src/simcore_service_payments/services/healthchecks.py index 97cab5d2486..1c0c3559dd0 100644 --- a/services/payments/src/simcore_service_payments/services/healthchecks.py +++ b/services/payments/src/simcore_service_payments/services/healthchecks.py @@ -53,31 +53,30 @@ # - no. log why? alert! # +import asyncio import logging +from models_library.healthchecks import LivenessResult from sqlalchemy.ext.asyncio import AsyncEngine from .payments_gateway import PaymentsGatewayApi +from .postgres import check_postgres_liveness from .resource_usage_tracker import ResourceUsageTrackerApi _logger = logging.getLogger(__name__) -async def check_payments_gateway_liveness( - payments_gateway_api: PaymentsGatewayApi, -) -> bool: - alive = await payments_gateway_api.check_liveness() - _logger.info( - "liveness check of '%s': %s", payments_gateway_api.client.base_url, alive +async def create_health_report( + gateway: PaymentsGatewayApi, + rut: ResourceUsageTrackerApi, + engine: AsyncEngine, +) -> dict[str, LivenessResult]: + gateway_liveness, rut_liveness, db_liveness = await asyncio.gather( + gateway.check_liveness(), rut.check_liveness(), check_postgres_liveness(engine) ) - return bool(alive) - -async def check_resource_usage_tracker_liveness(rut: ResourceUsageTrackerApi) -> bool: - alive = await rut.check_liveness() - _logger.info("liveness check of '%s': %s", rut.client.base_url, alive) - return bool(alive) - - -async def check_postgres_liveness(engine: AsyncEngine) -> bool: - raise NotImplementedError + return { + "payments_gateway": gateway_liveness, + "resource_usage_tracker": rut_liveness, + "postgres": db_liveness, + } diff --git a/services/payments/src/simcore_service_payments/services/postgres.py b/services/payments/src/simcore_service_payments/services/postgres.py index 59c7d0f950e..a5b4f159ef3 100644 --- a/services/payments/src/simcore_service_payments/services/postgres.py +++ b/services/payments/src/simcore_service_payments/services/postgres.py @@ -1,10 +1,31 @@ +import time + from fastapi import FastAPI +from models_library.healthchecks import IsNonResponsive, IsResponsive, LivenessResult from servicelib.db_async_engine import close_db_connection, connect_to_db +from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncEngine from ..core.settings import ApplicationSettings +def get_engine(app: FastAPI) -> AsyncEngine: + assert app.state.engine # nosec + return app.state.engine + + +async def check_postgres_liveness(engine: AsyncEngine) -> LivenessResult: + try: + tic = time.time() + # test + async with engine.connect() as connection: + await connection.execute("SELECT 1") + + return IsResponsive(elapsed=time.time() - tic) + except SQLAlchemyError as err: + return IsNonResponsive(reason=f"{err}") + + def setup_postgres(app: FastAPI): app.state.engine = None From a53f454e83df842bc6dec053ffbc3d550eb963a0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Sat, 11 Nov 2023 20:07:51 +0100 Subject: [PATCH 18/78] mypy --- .../src/simcore_service_payments/services/postgres.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/payments/src/simcore_service_payments/services/postgres.py b/services/payments/src/simcore_service_payments/services/postgres.py index a5b4f159ef3..5a731551611 100644 --- a/services/payments/src/simcore_service_payments/services/postgres.py +++ b/services/payments/src/simcore_service_payments/services/postgres.py @@ -11,16 +11,16 @@ def get_engine(app: FastAPI) -> AsyncEngine: assert app.state.engine # nosec - return app.state.engine + engine: AsyncEngine = app.state.engine + return engine async def check_postgres_liveness(engine: AsyncEngine) -> LivenessResult: try: tic = time.time() # test - async with engine.connect() as connection: - await connection.execute("SELECT 1") - + async with engine.connect(): + ... return IsResponsive(elapsed=time.time() - tic) except SQLAlchemyError as err: return IsNonResponsive(reason=f"{err}") From 9bfb2e1e6c1d9efc80bbcead369f15eab4ee31a7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:39:51 +0100 Subject: [PATCH 19/78] cleanup health --- .../api/rpc/_health.py | 26 --------- .../services/healthchecks.py | 55 ------------------- 2 files changed, 81 deletions(-) delete mode 100644 services/payments/src/simcore_service_payments/api/rpc/_health.py diff --git a/services/payments/src/simcore_service_payments/api/rpc/_health.py b/services/payments/src/simcore_service_payments/api/rpc/_health.py deleted file mode 100644 index b5323c6d801..00000000000 --- a/services/payments/src/simcore_service_payments/api/rpc/_health.py +++ /dev/null @@ -1,26 +0,0 @@ -import logging - -from fastapi import FastAPI -from models_library.healthchecks import LivenessResult -from servicelib.rabbitmq import RPCRouter - -from ...services.healthchecks import create_health_report -from ...services.payments_gateway import PaymentsGatewayApi -from ...services.postgres import get_engine -from ...services.resource_usage_tracker import ResourceUsageTrackerApi - -_logger = logging.getLogger(__name__) - - -router = RPCRouter() - - -@router.expose() -async def check_health( - app: FastAPI, -) -> dict[str, LivenessResult]: - return await create_health_report( - gateway=PaymentsGatewayApi.get_from_app_state(app), - rut=ResourceUsageTrackerApi.get_from_app_state(app), - engine=get_engine(app), - ) diff --git a/services/payments/src/simcore_service_payments/services/healthchecks.py b/services/payments/src/simcore_service_payments/services/healthchecks.py index 1c0c3559dd0..98774700f44 100644 --- a/services/payments/src/simcore_service_payments/services/healthchecks.py +++ b/services/payments/src/simcore_service_payments/services/healthchecks.py @@ -1,58 +1,3 @@ -""" - -## Types of health checks - -Based on the service types, we can categorize health checks based on the actions they take. - -- Reboot: When the target is unhealthy, the target should be restarted to recover to a working state. -Container and VM orchestration platforms typically perform reboots. -- Cut traffic: When the target is unhealthy, no traffic should be sent to the target. Service discovery -services and load balancers typically cut traffic from targets in one way or another. - -The difference between these is that rebooting attempts to actively repair the target, while cutting -traffic leaves room for the target to repair itself. - -In Kubernetes health-checks are called *probes*: -- The health check for reboots is called a *liveness probe*: "Check if the container is alive". -- The health check for cutting traffic is called a *readiness probe*: "Check if the container is ready to receive traffic". - -Taken from https://medium.com/polarsquad/how-should-i-answer-a-health-check-aa1fcf6e858e - - -## docker healthchecks: - - --interval=DURATION (default: 30s) - --timeout=DURATION (default: 30s) - --start-period=DURATION (default: 0s) - --retries=N (default: 3) - - The health check will first run *interval* seconds after the container is started, and - then again *interval* seconds after each previous check completes. - - If a single run of the check takes longer than *timeout* seconds then the check is considered to have failed (SEE HealthCheckFailed). - - It takes *retries* consecutive failures of the health check for the container to be considered **unhealthy**. - - *start period* provides initialization time for containers that need time to bootstrap. Probe failure during - that period will not be counted towards the maximum number of retries. - - However, if a health check succeeds during the *start period*, the container is considered started and all consecutive - failures will be counted towards the maximum number of retries. - -Taken from https://docs.docker.com/engine/reference/builder/#healthcheck -""" - -# can run some diagnostic tests to determine readiness and livelihood -# e.g. Can we do payments? -# -# - is the gateway ready? -# - no. log why? alert! -# - is the RUT reachable? -# - no. log why? alert! -# - is the database reachable? -# - no. log why? alert! -# - import asyncio import logging From 9ee4a3a9334f74e1abb855a48850fb2c73cb96a7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:46:30 +0100 Subject: [PATCH 20/78] undo liveness --- .../src/models_library/healthchecks.py | 28 ------------------- .../src/servicelib/fastapi/http_client.py | 8 ------ .../tests/fastapi/test_http_client.py | 6 ---- .../services/healthchecks.py | 27 ------------------ .../services/postgres.py | 15 ---------- 5 files changed, 84 deletions(-) delete mode 100644 packages/models-library/src/models_library/healthchecks.py delete mode 100644 services/payments/src/simcore_service_payments/services/healthchecks.py diff --git a/packages/models-library/src/models_library/healthchecks.py b/packages/models-library/src/models_library/healthchecks.py deleted file mode 100644 index 11e6f88162a..00000000000 --- a/packages/models-library/src/models_library/healthchecks.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Healthchecks: liveness probes and readiness probes -# -# SEE https://medium.com/polarsquad/how-should-i-answer-a-health-check-aa1fcf6e858e -# SEE https://docs.docker.com/engine/reference/builder/#healthcheck - - -from datetime import timedelta -from typing import TypeAlias - -from pydantic import BaseModel - - -class IsResponsive(BaseModel): - elapsed: timedelta # time elapsed to respond - - def __bool__(self) -> bool: - return True - - -class IsNonResponsive(BaseModel): - reason: str - - def __bool__(self) -> bool: - return False - - -LivenessResult: TypeAlias = IsResponsive | IsNonResponsive diff --git a/packages/service-library/src/servicelib/fastapi/http_client.py b/packages/service-library/src/servicelib/fastapi/http_client.py index 986973adcf2..2eb87400eee 100644 --- a/packages/service-library/src/servicelib/fastapi/http_client.py +++ b/packages/service-library/src/servicelib/fastapi/http_client.py @@ -3,7 +3,6 @@ import httpx from fastapi import FastAPI -from models_library.healthchecks import IsNonResponsive, IsResponsive, LivenessResult _logger = logging.getLogger(__name__) @@ -52,13 +51,6 @@ async def is_healhy(self) -> bool: except httpx.HTTPError: return False - async def check_liveness(self) -> LivenessResult: - try: - response = await self.client.get("/") - return IsResponsive(elapsed=response.elapsed) - except httpx.RequestError as err: - return IsNonResponsive(reason=f"{err}") - class AppStateMixin: """ diff --git a/packages/service-library/tests/fastapi/test_http_client.py b/packages/service-library/tests/fastapi/test_http_client.py index 7cbf40a726e..483051476cf 100644 --- a/packages/service-library/tests/fastapi/test_http_client.py +++ b/packages/service-library/tests/fastapi/test_http_client.py @@ -12,7 +12,6 @@ import respx from asgi_lifespan import LifespanManager from fastapi import FastAPI, status -from models_library.healthchecks import IsResponsive from servicelib.fastapi.http_client import AppStateMixin, BaseHttpApi @@ -99,10 +98,5 @@ class MyClientApi(BaseHttpApi, AppStateMixin): assert await api.ping() assert await api.is_healhy() - alive = await api.check_liveness() - assert bool(alive) - assert isinstance(alive, IsResponsive) - assert alive.elapsed.total_seconds() < 1 - # shutdown event assert api.client.is_closed diff --git a/services/payments/src/simcore_service_payments/services/healthchecks.py b/services/payments/src/simcore_service_payments/services/healthchecks.py deleted file mode 100644 index 98774700f44..00000000000 --- a/services/payments/src/simcore_service_payments/services/healthchecks.py +++ /dev/null @@ -1,27 +0,0 @@ -import asyncio -import logging - -from models_library.healthchecks import LivenessResult -from sqlalchemy.ext.asyncio import AsyncEngine - -from .payments_gateway import PaymentsGatewayApi -from .postgres import check_postgres_liveness -from .resource_usage_tracker import ResourceUsageTrackerApi - -_logger = logging.getLogger(__name__) - - -async def create_health_report( - gateway: PaymentsGatewayApi, - rut: ResourceUsageTrackerApi, - engine: AsyncEngine, -) -> dict[str, LivenessResult]: - gateway_liveness, rut_liveness, db_liveness = await asyncio.gather( - gateway.check_liveness(), rut.check_liveness(), check_postgres_liveness(engine) - ) - - return { - "payments_gateway": gateway_liveness, - "resource_usage_tracker": rut_liveness, - "postgres": db_liveness, - } diff --git a/services/payments/src/simcore_service_payments/services/postgres.py b/services/payments/src/simcore_service_payments/services/postgres.py index 5a731551611..59d1c6f18c4 100644 --- a/services/payments/src/simcore_service_payments/services/postgres.py +++ b/services/payments/src/simcore_service_payments/services/postgres.py @@ -1,9 +1,5 @@ -import time - from fastapi import FastAPI -from models_library.healthchecks import IsNonResponsive, IsResponsive, LivenessResult from servicelib.db_async_engine import close_db_connection, connect_to_db -from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncEngine from ..core.settings import ApplicationSettings @@ -15,17 +11,6 @@ def get_engine(app: FastAPI) -> AsyncEngine: return engine -async def check_postgres_liveness(engine: AsyncEngine) -> LivenessResult: - try: - tic = time.time() - # test - async with engine.connect(): - ... - return IsResponsive(elapsed=time.time() - tic) - except SQLAlchemyError as err: - return IsNonResponsive(reason=f"{err}") - - def setup_postgres(app: FastAPI): app.state.engine = None From 6eb57594aaf592c8a839210924430c778ed427f6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:21:48 +0100 Subject: [PATCH 21/78] fixtures --- services/payments/tests/unit/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index ce38f392e72..baae9d56be7 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -42,7 +42,7 @@ ) -@pytest.fixture +@pytest.fixture(scope="session") def is_pdb_enabled(request: pytest.FixtureRequest): """Returns true if tests are set to use interactive debugger, i.e. --pdb""" options = request.config.option @@ -322,7 +322,7 @@ def pytest_addoption(parser: pytest.Parser): ) -@pytest.fixture +@pytest.fixture(scope="session") def external_environment(request: pytest.FixtureRequest) -> EnvVarsDict: """ If a file under test folder prefixed with `.env-secret` is present, From f5c8fe59d3ba404b4a37bfbd9a1ef34221b06ac6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Nov 2023 15:12:08 +0100 Subject: [PATCH 22/78] vscode swagger extension --- .vscode/extensions.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 65f20197c67..432a09e56c0 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,5 +1,6 @@ { "recommendations": [ + "42Crunch.vscode-openapi", "charliermarsh.ruff", "eamodio.gitlens", "exiasr.hadolint", From accd7dffbd4e202b244a39483d8f30f7e427b840 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Nov 2023 15:12:28 +0100 Subject: [PATCH 23/78] WIP; error handling in gateway --- .../services/payments_gateway.py | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index bc011f9309c..c734913f30d 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -5,14 +5,18 @@ """ +import contextlib +import functools import logging import warnings +from collections.abc import Callable import httpx from fastapi import FastAPI from fastapi.encoders import jsonable_encoder -from httpx import URL +from httpx import URL, HTTPStatusError from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID +from pydantic.errors import PydanticErrorMixin from servicelib.fastapi.http_client import AppStateMixin, BaseHttpApi from simcore_service_payments.models.schemas.acknowledgements import ( AckPaymentWithPaymentMethod, @@ -21,6 +25,7 @@ from ..core.settings import ApplicationSettings from ..models.payments_gateway import ( BatchGetPaymentMethods, + ErrorModel, GetPaymentMethod, InitPayment, InitPaymentMethod, @@ -33,6 +38,38 @@ _logger = logging.getLogger(__name__) +class PaymentsGatewayError(PydanticErrorMixin, ValueError): + msg_template = "Payment-gateway got {status_code} for {operation_id}: {reason}" + + +@contextlib.contextmanager +def _raise_if_error(operation_id: str): + try: + + yield + + except HTTPStatusError as err: + + model = ErrorModel.parse_obj(err.response.json()) + _logger.debug("{}") + + raise PaymentsGatewayError( + operation_id=operation_id, + status_code=err.response.status_code, + reason=model.message, + model=ErrorModel.parse_obj(err.response.json()), + ) from err + + +def _handle_status_errors(coro: Callable): + @functools.wraps(coro) + async def _wrapper(self, *args, **kwargs): + with _raise_if_error(operation_id=coro.__name__): + return await coro(self, *args, **kwargs) + + return _wrapper + + class _GatewayApiAuth(httpx.Auth): def __init__(self, secret): self.token = secret @@ -49,6 +86,7 @@ class PaymentsGatewayApi(BaseHttpApi, AppStateMixin): # api: one-time-payment workflow # + @_handle_status_errors async def init_payment(self, payment: InitPayment) -> PaymentInitiated: response = await self.client.post( "/init", @@ -60,6 +98,7 @@ async def init_payment(self, payment: InitPayment) -> PaymentInitiated: def get_form_payment_url(self, id_: PaymentID) -> URL: return self.client.base_url.copy_with(path="/pay", params={"id": f"{id_}"}) + @_handle_status_errors async def cancel_payment( self, payment_initiated: PaymentInitiated ) -> PaymentCancelled: @@ -74,6 +113,7 @@ async def cancel_payment( # api: payment method workflows # + @_handle_status_errors async def init_payment_method( self, payment_method: InitPaymentMethod, @@ -92,6 +132,7 @@ def get_form_payment_method_url(self, id_: PaymentMethodID) -> URL: # CRUD + @_handle_status_errors async def get_many_payment_methods( self, ids_: list[PaymentMethodID] ) -> list[GetPaymentMethod]: @@ -102,15 +143,18 @@ async def get_many_payment_methods( response.raise_for_status() return PaymentMethodsBatch.parse_obj(response.json()).items + @_handle_status_errors async def get_payment_method(self, id_: PaymentMethodID) -> GetPaymentMethod: response = await self.client.get(f"/payment-methods/{id_}") response.raise_for_status() return GetPaymentMethod.parse_obj(response.json()) + @_handle_status_errors async def delete_payment_method(self, id_: PaymentMethodID) -> None: response = await self.client.delete(f"/payment-methods/{id_}") response.raise_for_status() + @_handle_status_errors async def init_payment_with_payment_method( self, id_: PaymentMethodID, payment: InitPayment ) -> PaymentInitiated: @@ -126,6 +170,7 @@ async def init_payment_with_payment_method( response.raise_for_status() return PaymentInitiated.parse_obj(response.json()) + @_handle_status_errors async def pay_with_payment_method( self, id_: PaymentMethodID, payment: InitPayment ) -> AckPaymentWithPaymentMethod: From 08cb2140e2882b1e73c91529563b18f37f52f385 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:25:51 +0100 Subject: [PATCH 24/78] updates OAS --- services/payments/openapi.json | 10 +++- .../models/schemas/acknowledgements.py | 50 +++++++++++++------ 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/services/payments/openapi.json b/services/payments/openapi.json index aeb658b7350..6e760361285 100644 --- a/services/payments/openapi.json +++ b/services/payments/openapi.json @@ -220,6 +220,13 @@ "type": "string", "title": "Message" }, + "provider_payment_id": { + "type": "string", + "maxLength": 50, + "minLength": 1, + "title": "Provider Payment Id", + "description": "Payment ID from the provider (e.g. stripe payment ID)" + }, "invoice_url": { "type": "string", "maxLength": 2083, @@ -245,10 +252,11 @@ "title": "AckPayment", "example": { "success": true, + "provider_payment_id": "pi_123ABC", "invoice_url": "https://invoices.com/id=12345", "saved": { "success": true, - "payment_method_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6" + "payment_method_id": "3FA85F64-5717-4562-B3FC-2C963F66AFA6" } } }, diff --git a/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py b/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py index 13a6494d9c5..59ef23ed51b 100644 --- a/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py +++ b/services/payments/src/simcore_service_payments/models/schemas/acknowledgements.py @@ -1,6 +1,7 @@ from typing import Any, ClassVar from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID +from models_library.basic_types import IDStr from pydantic import BaseModel, Field, HttpUrl, validator @@ -9,6 +10,17 @@ class _BaseAck(BaseModel): message: str = Field(default=None) +class _BaseAckPayment(_BaseAck): + provider_payment_id: IDStr = Field( + default=None, + description="Payment ID from the provider (e.g. stripe payment ID)", + ) + + invoice_url: HttpUrl | None = Field( + default=None, description="Link to invoice is required when success=true" + ) + + # # ACK payment-methods # @@ -22,22 +34,13 @@ class SavedPaymentMethod(AckPaymentMethod): payment_method_id: PaymentMethodID -class AckPaymentWithPaymentMethod(_BaseAck): - # NOTE: This model is equivalent to `AckPayment` below - # Nonetheless I decided to separate it for clarity in the OAS - # since in payments w/ payment-method the field `saved` will - # never be provided. - invoice_url: HttpUrl | None = Field( - default=None, description="Link to invoice is required when success=true" - ) - - # -# ACK one-time payments +# ACK payments # _ONE_TIME_SUCCESS: dict[str, Any] = { "success": True, + "provider_payment_id": "pi_123ABC", "invoice_url": "https://invoices.com/id=12345", } _EXAMPLES: list[dict[str, Any]] = [ @@ -48,7 +51,7 @@ class AckPaymentWithPaymentMethod(_BaseAck): **_ONE_TIME_SUCCESS, "saved": { "success": True, - "payment_method_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "payment_method_id": "3FA85F64-5717-4562-B3FC-2C963F66AFA6", }, }, # 2. one-time-payment successful but payment-method-saved failed @@ -67,10 +70,7 @@ class AckPaymentWithPaymentMethod(_BaseAck): ] -class AckPayment(_BaseAck): - invoice_url: HttpUrl | None = Field( - default=None, description="Link to invoice is required when success=true" - ) +class AckPayment(_BaseAckPayment): saved: SavedPaymentMethod | None = Field( default=None, @@ -94,6 +94,24 @@ def success_requires_invoice(cls, v, values): return v +class AckPaymentWithPaymentMethod(_BaseAckPayment): + # NOTE: This model is equivalent to `AckPayment`, nonetheless + # I decided to separate it for clarity in the OAS since in payments + # w/ payment-method the field `saved` will never be provided, + + payment_id: PaymentID = Field( + default=None, description="Payment ID from the gateway" + ) + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "example": { + **_ONE_TIME_SUCCESS, + "payment_id": "D19EE68B-B007-4B61-A8BC-32B7115FB244", + }, # shown in openapi.json + } + + assert PaymentID # nosec assert PaymentMethodID # nosec From 85387b6ccc50326ff5d5c0f52f10d9f33f1b71ce Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:26:33 +0100 Subject: [PATCH 25/78] updates version --- services/payments/scripts/fake_payment_gateway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/payments/scripts/fake_payment_gateway.py b/services/payments/scripts/fake_payment_gateway.py index 4401d7fc1e6..0df5e7b76f2 100644 --- a/services/payments/scripts/fake_payment_gateway.py +++ b/services/payments/scripts/fake_payment_gateway.py @@ -347,7 +347,7 @@ async def _app_lifespan(app: FastAPI): def create_app(): app = FastAPI( title="fake-payment-gateway", - version="0.2.1", + version="0.3.0", lifespan=_app_lifespan, debug=True, ) From 7adec3fc8bc2c006c6cfcca862f384178c9f8487 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:26:48 +0100 Subject: [PATCH 26/78] =?UTF-8?q?services/payments=20version:=201.3.1=20?= =?UTF-8?q?=E2=86=92=201.4.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/payments/VERSION | 2 +- services/payments/setup.cfg | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/payments/VERSION b/services/payments/VERSION index 3a3cd8cc8b0..88c5fb891dc 100644 --- a/services/payments/VERSION +++ b/services/payments/VERSION @@ -1 +1 @@ -1.3.1 +1.4.0 diff --git a/services/payments/setup.cfg b/services/payments/setup.cfg index 7a52ccb4934..3c4178f6f0a 100644 --- a/services/payments/setup.cfg +++ b/services/payments/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.3.1 +current_version = 1.4.0 commit = True message = services/payments version: {current_version} → {new_version} tag = False @@ -9,6 +9,6 @@ commit_args = --no-verify [tool:pytest] asyncio_mode = auto -markers = +markers = testit: "marks test to run during development" acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." From 6d318e656595d8886db144ef650662fb304501d2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:27:59 +0100 Subject: [PATCH 27/78] version --- services/payments/openapi.json | 2 +- services/payments/setup.cfg | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/payments/openapi.json b/services/payments/openapi.json index 6e760361285..0a0590edf31 100644 --- a/services/payments/openapi.json +++ b/services/payments/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "simcore-service-payments web API", "description": " Service that manages creation and validation of registration payments", - "version": "1.3.1" + "version": "1.4.0" }, "paths": { "/": { diff --git a/services/payments/setup.cfg b/services/payments/setup.cfg index 3c4178f6f0a..c5891a3d950 100644 --- a/services/payments/setup.cfg +++ b/services/payments/setup.cfg @@ -6,9 +6,10 @@ tag = False commit_args = --no-verify [bumpversion:file:VERSION] +[bumpversion:file:openapi.json] [tool:pytest] asyncio_mode = auto -markers = +markers = testit: "marks test to run during development" acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." From 52cdcc52177b8b15457d76e0f50f9b42357e2ecb Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 11:36:41 +0100 Subject: [PATCH 28/78] external mark --- services/payments/Makefile | 4 ++-- services/payments/setup.cfg | 1 + .../simcore_service_payments/services/payments_gateway.py | 7 ++++++- .../payments/tests/unit/test_services_payments_gateway.py | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/services/payments/Makefile b/services/payments/Makefile index db3152765a6..80f2cd6a8e9 100644 --- a/services/payments/Makefile +++ b/services/payments/Makefile @@ -23,8 +23,8 @@ external ?= .env-secret test-dev-unit-external: ## runs test-dev against external service defined in $(external) envfile # Running tests using external environ '$(external)' - $(MAKE) test-dev-unit pytest-parameters="--external-envfile=$(external)" + $(MAKE) test-dev-unit pytest-parameters="--external-envfile=$(external) -m external_test" test-ci-unit-external: ## runs test-ci against external service defined in $(external) envfile # Running tests using external environ '$(external)' - $(MAKE) test-ci-unit pytest-parameters="--external-envfile=$(external)" + $(MAKE) test-ci-unit pytest-parameters="--external-envfile=$(external) -m external_test" diff --git a/services/payments/setup.cfg b/services/payments/setup.cfg index c5891a3d950..7b9a60b6583 100644 --- a/services/payments/setup.cfg +++ b/services/payments/setup.cfg @@ -13,3 +13,4 @@ asyncio_mode = auto markers = testit: "marks test to run during development" acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." + external_test: "marks tests that *can* be run against an external configuration passed by --external-envfile" diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index c734913f30d..6ed785e1f5a 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -51,7 +51,12 @@ def _raise_if_error(operation_id: str): except HTTPStatusError as err: model = ErrorModel.parse_obj(err.response.json()) - _logger.debug("{}") + _logger.debug( + "status error %d: %s\n%s", + err.response.status_code, + err, + model.json(indent=1), + ) raise PaymentsGatewayError( operation_id=operation_id, diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index d88606d87ce..4b5e6a4048f 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -124,6 +124,7 @@ async def test_one_time_payment_workflow( assert mock_payments_gateway_service_or_none.routes["cancel_payment"].called +@pytest.mark.external_test() async def test_payment_methods_workflow( app: FastAPI, faker: Faker, From fe05546b99129c916e4bc5e592b6f4e17cc9cbcd Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 11:38:04 +0100 Subject: [PATCH 29/78] extension --- .vscode/extensions.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 432a09e56c0..02968d94c72 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -8,6 +8,7 @@ "ms-python.black-formatter", "ms-python.pylint", "ms-python.python", + "ms-vscode.makefile-tools", "njpwerner.autodocstring", "samuelcolvin.jinjahtml", "timonwong.shellcheck", From 6e01880d9983da7ba29d5ec08cf3df14f15f10fd Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 12:33:46 +0100 Subject: [PATCH 30/78] updates errors --- .../services/payments_gateway.py | 26 ++++++++++--------- .../unit/test_services_payments_gateway.py | 8 +++--- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index 6ed785e1f5a..233ee860e57 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -10,12 +10,14 @@ import logging import warnings from collections.abc import Callable +from contextlib import suppress import httpx from fastapi import FastAPI from fastapi.encoders import jsonable_encoder from httpx import URL, HTTPStatusError from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID +from pydantic import ValidationError from pydantic.errors import PydanticErrorMixin from servicelib.fastapi.http_client import AppStateMixin, BaseHttpApi from simcore_service_payments.models.schemas.acknowledgements import ( @@ -42,6 +44,13 @@ class PaymentsGatewayError(PydanticErrorMixin, ValueError): msg_template = "Payment-gateway got {status_code} for {operation_id}: {reason}" +def _parse_raw_or_none(text: str | None): + if text: + with suppress(ValidationError): + return ErrorModel.parse_raw(text) + return None + + @contextlib.contextmanager def _raise_if_error(operation_id: str): try: @@ -49,20 +58,13 @@ def _raise_if_error(operation_id: str): yield except HTTPStatusError as err: - - model = ErrorModel.parse_obj(err.response.json()) - _logger.debug( - "status error %d: %s\n%s", - err.response.status_code, - err, - model.json(indent=1), - ) + model = _parse_raw_or_none(err.response.text) raise PaymentsGatewayError( - operation_id=operation_id, - status_code=err.response.status_code, - reason=model.message, - model=ErrorModel.parse_obj(err.response.json()), + operation_id=f"PaymentsGatewayApi.{operation_id}", + reason=model.message if model else f"{err}", + http_status_error=err, + model=model, ) from err diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index 4b5e6a4048f..2e58d535630 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -4,7 +4,6 @@ # pylint: disable=too-many-arguments -import httpx import pytest from faker import Faker from fastapi import FastAPI, status @@ -17,6 +16,7 @@ ) from simcore_service_payments.services.payments_gateway import ( PaymentsGatewayApi, + PaymentsGatewayError, setup_payments_gateway, ) @@ -185,10 +185,12 @@ async def test_payment_methods_workflow( # delete payment-method await payments_gateway_api.delete_payment_method(payment_method_id) - with pytest.raises(httpx.HTTPStatusError) as err_info: + with pytest.raises(PaymentsGatewayError) as err_info: await payments_gateway_api.get_payment_method(payment_method_id) - http_status_error = err_info.value + assert err_info.value.operation_id == "payments_gateway.get_payment_method" + + http_status_error = err_info.value.http_status_error assert http_status_error.response.status_code == status.HTTP_404_NOT_FOUND if mock_payments_gateway_service_or_none: From f9c87cd84033471016434651f6409e69b8a06e12 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 13:45:32 +0100 Subject: [PATCH 31/78] rename --- services/payments/Makefile | 4 ++-- services/payments/setup.cfg | 2 +- .../payments/tests/unit/test_services_payments_gateway.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/payments/Makefile b/services/payments/Makefile index 80f2cd6a8e9..0af63b77c0c 100644 --- a/services/payments/Makefile +++ b/services/payments/Makefile @@ -23,8 +23,8 @@ external ?= .env-secret test-dev-unit-external: ## runs test-dev against external service defined in $(external) envfile # Running tests using external environ '$(external)' - $(MAKE) test-dev-unit pytest-parameters="--external-envfile=$(external) -m external_test" + $(MAKE) test-dev-unit pytest-parameters="--external-envfile=$(external) -m can_run_against_external" test-ci-unit-external: ## runs test-ci against external service defined in $(external) envfile # Running tests using external environ '$(external)' - $(MAKE) test-ci-unit pytest-parameters="--external-envfile=$(external) -m external_test" + $(MAKE) test-ci-unit pytest-parameters="--external-envfile=$(external) -m can_run_against_external" diff --git a/services/payments/setup.cfg b/services/payments/setup.cfg index 7b9a60b6583..cb32927a81b 100644 --- a/services/payments/setup.cfg +++ b/services/payments/setup.cfg @@ -13,4 +13,4 @@ asyncio_mode = auto markers = testit: "marks test to run during development" acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." - external_test: "marks tests that *can* be run against an external configuration passed by --external-envfile" + can_run_against_external: "marks tests that *can* be run against an external configuration passed by --external-envfile" diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index 2e58d535630..9a8287e628d 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -124,7 +124,7 @@ async def test_one_time_payment_workflow( assert mock_payments_gateway_service_or_none.routes["cancel_payment"].called -@pytest.mark.external_test() +@pytest.mark.can_run_against_external() async def test_payment_methods_workflow( app: FastAPI, faker: Faker, From 66846ebbe7adf2455a44eafd374bf30669c065d5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 14:37:05 +0100 Subject: [PATCH 32/78] fixes retries --- .../db/payments_transactions_repo.py | 2 +- .../services/payments.py | 46 +++++++++++++------ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py index 0b7e972be76..8cfde93980d 100644 --- a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py @@ -36,10 +36,10 @@ async def insert_init_payment_transaction( initiated_at: datetime.datetime, ) -> PaymentID: """Annotates init-payment transaction + Raises: PaymentAlreadyExistsError """ - try: async with self.db_engine.begin() as conn: await conn.execute( diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index ff588648d5d..b08d47a39c2 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -25,9 +25,15 @@ PaymentTransactionState, ) from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo +from tenacity import AsyncRetrying +from tenacity.retry import retry_if_exception_type from .._constants import RUT -from ..core.errors import PaymentAlreadyAckedError, PaymentNotFoundError +from ..core.errors import ( + PaymentAlreadyAckedError, + PaymentAlreadyExistsError, + PaymentNotFoundError, +) from ..db.payments_transactions_repo import PaymentsTransactionsRepo from ..models.db import PaymentsTransactionsDB from ..models.payments_gateway import InitPayment, PaymentInitiated @@ -174,7 +180,7 @@ async def on_payment_completed( ) -async def init_payment_with_payment_method( +async def init_payment_with_payment_method( # noqa: PLR0913 gateway: PaymentsGatewayApi, repo_transactions: PaymentsTransactionsRepo, repo_methods: PaymentsMethodsRepo, @@ -231,7 +237,7 @@ async def init_payment_with_payment_method( ) -async def pay_with_payment_method( +async def pay_with_payment_method( # noqa: PLR0913 gateway: PaymentsGatewayApi, repo_transactions: PaymentsTransactionsRepo, repo_methods: PaymentsMethodsRepo, @@ -253,19 +259,7 @@ async def pay_with_payment_method( payment_method_id, user_id=user_id, wallet_id=wallet_id ) - # TODO: retry if PaymentAlreadyExistsError # TODO: async with repo_transactions.begin(): should set a scope transaction inside - payment_id = await repo_transactions.insert_init_payment_transaction( - payment_id=f"{uuid.uuid4()}", # TODO: check with Dennis whether ack_payment.payment_id - price_dollars=amount_dollars, - osparc_credits=target_credits, - product_name=product_name, - user_id=user_id, - user_email=user_email, - wallet_id=wallet_id, - comment=comment, - initiated_at=initiated_at, - ) ack: AckPaymentWithPaymentMethod = await gateway.pay_with_payment_method( acked.payment_method_id, @@ -278,6 +272,28 @@ async def pay_with_payment_method( ), ) + payment_id = ack.payment_id + + async for attempt in AsyncRetrying( + stop_after_attempt=3, + retry=retry_if_exception_type(PaymentAlreadyExistsError), + reraise=True, + ): + with attempt: + payment_id = await repo_transactions.insert_init_payment_transaction( + ack.payment_id or f"{uuid.uuid4()}", + price_dollars=amount_dollars, + osparc_credits=target_credits, + product_name=product_name, + user_id=user_id, + user_email=user_email, + wallet_id=wallet_id, + comment=comment, + initiated_at=initiated_at, + ) + + assert payment_id is not None # nosec + transaction = await repo_transactions.update_ack_payment_transaction( payment_id=payment_id, completion_state=( From 7fc56283b3092a4a3f7111c1145fec8a37b8411c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 14:42:53 +0100 Subject: [PATCH 33/78] minor fixtures --- services/payments/tests/unit/conftest.py | 2 ++ services/payments/tests/unit/test_services_payments_gateway.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index baae9d56be7..f844478a7b6 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -272,6 +272,8 @@ def _pay(request: httpx.Request, pm_id: PaymentMethodID): success=True, message=f"Payment '{payment_id}' with payment-method '{pm_id}'", invoice_url=faker.url(), + provider_payment_id="pi_123456ABCDEFG123456ABCDE", + payment_id=payment_id, ) ), ) diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index 9a8287e628d..edbc88a6b2b 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -188,7 +188,7 @@ async def test_payment_methods_workflow( with pytest.raises(PaymentsGatewayError) as err_info: await payments_gateway_api.get_payment_method(payment_method_id) - assert err_info.value.operation_id == "payments_gateway.get_payment_method" + assert err_info.value.operation_id == "PaymentsGateway.get_payment_method" http_status_error = err_info.value.http_status_error assert http_status_error.response.status_code == status.HTTP_404_NOT_FOUND From 7a6d31b392fe7d3cf769e471004f48e1c6ba2076 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 14:46:21 +0100 Subject: [PATCH 34/78] fixes test --- services/payments/tests/unit/test_services_payments_gateway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index edbc88a6b2b..fd965244767 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -188,7 +188,7 @@ async def test_payment_methods_workflow( with pytest.raises(PaymentsGatewayError) as err_info: await payments_gateway_api.get_payment_method(payment_method_id) - assert err_info.value.operation_id == "PaymentsGateway.get_payment_method" + assert err_info.value.operation_id == "PaymentsGatewayApi.get_payment_method" http_status_error = err_info.value.http_status_error assert http_status_error.response.status_code == status.HTTP_404_NOT_FOUND From a0d5d24de73d180d5fbecb5e181d23133300ae59 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 15:06:35 +0100 Subject: [PATCH 35/78] pay-w-cc in webserver --- api/specs/web-server/_wallets.py | 6 +- .../api/v0/openapi.yaml | 6 +- .../payments/_onetime_api.py | 14 ++++ .../payments/_rpc.py | 44 ++++++++++++ .../simcore_service_webserver/payments/api.py | 2 + .../wallets/_payments_handlers.py | 68 +++++++++++-------- 6 files changed, 104 insertions(+), 36 deletions(-) diff --git a/api/specs/web-server/_wallets.py b/api/specs/web-server/_wallets.py index c93054a2cec..3b0222c2c55 100644 --- a/api/specs/web-server/_wallets.py +++ b/api/specs/web-server/_wallets.py @@ -168,10 +168,10 @@ async def delete_payment_method( @router.post( "/wallets/{wallet_id}/payments-methods/{payment_method_id}:pay", response_model=Envelope[WalletPaymentInitiated], - response_description="Payment initialized", - status_code=status.HTTP_202_ACCEPTED, + response_description="Pay with payment-method", + status_code=status.HTTP_202_ACCEPTED, # TODO: check in front-end ) -async def init_payment_with_payment_method( +async def pay_with_payment_method( wallet_id: WalletID, payment_method_id: PaymentMethodID, _body: CreateWalletPayment ): ... diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index d84d9c55f10..652874156aa 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -4312,8 +4312,8 @@ paths: post: tags: - wallets - summary: Init Payment With Payment Method - operationId: init_payment_with_payment_method + summary: Pay With Payment Method + operationId: pay_with_payment_method parameters: - required: true schema: @@ -4339,7 +4339,7 @@ paths: required: true responses: '202': - description: Payment initialized + description: Pay with payment-method content: application/json: schema: diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index 9a49028cbe8..0097954c0c4 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -285,6 +285,20 @@ async def cancel_payment_to_wallet( ) +async def pay_with_payment_method( + app: web.Application, + *, + price_dollars: Decimal, + osparc_credits: Decimal, + product_name: str, + user_id: UserID, + wallet_id: WalletID, + payment_method_id: PaymentMethodID, + comment: str | None, +) -> PaymentTransaction: + raise NotImplementedError + + async def list_user_payments_page( app: web.Application, product_name: str, diff --git a/services/web/server/src/simcore_service_webserver/payments/_rpc.py b/services/web/server/src/simcore_service_webserver/payments/_rpc.py index bd4a2ec74b9..326a3d436dd 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_rpc.py +++ b/services/web/server/src/simcore_service_webserver/payments/_rpc.py @@ -3,6 +3,7 @@ """ import logging +import warnings from decimal import Decimal from aiohttp import web @@ -12,6 +13,7 @@ PaymentMethodGet, PaymentMethodID, PaymentMethodInitiated, + PaymentTransaction, WalletPaymentInitiated, ) from models_library.basic_types import IDStr @@ -225,6 +227,11 @@ async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=t user_email: EmailStr, comment: str | None = None, ) -> WalletPaymentInitiated: + warnings.warn( + f"{__name__}.init_payment_with_payment_method is deprecated. Use instead pay_with_payment_method", + DeprecationWarning, + stacklevel=1, + ) rpc_client = app[_APP_PAYMENTS_RPC_CLIENT_KEY] result = await rpc_client.request( @@ -243,3 +250,40 @@ async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=t ) assert isinstance(result, WalletPaymentInitiated) # nosec return result + + +@log_decorator(_logger, level=logging.DEBUG) +async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-arguments + app: web.Application, + *, + payment_method_id: PaymentMethodID, + amount_dollars: Decimal, + target_credits: Decimal, + product_name: str, + wallet_id: WalletID, + wallet_name: str, + user_id: UserID, + user_name: str, + user_email: EmailStr, + comment: str | None = None, +) -> WalletPaymentInitiated: + + rpc_client = app[_APP_PAYMENTS_RPC_CLIENT_KEY] + + result = await rpc_client.request( + PAYMENTS_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "pay_with_payment_method"), + payment_method_id=payment_method_id, + amount_dollars=amount_dollars, + target_credits=target_credits, + product_name=product_name, + wallet_id=wallet_id, + wallet_name=wallet_name, + user_id=user_id, + user_name=user_name, + user_email=user_email, + comment=comment, + ) + + assert isinstance(result, PaymentTransaction) + return result diff --git a/services/web/server/src/simcore_service_webserver/payments/api.py b/services/web/server/src/simcore_service_webserver/payments/api.py index 4dbdd9d4a0b..0e3415457d6 100644 --- a/services/web/server/src/simcore_service_webserver/payments/api.py +++ b/services/web/server/src/simcore_service_webserver/payments/api.py @@ -13,6 +13,7 @@ cancel_payment_to_wallet, init_creation_of_wallet_payment, list_user_payments_page, + pay_with_payment_method, ) __all__: tuple[str, ...] = ( @@ -25,6 +26,7 @@ "get_wallet_payment_method", "init_creation_of_wallet_payment_method", "list_wallet_payment_methods", + "pay_with_payment_method", "replace_wallet_payment_autorecharge", ) # nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 38a327a3a78..13700c49d62 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -1,4 +1,5 @@ import logging +from decimal import Decimal from aiohttp import web from models_library.api_schemas_webserver.wallets import ( @@ -33,6 +34,7 @@ init_creation_of_wallet_payment_method, list_user_payments_page, list_wallet_payment_methods, + pay_with_payment_method, replace_wallet_payment_autorecharge, ) from ..products.api import get_current_product_credit_price @@ -48,30 +50,16 @@ _logger = logging.getLogger(__name__) -async def _init_creation_of_payments( - request: web.Request, - user_id, - product_name, - wallet_id, - payment_method_id, - init: CreateWalletPayment, -) -> WalletPaymentInitiated: +async def _eval_osparc_credits_or_raise( + request: web.Request, price_dollars: Decimal +) -> Decimal: # Conversion usd_per_credit = await get_current_product_credit_price(request) if not usd_per_credit: # '0 or None' should raise raise web.HTTPConflict(reason=MSG_PRICE_NOT_DEFINED_ERROR) - return await init_creation_of_wallet_payment( - request.app, - user_id=user_id, - product_name=product_name, - wallet_id=wallet_id, - osparc_credits=init.price_dollars / usd_per_credit, - comment=init.comment, - price_dollars=init.price_dollars, - payment_method_id=payment_method_id, - ) + return price_dollars / usd_per_credit routes = web.RouteTableDef() @@ -100,13 +88,18 @@ async def _create_payment(request: web.Request): extra=get_log_record_extra(user_id=req_ctx.user_id), ): - payment: WalletPaymentInitiated = await _init_creation_of_payments( - request, + osparc_credits = await _eval_osparc_credits_or_raise( + request, body_params.price_dollars + ) + + payment: WalletPaymentInitiated = await init_creation_of_wallet_payment( + request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name, wallet_id=wallet_id, - payment_method_id=None, - init=body_params, + osparc_credits=osparc_credits, + comment=body_params.comment, + price_dollars=body_params.price_dollars, ) return envelope_json_response(payment, web.HTTPCreated) @@ -322,10 +315,7 @@ async def _delete_payment_method(request: web.Request): @login_required @permission_required("wallets.*") @handle_wallets_exceptions -async def _init_payment_with_payment_method(request: web.Request): - """Triggers the creation of a new payment method. - Note that creating a payment-method follows the init-prompt-ack flow - """ +async def _pay_with_payment_method(request: web.Request): req_ctx = WalletsRequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(PaymentMethodsPathParams, request) body_params = await parse_request_body_as(CreateWalletPayment, request) @@ -341,16 +331,34 @@ async def _init_payment_with_payment_method(request: web.Request): extra=get_log_record_extra(user_id=req_ctx.user_id), ): - payment: WalletPaymentInitiated = await _init_creation_of_payments( - request, + osparc_credits = await _eval_osparc_credits_or_raise( + request, body_params.price_dollars + ) + + payment: PaymentTransaction = await pay_with_payment_method( + request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name, wallet_id=wallet_id, payment_method_id=path_params.payment_method_id, - init=body_params, + osparc_credits=osparc_credits, + comment=body_params.comment, + price_dollars=body_params.price_dollars, ) - return envelope_json_response(payment, web.HTTPAccepted) + # TODO: how is front-end reacting? Should i simply trigger socketit from here as a background task?? + + # NOTE: Due to the design change in https://github.com/ITISFoundation/osparc-simcore/pull/5017 + # we decided not to change the return value to avoid changing the front-end logic + # instead we emulate a init-prompt-ack workflow by firing a background task that acks payment + # + return envelope_json_response( + WalletPaymentInitiated( + payment_id=payment.payment_id, + payment_form_url=None, + ), + web.HTTPAccepted, + ) # From 2cc2d35db456f3a4db0ef67f68656667ab1746d7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 15:45:18 +0100 Subject: [PATCH 36/78] fixes tst --- .../simcore_service_payments/services/payments.py | 3 ++- .../wallets/_payments_handlers.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index b08d47a39c2..f0d46d5108e 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -27,6 +27,7 @@ from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo from tenacity import AsyncRetrying from tenacity.retry import retry_if_exception_type +from tenacity.stop import stop_after_attempt from .._constants import RUT from ..core.errors import ( @@ -275,7 +276,7 @@ async def pay_with_payment_method( # noqa: PLR0913 payment_id = ack.payment_id async for attempt in AsyncRetrying( - stop_after_attempt=3, + stop=stop_after_attempt(3), retry=retry_if_exception_type(PaymentAlreadyExistsError), reraise=True, ): diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 13700c49d62..28e371d1cdd 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -1,5 +1,4 @@ import logging -from decimal import Decimal from aiohttp import web from models_library.api_schemas_webserver.wallets import ( @@ -12,8 +11,10 @@ ReplaceWalletAutoRecharge, WalletPaymentInitiated, ) +from models_library.basic_types import NonNegativeDecimal from models_library.rest_pagination import Page, PageQueryParameters from models_library.rest_pagination_utils import paginate_data +from pydantic import parse_obj_as from servicelib.aiohttp.requests_validation import ( parse_request_body_as, parse_request_path_parameters_as, @@ -50,16 +51,16 @@ _logger = logging.getLogger(__name__) -async def _eval_osparc_credits_or_raise( - request: web.Request, price_dollars: Decimal -) -> Decimal: +async def _eval_total_credits_or_raise( + request: web.Request, amount_dollars: NonNegativeDecimal +) -> NonNegativeDecimal: # Conversion usd_per_credit = await get_current_product_credit_price(request) if not usd_per_credit: # '0 or None' should raise raise web.HTTPConflict(reason=MSG_PRICE_NOT_DEFINED_ERROR) - return price_dollars / usd_per_credit + return parse_obj_as(NonNegativeDecimal, amount_dollars / usd_per_credit) routes = web.RouteTableDef() @@ -88,7 +89,7 @@ async def _create_payment(request: web.Request): extra=get_log_record_extra(user_id=req_ctx.user_id), ): - osparc_credits = await _eval_osparc_credits_or_raise( + osparc_credits = await _eval_total_credits_or_raise( request, body_params.price_dollars ) @@ -331,7 +332,7 @@ async def _pay_with_payment_method(request: web.Request): extra=get_log_record_extra(user_id=req_ctx.user_id), ): - osparc_credits = await _eval_osparc_credits_or_raise( + osparc_credits = await _eval_total_credits_or_raise( request, body_params.price_dollars ) From 4fbfac0b57ef6b0b2e99d6ca7dad3255a85d71d2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 15:52:34 +0100 Subject: [PATCH 37/78] minor test failures --- .../src/models_library/api_schemas_webserver/wallets.py | 4 ++-- .../src/simcore_service_webserver/api/v0/openapi.yaml | 1 + .../simcore_service_webserver/wallets/_payments_handlers.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py index 47d328b1df2..eccbe6d774f 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py @@ -4,7 +4,7 @@ from pydantic import Field, HttpUrl -from ..basic_types import IDStr, NonNegativeDecimal +from ..basic_types import AmountDecimal, IDStr, NonNegativeDecimal from ..users import GroupID from ..utils.pydantic_tools_extension import FieldNotRequired from ..wallets import WalletID, WalletStatus @@ -55,7 +55,7 @@ class PutWalletBodyParams(OutputSchema): class CreateWalletPayment(InputSchema): - price_dollars: Decimal + price_dollars: AmountDecimal comment: str = FieldNotRequired(max_length=100) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 652874156aa..99d850173fb 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -5256,6 +5256,7 @@ components: properties: priceDollars: title: Pricedollars + minimum: 0.0 type: number comment: title: Comment diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 28e371d1cdd..0495429419c 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -11,7 +11,7 @@ ReplaceWalletAutoRecharge, WalletPaymentInitiated, ) -from models_library.basic_types import NonNegativeDecimal +from models_library.basic_types import AmountDecimal, NonNegativeDecimal from models_library.rest_pagination import Page, PageQueryParameters from models_library.rest_pagination_utils import paginate_data from pydantic import parse_obj_as @@ -52,7 +52,7 @@ async def _eval_total_credits_or_raise( - request: web.Request, amount_dollars: NonNegativeDecimal + request: web.Request, amount_dollars: AmountDecimal ) -> NonNegativeDecimal: # Conversion usd_per_credit = await get_current_product_credit_price(request) @@ -311,7 +311,7 @@ async def _delete_payment_method(request: web.Request): @routes.post( f"/{VTAG}/wallets/{{wallet_id}}/payments-methods/{{payment_method_id}}:pay", - name="init_payment_with_payment_method", + name="pay_with_payment_method", ) @login_required @permission_required("wallets.*") From d4e51a8e0ebdb975b814d8e7e039cbe5ccac854b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:09:38 +0100 Subject: [PATCH 38/78] adds fake in pay-with-payment-method --- .../payments/_onetime_api.py | 80 ++++++++++++++++++- .../payments/_rpc.py | 2 +- .../with_dbs/03/wallets/payments/conftest.py | 27 ++++--- 3 files changed, 96 insertions(+), 13 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index 0097954c0c4..d592348bf9a 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -285,6 +285,41 @@ async def cancel_payment_to_wallet( ) +async def _fake_pay_with_payment_method( + app, + amount_dollars, + target_credits, + product_name, + wallet_id, + wallet_name, + user_id, + user_name, + user_email, + payment_method_id: PaymentMethodID, + comment, +) -> PaymentTransaction: + + assert user_name # nosec + assert wallet_name # nosec + + payment_inited = await _fake_init_payment( + app, + amount_dollars, + target_credits, + product_name, + wallet_id, + user_id, + user_email, + comment, + ) + return await _ack_creation_of_wallet_payment( + app, + payment_id=payment_inited.payment_id, + completion_state=PaymentTransactionState.SUCCESS, + message=f"Fake payment completed with payment-id = {payment_method_id}", + ) + + async def pay_with_payment_method( app: web.Application, *, @@ -296,7 +331,50 @@ async def pay_with_payment_method( payment_method_id: PaymentMethodID, comment: str | None, ) -> PaymentTransaction: - raise NotImplementedError + + # wallet: check permissions + await raise_for_wallet_payments_permissions( + app, user_id=user_id, wallet_id=wallet_id, product_name=product_name + ) + user_wallet = await get_wallet_by_user( + app, user_id=user_id, wallet_id=wallet_id, product_name=product_name + ) + assert user_wallet.wallet_id == wallet_id # nosec + + # user info + user = await get_user_name_and_email(app, user_id=user_id) + + settings: PaymentsSettings = get_plugin_settings(app) + if settings.PAYMENTS_FAKE_COMPLETION: + return await _fake_pay_with_payment_method( + app, + payment_method_id=payment_method_id, + amount_dollars=price_dollars, + target_credits=osparc_credits, + product_name=product_name, + wallet_id=wallet_id, + wallet_name=user_wallet.name, + user_id=user_id, + user_name=user.name, + user_email=user.email, + comment=comment, + ) + + assert not settings.PAYMENTS_FAKE_COMPLETION # nosec + + return await _rpc.pay_with_payment_method( + app, + payment_method_id=payment_method_id, + amount_dollars=price_dollars, + target_credits=osparc_credits, + product_name=product_name, + wallet_id=wallet_id, + wallet_name=user_wallet.name, + user_id=user_id, + user_name=user.name, + user_email=user.email, + comment=comment, + ) async def list_user_payments_page( diff --git a/services/web/server/src/simcore_service_webserver/payments/_rpc.py b/services/web/server/src/simcore_service_webserver/payments/_rpc.py index 326a3d436dd..7b60475d501 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_rpc.py +++ b/services/web/server/src/simcore_service_webserver/payments/_rpc.py @@ -266,7 +266,7 @@ async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-a user_name: str, user_email: EmailStr, comment: str | None = None, -) -> WalletPaymentInitiated: +) -> PaymentTransaction: rpc_client = app[_APP_PAYMENTS_RPC_CLIENT_KEY] diff --git a/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py b/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py index e94b6570b62..d8a571f17f5 100644 --- a/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py @@ -20,6 +20,7 @@ PaymentMethodGet, PaymentMethodID, PaymentMethodInitiated, + PaymentTransaction, WalletGet, ) from models_library.basic_types import IDStr @@ -41,6 +42,7 @@ from simcore_service_webserver.payments._onetime_api import ( _fake_cancel_payment, _fake_init_payment, + _fake_pay_with_payment_method, ) from simcore_service_webserver.payments.settings import ( PaymentsSettings, @@ -200,24 +202,27 @@ async def _pay( user_name: str, user_email: EmailStr, comment: str | None = None, - ): + ) -> PaymentTransaction: + assert await _get( app, payment_method_id=payment_method_id, user_id=user_id, wallet_id=wallet_id, ) - return await _init( + + return await _fake_pay_with_payment_method( app, - amount_dollars=amount_dollars, - target_credits=target_credits, - product_name=product_name, - wallet_id=wallet_id, - wallet_name=wallet_name, - user_id=user_id, - user_name=user_name, - user_email=user_email, - comment=comment, + amount_dollars, + target_credits, + product_name, + wallet_id, + wallet_name, + user_id, + user_name, + user_email, + payment_method_id, + comment, ) return { From c51c4a31ac66a9ba5dcaacec05c488ee58260542 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:37:14 +0100 Subject: [PATCH 39/78] minor --- .../simcore_service_webserver/payments/_rpc.py | 2 +- .../wallets/payments/test_payments_methods.py | 18 +----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/payments/_rpc.py b/services/web/server/src/simcore_service_webserver/payments/_rpc.py index 7b60475d501..9b758a4361d 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_rpc.py +++ b/services/web/server/src/simcore_service_webserver/payments/_rpc.py @@ -285,5 +285,5 @@ async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-a comment=comment, ) - assert isinstance(result, PaymentTransaction) + assert isinstance(result, PaymentTransaction) # nosec return result diff --git a/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py b/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py index 9b1fe418560..25a218204b6 100644 --- a/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py +++ b/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py @@ -26,15 +26,9 @@ from pytest_mock import MockerFixture from pytest_simcore.helpers.utils_assert import assert_status from simcore_postgres_database.models.payments_methods import InitPromptAckFlowState -from simcore_postgres_database.models.payments_transactions import ( - PaymentTransactionState, -) from simcore_service_webserver.payments._methods_api import ( _ack_creation_of_wallet_payment_method, ) -from simcore_service_webserver.payments._onetime_api import ( - _ack_creation_of_wallet_payment, -) from simcore_service_webserver.payments.settings import PaymentsSettings from simcore_service_webserver.payments.settings import ( get_plugin_settings as get_payments_plugin_settings, @@ -376,18 +370,8 @@ async def test_one_time_payment_with_payment_method( assert mock_rpc_payments_service_api["init_payment_with_payment_method"].called assert payment.payment_id - assert payment.payment_form_url - assert payment.payment_form_url.host == "some-fake-gateway.com" - assert payment.payment_form_url.query - assert payment.payment_form_url.query.endswith(payment.payment_id) + assert payment.payment_form_url is None - # Complete - await _ack_creation_of_wallet_payment( - client.app, - payment_id=payment.payment_id, - completion_state=PaymentTransactionState.SUCCESS, - invoice_url=faker.url(), - ) # check notification to RUT (fake) assert mock_rut_add_credits_to_wallet.called mock_rut_add_credits_to_wallet.assert_called_once() From f5fdd0a5751b4b5f4d27573f65c3e95f96cc3d32 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:40:31 +0100 Subject: [PATCH 40/78] rm todo --- .../payments/src/simcore_service_payments/services/payments.py | 2 -- .../src/simcore_service_webserver/wallets/_payments_handlers.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index f0d46d5108e..bb65f0d5619 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -260,8 +260,6 @@ async def pay_with_payment_method( # noqa: PLR0913 payment_method_id, user_id=user_id, wallet_id=wallet_id ) - # TODO: async with repo_transactions.begin(): should set a scope transaction inside - ack: AckPaymentWithPaymentMethod = await gateway.pay_with_payment_method( acked.payment_method_id, payment=InitPayment( diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 0495429419c..81b5f8064d3 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -347,8 +347,6 @@ async def _pay_with_payment_method(request: web.Request): price_dollars=body_params.price_dollars, ) - # TODO: how is front-end reacting? Should i simply trigger socketit from here as a background task?? - # NOTE: Due to the design change in https://github.com/ITISFoundation/osparc-simcore/pull/5017 # we decided not to change the return value to avoid changing the front-end logic # instead we emulate a init-prompt-ack workflow by firing a background task that acks payment From be1193ac1888cb707ca2df0d0ab1f753fcf51fd8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:10:39 +0100 Subject: [PATCH 41/78] minor test --- .../payments/_onetime_api.py | 12 +++++++++--- .../src/simcore_service_webserver/payments/api.py | 6 ++++-- .../wallets/_payments_handlers.py | 13 ++++++++++++- .../unit/with_dbs/03/wallets/payments/conftest.py | 4 ++-- .../03/wallets/payments/test_payments_methods.py | 4 ++-- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index d592348bf9a..a33be2d4b12 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -210,6 +210,7 @@ async def _ack_creation_of_wallet_payment( completion_state: PaymentTransactionState, message: str | None = None, invoice_url: HttpUrl | None = None, + nofity_enabled: bool = True, ) -> PaymentTransaction: # # NOTE: implements endpoint in payment service hit by the gateway @@ -231,7 +232,10 @@ async def _ack_creation_of_wallet_payment( payment = _to_api_model(transaction) # notifying front-end via web-sockets - await notify_payment_completed(app, user_id=transaction.user_id, payment=payment) + if nofity_enabled: + await notify_payment_completed( + app, user_id=transaction.user_id, payment=payment + ) if completion_state == PaymentTransactionState.SUCCESS: # notifying RUT @@ -302,7 +306,7 @@ async def _fake_pay_with_payment_method( assert user_name # nosec assert wallet_name # nosec - payment_inited = await _fake_init_payment( + inited = await _fake_init_payment( app, amount_dollars, target_credits, @@ -314,9 +318,11 @@ async def _fake_pay_with_payment_method( ) return await _ack_creation_of_wallet_payment( app, - payment_id=payment_inited.payment_id, + payment_id=inited.payment_id, completion_state=PaymentTransactionState.SUCCESS, message=f"Fake payment completed with payment-id = {payment_method_id}", + invoice_url=f"https:/fake-invoice.com/?id={inited.payment_id}", + nofity_enabled=False, ) diff --git a/services/web/server/src/simcore_service_webserver/payments/api.py b/services/web/server/src/simcore_service_webserver/payments/api.py index 0e3415457d6..171ebd5e574 100644 --- a/services/web/server/src/simcore_service_webserver/payments/api.py +++ b/services/web/server/src/simcore_service_webserver/payments/api.py @@ -15,17 +15,19 @@ list_user_payments_page, pay_with_payment_method, ) +from ._socketio import notify_payment_completed __all__: tuple[str, ...] = ( "cancel_creation_of_wallet_payment_method", "cancel_payment_to_wallet", - "init_creation_of_wallet_payment", "delete_wallet_payment_method", - "list_user_payments_page", "get_wallet_payment_autorecharge", "get_wallet_payment_method", "init_creation_of_wallet_payment_method", + "init_creation_of_wallet_payment", + "list_user_payments_page", "list_wallet_payment_methods", + "notify_payment_completed", "pay_with_payment_method", "replace_wallet_payment_autorecharge", ) diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 81b5f8064d3..c4cac0680f2 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -15,6 +15,7 @@ from models_library.rest_pagination import Page, PageQueryParameters from models_library.rest_pagination_utils import paginate_data from pydantic import parse_obj_as +from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from servicelib.aiohttp.requests_validation import ( parse_request_body_as, parse_request_path_parameters_as, @@ -22,6 +23,7 @@ ) from servicelib.logging_utils import get_log_record_extra, log_context from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON +from servicelib.utils import fire_and_forget_task from .._meta import API_VTAG as VTAG from ..login.decorators import login_required @@ -35,6 +37,7 @@ init_creation_of_wallet_payment_method, list_user_payments_page, list_wallet_payment_methods, + notify_payment_completed, pay_with_payment_method, replace_wallet_payment_autorecharge, ) @@ -350,7 +353,15 @@ async def _pay_with_payment_method(request: web.Request): # NOTE: Due to the design change in https://github.com/ITISFoundation/osparc-simcore/pull/5017 # we decided not to change the return value to avoid changing the front-end logic # instead we emulate a init-prompt-ack workflow by firing a background task that acks payment - # + + fire_and_forget_task( + notify_payment_completed( + request.app, user_id=req_ctx.user_id, payment=payment + ), + task_suffix_name=f"{__name__}._pay_with_payment_method", + fire_and_forget_tasks_collection=request.app[APP_FIRE_AND_FORGET_TASKS_KEY], + ) + return envelope_json_response( WalletPaymentInitiated( payment_id=payment.payment_id, diff --git a/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py b/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py index d8a571f17f5..51df97d8567 100644 --- a/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/wallets/payments/conftest.py @@ -261,8 +261,8 @@ async def _pay( autospec=True, side_effect=_del, ), - "init_payment_with_payment_method": mocker.patch( - "simcore_service_webserver.payments._onetime_api._rpc.init_payment_with_payment_method", + "pay_with_payment_method": mocker.patch( + "simcore_service_webserver.payments._onetime_api._rpc.pay_with_payment_method", autospec=True, side_effect=_pay, ), diff --git a/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py b/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py index 25a218204b6..c18ce0973dc 100644 --- a/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py +++ b/services/web/server/tests/unit/with_dbs/03/wallets/payments/test_payments_methods.py @@ -348,7 +348,7 @@ async def test_one_time_payment_with_payment_method( ) assert ( - client.app.router["init_payment_with_payment_method"] + client.app.router["pay_with_payment_method"] .url_for( wallet_id=f"{logged_user_wallet.wallet_id}", payment_method_id=wallet_payment_method_id, @@ -367,7 +367,7 @@ async def test_one_time_payment_with_payment_method( data, error = await assert_status(response, web.HTTPAccepted) assert error is None payment = WalletPaymentInitiated.parse_obj(data) - assert mock_rpc_payments_service_api["init_payment_with_payment_method"].called + assert mock_rpc_payments_service_api["pay_with_payment_method"].called assert payment.payment_id assert payment.payment_form_url is None From 2f295599f91ac198909134f1716fe065a119a1a4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:18:48 +0100 Subject: [PATCH 42/78] minor --- .../src/simcore_service_webserver/payments/_onetime_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index a33be2d4b12..352d7c8a2c4 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -289,7 +289,7 @@ async def cancel_payment_to_wallet( ) -async def _fake_pay_with_payment_method( +async def _fake_pay_with_payment_method( # noqa: PLR0913 pylint: disable=too-many-arguments app, amount_dollars, target_credits, @@ -321,7 +321,7 @@ async def _fake_pay_with_payment_method( payment_id=inited.payment_id, completion_state=PaymentTransactionState.SUCCESS, message=f"Fake payment completed with payment-id = {payment_method_id}", - invoice_url=f"https:/fake-invoice.com/?id={inited.payment_id}", + invoice_url=f"https:/fake-invoice.com/?id={inited.payment_id}", # type: ignore nofity_enabled=False, ) From b988268b13f9eb5247d8700af1e56dc9c935360b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 19:00:41 +0100 Subject: [PATCH 43/78] minor --- .../src/simcore_service_webserver/payments/_onetime_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index 352d7c8a2c4..de7026cc698 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -321,7 +321,7 @@ async def _fake_pay_with_payment_method( # noqa: PLR0913 pylint: disable=too-ma payment_id=inited.payment_id, completion_state=PaymentTransactionState.SUCCESS, message=f"Fake payment completed with payment-id = {payment_method_id}", - invoice_url=f"https:/fake-invoice.com/?id={inited.payment_id}", # type: ignore + invoice_url=f"https://fake-invoice.com/?id={inited.payment_id}", # type: ignore nofity_enabled=False, ) From b2ec59d06bfecf936274780b6a788b1752fe3e01 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 21:34:39 +0100 Subject: [PATCH 44/78] changes model of PaymentMethod --- .../api_schemas_webserver/wallets.py | 22 +++++++++---------- .../models/payments_gateway.py | 15 +++++-------- .../simcore_service_payments/models/utils.py | 3 --- 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py index eccbe6d774f..85a96b73e50 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/wallets.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/wallets.py @@ -124,14 +124,11 @@ class Config(OutputSchema.Config): class PaymentMethodGet(OutputSchema): idr: PaymentMethodID wallet_id: WalletID - card_holder_name: str - card_number_masked: str - card_type: str - expiration_month: int - expiration_year: int - street_address: str - zipcode: str - country: str + card_holder_name: str | None = None + card_number_masked: str | None = None + card_type: str | None = None + expiration_month: int | None = None + expiration_year: int | None = None created: datetime auto_recharge: bool = Field( default=False, @@ -149,12 +146,15 @@ class Config(OutputSchema.Config): "cardType": "Visa", "expirationMonth": 10, "expirationYear": 2025, - "streetAddress": "123 Main St", - "zipcode": "12345", - "country": "United States", "created": "2023-09-13T15:30:00Z", "autoRecharge": "False", }, + { + "idr": "pm_1234567890", + "walletId": 3, + "created": "2024-09-13T15:30:00Z", + "autoRecharge": "False", + }, ], } diff --git a/services/payments/src/simcore_service_payments/models/payments_gateway.py b/services/payments/src/simcore_service_payments/models/payments_gateway.py index b21c118d0f4..6614f67d303 100644 --- a/services/payments/src/simcore_service_payments/models/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/models/payments_gateway.py @@ -51,15 +51,12 @@ class PaymentMethodInitiated(BaseModel): class GetPaymentMethod(BaseModel): - idr: PaymentMethodID - card_holder_name: str - card_number_masked: str - card_type: str - expiration_month: int - expiration_year: int - street_address: str - zipcode: str - country: str + id: PaymentMethodID + card_holder_name: str | None = None + card_number_masked: str | None = None + card_type: str | None = None + expiration_month: int | None = None + expiration_year: int | None = None created: datetime diff --git a/services/payments/src/simcore_service_payments/models/utils.py b/services/payments/src/simcore_service_payments/models/utils.py index 3326ca1be08..af5cbfe537e 100644 --- a/services/payments/src/simcore_service_payments/models/utils.py +++ b/services/payments/src/simcore_service_payments/models/utils.py @@ -15,9 +15,6 @@ def merge_models(got: GetPaymentMethod, acked: PaymentsMethodsDB) -> PaymentMeth card_type=got.card_type, expiration_month=got.expiration_month, expiration_year=got.expiration_year, - street_address=got.street_address, - zipcode=got.zipcode, - country=got.country, created=acked.completed_at, auto_recharge=False, # this will be fileld in the web/server ) From 87d5a01c88dccfd1c8adcfd0563f77aa828c5a6b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 21:37:41 +0100 Subject: [PATCH 45/78] minor --- .../api/v0/openapi.yaml | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 99d850173fb..15d76e2571f 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -5256,8 +5256,11 @@ components: properties: priceDollars: title: Pricedollars - minimum: 0.0 + exclusiveMaximum: true + exclusiveMinimum: true type: number + maximum: 1000000.0 + minimum: 0.0 comment: title: Comment maxLength: 100 @@ -7546,14 +7549,6 @@ components: required: - idr - walletId - - cardHolderName - - cardNumberMasked - - cardType - - expirationMonth - - expirationYear - - streetAddress - - zipcode - - country - created type: object properties: @@ -7582,15 +7577,6 @@ components: expirationYear: title: Expirationyear type: integer - streetAddress: - title: Streetaddress - type: string - zipcode: - title: Zipcode - type: string - country: - title: Country - type: string created: title: Created type: string From 280aace943dac32ab0ea89bc3372cbcc2465a26e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 21:42:04 +0100 Subject: [PATCH 46/78] minor --- services/payments/tests/unit/test_services_payments_gateway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/payments/tests/unit/test_services_payments_gateway.py b/services/payments/tests/unit/test_services_payments_gateway.py index fd965244767..8a0b34f9350 100644 --- a/services/payments/tests/unit/test_services_payments_gateway.py +++ b/services/payments/tests/unit/test_services_payments_gateway.py @@ -160,7 +160,7 @@ async def test_payment_methods_workflow( got_payment_method = await payments_gateway_api.get_payment_method( payment_method_id ) - assert got_payment_method.idr == payment_method_id + assert got_payment_method.id == payment_method_id print(got_payment_method.json(indent=2)) # list payment-methods From 2b74a6902e4382bee42dff6686cac219cf896065 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:49:43 +0100 Subject: [PATCH 47/78] fix fixtue --- .../src/pytest_simcore/helpers/rawdata_fakers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py index 0b5564f9d20..b30b1820a5e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py @@ -251,15 +251,12 @@ def random_api_key(product_name: str, user_id: int, **overrides) -> dict[str, An def random_payment_method_data(**overrides) -> dict[str, Any]: # Produces data for GetPaymentMethod data = { - "idr": FAKE.uuid4(), + "id": FAKE.uuid4(), "card_holder_name": FAKE.name(), "card_number_masked": f"**** **** **** {FAKE.credit_card_number()[:4]}", "card_type": FAKE.credit_card_provider(), "expiration_month": FAKE.random_int(min=1, max=12), "expiration_year": FAKE.future_date().year, - "street_address": FAKE.street_address(), - "zipcode": FAKE.zipcode(), - "country": FAKE.country(), "created": utcnow(), } data.update(**overrides) From c9d6a8bef70c84af0f81dd3b0d47b70dc4ce1abe Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Nov 2023 22:51:46 +0100 Subject: [PATCH 48/78] adapt fxiture --- .../src/simcore_service_webserver/payments/_methods_api.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/payments/_methods_api.py b/services/web/server/src/simcore_service_webserver/payments/_methods_api.py index 354f87ae67b..c4b2b56309c 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_methods_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_methods_api.py @@ -40,15 +40,12 @@ def _generate_fake_data(fake: Faker): return { - "idr": fake.uuid4(), + "id": fake.uuid4(), "card_holder_name": fake.name(), "card_number_masked": f"**** **** **** {fake.credit_card_number()[:4]}", "card_type": fake.credit_card_provider(), "expiration_month": fake.random_int(min=1, max=12), "expiration_year": fake.future_date().year, - "street_address": fake.street_address(), - "zipcode": fake.zipcode(), - "country": fake.country(), } From d7bb6fffff98199f46522a752dc3ce8597084fd8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 16 Nov 2023 00:56:26 +0100 Subject: [PATCH 49/78] @sanderegg review: remvoes deprecated --- api/specs/web-server/_wallets.py | 2 +- .../api/rpc/_payments_methods.py | 39 ------------- .../services/payments.py | 58 ------------------- .../services/payments_gateway.py | 17 ------ .../payments/_onetime_api.py | 45 +++++--------- .../payments/_rpc.py | 41 ------------- 6 files changed, 15 insertions(+), 187 deletions(-) diff --git a/api/specs/web-server/_wallets.py b/api/specs/web-server/_wallets.py index 3b0222c2c55..9a0643073d2 100644 --- a/api/specs/web-server/_wallets.py +++ b/api/specs/web-server/_wallets.py @@ -169,7 +169,7 @@ async def delete_payment_method( "/wallets/{wallet_id}/payments-methods/{payment_method_id}:pay", response_model=Envelope[WalletPaymentInitiated], response_description="Pay with payment-method", - status_code=status.HTTP_202_ACCEPTED, # TODO: check in front-end + status_code=status.HTTP_202_ACCEPTED, ) async def pay_with_payment_method( wallet_id: WalletID, payment_method_id: PaymentMethodID, _body: CreateWalletPayment diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py index 81f485a417a..ef531a5b585 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py @@ -1,5 +1,4 @@ import logging -import warnings from decimal import Decimal from fastapi import FastAPI @@ -7,7 +6,6 @@ PaymentMethodGet, PaymentMethodID, PaymentMethodInitiated, - WalletPaymentInitiated, ) from models_library.basic_types import IDStr from models_library.users import UserID @@ -113,43 +111,6 @@ async def delete_payment_method( ) -@router.expose() -async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-arguments - app: FastAPI, - *, - payment_method_id: PaymentMethodID, - amount_dollars: Decimal, - target_credits: Decimal, - product_name: str, - wallet_id: WalletID, - wallet_name: str, - user_id: UserID, - user_name: str, - user_email: EmailStr, - comment: str | None = None, -) -> WalletPaymentInitiated: - warnings.warn( - f"{__name__}.init_payment_with_payment_method is deprecated. Use instead pay_with_payment_method", - DeprecationWarning, - stacklevel=1, - ) - return await payments.init_payment_with_payment_method( - gateway=PaymentsGatewayApi.get_from_app_state(app), - repo_transactions=PaymentsTransactionsRepo(db_engine=app.state.engine), - repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine), - payment_method_id=payment_method_id, - amount_dollars=amount_dollars, - target_credits=target_credits, - product_name=product_name, - wallet_id=wallet_id, - wallet_name=wallet_name, - user_id=user_id, - user_name=user_name, - user_email=user_email, - comment=comment, - ) - - @router.expose() async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-arguments app: FastAPI, diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index bb65f0d5619..5fa4f374c33 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -7,7 +7,6 @@ import logging import uuid -import warnings from decimal import Decimal import arrow @@ -181,63 +180,6 @@ async def on_payment_completed( ) -async def init_payment_with_payment_method( # noqa: PLR0913 - gateway: PaymentsGatewayApi, - repo_transactions: PaymentsTransactionsRepo, - repo_methods: PaymentsMethodsRepo, - *, - payment_method_id: PaymentMethodID, - amount_dollars: Decimal, - target_credits: Decimal, - product_name: str, - wallet_id: WalletID, - wallet_name: str, - user_id: UserID, - user_name: str, - user_email: EmailStr, - comment: str | None = None, -) -> WalletPaymentInitiated: - warnings.warn( - f"{__name__}.init_payment_with_payment_method is deprecated. Use instead pay_with_payment_method", - DeprecationWarning, - stacklevel=1, - ) - - initiated_at = arrow.utcnow().datetime - - acked = await repo_methods.get_payment_method( - payment_method_id, user_id=user_id, wallet_id=wallet_id - ) - - payment_inited = await gateway.init_payment_with_payment_method( - acked.payment_method_id, - payment=InitPayment( - amount_dollars=amount_dollars, - credits=target_credits, - user_name=user_name, - user_email=user_email, - wallet_name=wallet_name, - ), - ) - - payment_id = await repo_transactions.insert_init_payment_transaction( - payment_id=payment_inited.payment_id, - price_dollars=amount_dollars, - osparc_credits=target_credits, - product_name=product_name, - user_id=user_id, - user_email=user_email, - wallet_id=wallet_id, - comment=comment, - initiated_at=initiated_at, - ) - - return WalletPaymentInitiated( - payment_id=f"{payment_id}", - payment_form_url=None, - ) - - async def pay_with_payment_method( # noqa: PLR0913 gateway: PaymentsGatewayApi, repo_transactions: PaymentsTransactionsRepo, diff --git a/services/payments/src/simcore_service_payments/services/payments_gateway.py b/services/payments/src/simcore_service_payments/services/payments_gateway.py index 233ee860e57..c9d4174e054 100644 --- a/services/payments/src/simcore_service_payments/services/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/services/payments_gateway.py @@ -8,7 +8,6 @@ import contextlib import functools import logging -import warnings from collections.abc import Callable from contextlib import suppress @@ -161,22 +160,6 @@ async def delete_payment_method(self, id_: PaymentMethodID) -> None: response = await self.client.delete(f"/payment-methods/{id_}") response.raise_for_status() - @_handle_status_errors - async def init_payment_with_payment_method( - self, id_: PaymentMethodID, payment: InitPayment - ) -> PaymentInitiated: - warnings.warn( - f"{__name__}.init_payment_with_payment_method is deprecated. Use instead pay_with_payment_method", - DeprecationWarning, - stacklevel=1, - ) - response = await self.client.post( - f"/payment-methods/{id_}:pay", - json=jsonable_encoder(payment), - ) - response.raise_for_status() - return PaymentInitiated.parse_obj(response.json()) - @_handle_status_errors async def pay_with_payment_method( self, id_: PaymentMethodID, payment: InitPayment diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index de7026cc698..6e96a891f41 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -135,7 +135,6 @@ async def init_creation_of_wallet_payment( user_id: UserID, wallet_id: WalletID, comment: str | None, - payment_method_id: PaymentMethodID | None = None, ) -> WalletPaymentInitiated: """ @@ -157,38 +156,22 @@ async def init_creation_of_wallet_payment( user = await get_user_name_and_email(app, user_id=user_id) settings: PaymentsSettings = get_plugin_settings(app) payment_inited: WalletPaymentInitiated - if payment_method_id is None: - if settings.PAYMENTS_FAKE_COMPLETION: - payment_inited = await _fake_init_payment( - app, - price_dollars, - osparc_credits, - product_name, - wallet_id, - user_id, - user.email, - comment, - ) - else: - # call to payment-service - assert not settings.PAYMENTS_FAKE_COMPLETION # nosec - payment_inited = await _rpc.init_payment( - app, - amount_dollars=price_dollars, - target_credits=osparc_credits, - product_name=product_name, - wallet_id=wallet_id, - wallet_name=user_wallet.name, - user_id=user_id, - user_name=user.name, - user_email=user.email, - comment=comment, - ) + if settings.PAYMENTS_FAKE_COMPLETION: + payment_inited = await _fake_init_payment( + app, + price_dollars, + osparc_credits, + product_name, + wallet_id, + user_id, + user.email, + comment, + ) else: - assert payment_method_id is not None # nosec - payment_inited = await _rpc.init_payment_with_payment_method( + # call to payment-service + assert not settings.PAYMENTS_FAKE_COMPLETION # nosec + payment_inited = await _rpc.init_payment( app, - payment_method_id=payment_method_id, amount_dollars=price_dollars, target_credits=osparc_credits, product_name=product_name, diff --git a/services/web/server/src/simcore_service_webserver/payments/_rpc.py b/services/web/server/src/simcore_service_webserver/payments/_rpc.py index 9b758a4361d..e5c0d2680a3 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_rpc.py +++ b/services/web/server/src/simcore_service_webserver/payments/_rpc.py @@ -3,7 +3,6 @@ """ import logging -import warnings from decimal import Decimal from aiohttp import web @@ -212,46 +211,6 @@ async def delete_payment_method( assert result is None # nosec -@log_decorator(_logger, level=logging.DEBUG) -async def init_payment_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-arguments - app: web.Application, - *, - payment_method_id: PaymentMethodID, - amount_dollars: Decimal, - target_credits: Decimal, - product_name: str, - wallet_id: WalletID, - wallet_name: str, - user_id: UserID, - user_name: str, - user_email: EmailStr, - comment: str | None = None, -) -> WalletPaymentInitiated: - warnings.warn( - f"{__name__}.init_payment_with_payment_method is deprecated. Use instead pay_with_payment_method", - DeprecationWarning, - stacklevel=1, - ) - rpc_client = app[_APP_PAYMENTS_RPC_CLIENT_KEY] - - result = await rpc_client.request( - PAYMENTS_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "init_payment_with_payment_method"), - payment_method_id=payment_method_id, - amount_dollars=amount_dollars, - target_credits=target_credits, - product_name=product_name, - wallet_id=wallet_id, - wallet_name=wallet_name, - user_id=user_id, - user_name=user_name, - user_email=user_email, - comment=comment, - ) - assert isinstance(result, WalletPaymentInitiated) # nosec - return result - - @log_decorator(_logger, level=logging.DEBUG) async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-arguments app: web.Application, From e1b43e70217e72eecadfd395efe5ce5f3f7ba7da Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 16 Nov 2023 00:59:07 +0100 Subject: [PATCH 50/78] @sanderegg review: typo --- .../src/simcore_service_webserver/payments/_onetime_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index 6e96a891f41..0c460fb49e8 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -193,7 +193,7 @@ async def _ack_creation_of_wallet_payment( completion_state: PaymentTransactionState, message: str | None = None, invoice_url: HttpUrl | None = None, - nofity_enabled: bool = True, + nofify_enabled: bool = True, ) -> PaymentTransaction: # # NOTE: implements endpoint in payment service hit by the gateway @@ -215,7 +215,7 @@ async def _ack_creation_of_wallet_payment( payment = _to_api_model(transaction) # notifying front-end via web-sockets - if nofity_enabled: + if nofify_enabled: await notify_payment_completed( app, user_id=transaction.user_id, payment=payment ) @@ -305,7 +305,7 @@ async def _fake_pay_with_payment_method( # noqa: PLR0913 pylint: disable=too-ma completion_state=PaymentTransactionState.SUCCESS, message=f"Fake payment completed with payment-id = {payment_method_id}", invoice_url=f"https://fake-invoice.com/?id={inited.payment_id}", # type: ignore - nofity_enabled=False, + nofify_enabled=False, ) From d644d1974186724327db4c23d51cdaef5c3845fe Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 16 Nov 2023 01:06:45 +0100 Subject: [PATCH 51/78] fixes bad fixture --- .../src/pytest_simcore/helpers/rawdata_fakers.py | 1 + services/payments/tests/unit/conftest.py | 2 +- .../src/simcore_service_webserver/payments/_onetime_api.py | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py index b30b1820a5e..036d0a9c126 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py @@ -259,5 +259,6 @@ def random_payment_method_data(**overrides) -> dict[str, Any]: "expiration_year": FAKE.future_date().year, "created": utcnow(), } + assert set(overrides.keys()).issubset(data.keys()) data.update(**overrides) return data diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index f844478a7b6..086f6052d98 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -215,7 +215,7 @@ def _init(request: httpx.Request): pm_id = faker.uuid4() _payment_methods[pm_id] = PaymentMethodInfoTuple( init=InitPaymentMethod.parse_raw(request.content), - get=GetPaymentMethod(**random_payment_method_data(idr=pm_id)), + get=GetPaymentMethod(**random_payment_method_data(id=pm_id)), ) return httpx.Response( diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index 0c460fb49e8..00ce812f7fa 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -193,7 +193,7 @@ async def _ack_creation_of_wallet_payment( completion_state: PaymentTransactionState, message: str | None = None, invoice_url: HttpUrl | None = None, - nofify_enabled: bool = True, + notify_enabled: bool = True, ) -> PaymentTransaction: # # NOTE: implements endpoint in payment service hit by the gateway @@ -215,7 +215,7 @@ async def _ack_creation_of_wallet_payment( payment = _to_api_model(transaction) # notifying front-end via web-sockets - if nofify_enabled: + if notify_enabled: await notify_payment_completed( app, user_id=transaction.user_id, payment=payment ) @@ -305,7 +305,7 @@ async def _fake_pay_with_payment_method( # noqa: PLR0913 pylint: disable=too-ma completion_state=PaymentTransactionState.SUCCESS, message=f"Fake payment completed with payment-id = {payment_method_id}", invoice_url=f"https://fake-invoice.com/?id={inited.payment_id}", # type: ignore - nofify_enabled=False, + notify_enabled=False, ) From ad425ca51be1e12dcfe8bea7860605dfa3470f2e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 16 Nov 2023 01:49:08 +0100 Subject: [PATCH 52/78] updates info on CC --- .../osparc/desktop/paymentMethods/PaymentMethodDetails.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js index 80fbd659e38..30f2bba4623 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js +++ b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js @@ -51,9 +51,6 @@ qx.Class.define("osparc.desktop.paymentMethods.PaymentMethodDetails", { [this.tr("Type"), paymentMethodData["cardType"]], [this.tr("Number"), paymentMethodData["cardNumberMasked"]], [this.tr("Expiration date"), paymentMethodData["expirationMonth"] + "/" + paymentMethodData["expirationYear"]], - [this.tr("Address"), paymentMethodData["streetAddress"]], - [this.tr("ZIP code"), paymentMethodData["zipcode"]], - [this.tr("Country"), paymentMethodData["country"]] ].forEach((pair, idx) => { this._add(new qx.ui.basic.Label(pair[0]).set({ font: "text-14" From c698c6b1bac79246f5e8e53d0e3f97c39fff8939 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 16 Nov 2023 01:57:52 +0100 Subject: [PATCH 53/78] fe --- .../class/osparc/desktop/paymentMethods/PaymentMethodDetails.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js index 30f2bba4623..49dbab989ed 100644 --- a/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js +++ b/services/static-webserver/client/source/class/osparc/desktop/paymentMethods/PaymentMethodDetails.js @@ -50,7 +50,7 @@ qx.Class.define("osparc.desktop.paymentMethods.PaymentMethodDetails", { [this.tr("Holder name"), paymentMethodData["cardHolderName"]], [this.tr("Type"), paymentMethodData["cardType"]], [this.tr("Number"), paymentMethodData["cardNumberMasked"]], - [this.tr("Expiration date"), paymentMethodData["expirationMonth"] + "/" + paymentMethodData["expirationYear"]], + [this.tr("Expiration date"), paymentMethodData["expirationMonth"] + "/" + paymentMethodData["expirationYear"]] ].forEach((pair, idx) => { this._add(new qx.ui.basic.Label(pair[0]).set({ font: "text-14" From 105af222ab6086fc717faa905fdde3cfb2422839 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Sun, 12 Nov 2023 15:02:59 +0100 Subject: [PATCH 54/78] daily work --- .../api/rpc/_payments_methods.py | 2 +- .../simcore_service_payments/core/errors.py | 4 + .../db/auto_recharge_repo.py | 76 ++++++++++ .../db/payments_methods_repo.py | 4 +- .../services/auto_recharge.py | 137 ++++++++++++++++++ .../services/auto_recharge_process_message.py | 42 +++++- .../services/payments_methods.py | 4 +- .../unit/test_db_payments_methods_repo.py | 4 +- 8 files changed, 264 insertions(+), 9 deletions(-) create mode 100644 services/payments/src/simcore_service_payments/db/auto_recharge_repo.py create mode 100644 services/payments/src/simcore_service_payments/services/auto_recharge.py diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py index ef531a5b585..5e44f2ac917 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py @@ -69,7 +69,7 @@ async def list_payment_methods( user_id: UserID, wallet_id: WalletID, ) -> list[PaymentMethodGet]: - return await payments_methods.list_payments_methods( + return await payments_methods.list_successful_payment_methods( gateway=PaymentsGatewayApi.get_from_app_state(app), repo=PaymentsMethodsRepo(db_engine=app.state.engine), user_id=user_id, diff --git a/services/payments/src/simcore_service_payments/core/errors.py b/services/payments/src/simcore_service_payments/core/errors.py index d7133697760..1bc228f173f 100644 --- a/services/payments/src/simcore_service_payments/core/errors.py +++ b/services/payments/src/simcore_service_payments/core/errors.py @@ -33,3 +33,7 @@ class PaymentMethodAlreadyAckedError(PaymentsMethodsError): class PaymentMethodUniqueViolationError(PaymentsMethodsError): msg_template = "Payment method '{payment_method_id}' aready exists" + + +class InvalidPaymentMethodError(PaymentsMethodsError): + msg_template = "Invalid payment method '{payment_method_id}'" diff --git a/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py b/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py new file mode 100644 index 00000000000..53743a8fa8d --- /dev/null +++ b/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py @@ -0,0 +1,76 @@ +from typing import TypeAlias + +from models_library.api_schemas_webserver.wallets import PaymentMethodID +from models_library.basic_types import NonNegativeDecimal +from models_library.users import UserID +from models_library.wallets import WalletID +from pydantic import BaseModel, PositiveInt +from simcore_postgres_database.utils_payments_autorecharge import AutoRechargeStmts + +from ..core.errors import InvalidPaymentMethodError +from .base import BaseRepository + +AutoRechargeID: TypeAlias = PositiveInt + + +class PaymentsAutorechargeDB(BaseModel): + wallet_id: WalletID + enabled: bool + primary_payment_method_id: PaymentMethodID + top_up_amount_in_usd: NonNegativeDecimal + monthly_limit_in_usd: NonNegativeDecimal | None + + class Config: + orm_mode = True + + +class AutoRechargeRepo(BaseRepository): + async def get_wallet_autorecharge( + self, + wallet_id: WalletID, + ) -> PaymentsAutorechargeDB | None: + """Annotates init-payment transaction + Raises: + PaymentAlreadyExistsError + """ + + async with self.db_engine.begin() as conn: + stmt = AutoRechargeStmts.get_wallet_autorecharge(wallet_id) + result = await conn.execute(stmt) + row = await result.first() + return PaymentsAutorechargeDB.from_orm(row) if row else None + + async def replace_wallet_autorecharge( + self, + user_id: UserID, + wallet_id: WalletID, + new: PaymentsAutorechargeDB, + ) -> PaymentsAutorechargeDB: + """ + Raises: + InvalidPaymentMethodError: if `new` includes some invalid 'primary_payment_method_id' + + """ + async with self.db_engine.begin() as conn: + stmt = AutoRechargeStmts.is_valid_payment_method( + user_id=user_id, + wallet_id=new.wallet_id, + payment_method_id=new.primary_payment_method_id, + ) + + if await conn.scalar(stmt) != new.primary_payment_method_id: + raise InvalidPaymentMethodError( + payment_method_id=new.primary_payment_method_id + ) + + stmt = AutoRechargeStmts.upsert_wallet_autorecharge( + wallet_id=wallet_id, + enabled=new.enabled, + primary_payment_method_id=new.primary_payment_method_id, + top_up_amount_in_usd=new.top_up_amount_in_usd, + monthly_limit_in_usd=new.monthly_limit_in_usd, + ) + result = await conn.execute(stmt) + row = await result.first() + assert row # nosec + return PaymentsAutorechargeDB.from_orm(row) diff --git a/services/payments/src/simcore_service_payments/db/payments_methods_repo.py b/services/payments/src/simcore_service_payments/db/payments_methods_repo.py index 26a901abbfd..105a6a704e5 100644 --- a/services/payments/src/simcore_service_payments/db/payments_methods_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_methods_repo.py @@ -115,7 +115,7 @@ async def insert_payment_method( state_message=state_message, ) - async def list_user_payment_methods( + async def list_user_successful_payment_methods( self, *, user_id: UserID, @@ -135,7 +135,7 @@ async def list_user_payment_methods( rows = result.fetchall() or [] return parse_obj_as(list[PaymentsMethodsDB], rows) - async def get_payment_method( + async def get_successful_payment_method_( self, payment_method_id: PaymentMethodID, *, diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge.py b/services/payments/src/simcore_service_payments/services/auto_recharge.py new file mode 100644 index 00000000000..3cc53c1b8b3 --- /dev/null +++ b/services/payments/src/simcore_service_payments/services/auto_recharge.py @@ -0,0 +1,137 @@ +import logging + +from fastapi import FastAPI +from models_library.api_schemas_webserver.wallets import ( + GetWalletAutoRecharge, + ReplaceWalletAutoRecharge, +) +from models_library.basic_types import NonNegativeDecimal +from models_library.products import ProductName +from models_library.users import UserID +from models_library.wallets import WalletID + +# from ._methods_db import list_successful_payment_methods +# from ._onetime_api import raise_for_wallet_payments_permissions +from ..core.settings import ApplicationSettings +from ..db.auto_recharge_repo import AutoRechargeRepo, PaymentsAutorechargeDB +from ..db.payments_methods_repo import PaymentsMethodsRepo + +_logger = logging.getLogger(__name__) + + +def _from_db_to_api_model( + db_model: PaymentsAutorechargeDB, min_balance_in_credits: NonNegativeDecimal +) -> GetWalletAutoRecharge: + return GetWalletAutoRecharge( + enabled=db_model.enabled, + payment_method_id=db_model.primary_payment_method_id, + min_balance_in_credits=min_balance_in_credits, + top_up_amount_in_usd=db_model.top_up_amount_in_usd, + monthly_limit_in_usd=db_model.monthly_limit_in_usd, + ) + + +def _from_api_to_db_model( + wallet_id: WalletID, api_model: ReplaceWalletAutoRecharge +) -> PaymentsAutorechargeDB: + return PaymentsAutorechargeDB( + wallet_id=wallet_id, + enabled=api_model.enabled, + primary_payment_method_id=api_model.payment_method_id, + top_up_amount_in_usd=api_model.top_up_amount_in_usd, + monthly_limit_in_usd=api_model.monthly_limit_in_usd, + ) + + +# +# payment-autorecharge api +# + +_NEWEST = 0 + + +async def get_wallet_payment_autorecharge( + settings: ApplicationSettings, + auto_recharge_repo: AutoRechargeRepo, + *, + wallet_id: WalletID, +) -> GetWalletAutoRecharge | None: + # TODO: Permissions will be checked in webserver! + payments_autorecharge_db: PaymentsAutorechargeDB | None = ( + await auto_recharge_repo.get_wallet_autorecharge(wallet_id=wallet_id) + ) + if payments_autorecharge_db: + return GetWalletAutoRecharge( + enabled=payments_autorecharge_db.enabled, + payment_method_id=payments_autorecharge_db.primary_payment_method_id, + min_balance_in_credits=100.0, # TODO: take from settings + top_up_amount_in_usd=payments_autorecharge_db.top_up_amount_in_usd, + monthly_limit_in_usd=payments_autorecharge_db.monthly_limit_in_usd, + ) + return None + + +async def get_wallet_payment_autorecharge_with_default( + app: FastAPI, + auto_recharge_repo: AutoRechargeRepo, + payments_method_repo: PaymentsMethodsRepo, + *, + product_name: ProductName, + user_id: UserID, + wallet_id: WalletID, +) -> GetWalletAutoRecharge: + # TODO: Ask Pedro + # await raise_for_wallet_payments_permissions( + # app, user_id=user_id, wallet_id=wallet_id, product_name=product_name + # ) + wallet_autorecharge = await get_wallet_payment_autorecharge( + app, + auto_recharge_repo, + wallet_id=wallet_id, + ) + if not wallet_autorecharge: + payment_method_id = None + wallet_payment_methods = ( + await payments_method_repo.list_user_successful_payment_methods( + user_id=user_id, + wallet_id=wallet_id, + ) + ) + if wallet_payment_methods: + payment_method_id = wallet_payment_methods[_NEWEST].payment_method_id + + return GetWalletAutoRecharge( + enabled=False, + payment_method_id=payment_method_id, + min_balance_in_credits=settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS, + top_up_amount_in_usd=settings.PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT, + monthly_limit_in_usd=settings.PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT, + ) + + return wallet_autorecharge + + +async def replace_wallet_payment_autorecharge( + app: FastAPI, + repo: AutoRechargeRepo, + *, + product_name: ProductName, + user_id: UserID, + wallet_id: WalletID, + new: ReplaceWalletAutoRecharge, +) -> GetWalletAutoRecharge: + # TODO: Ask Pedro + # await raise_for_wallet_payments_permissions( + # app, user_id=user_id, wallet_id=wallet_id, product_name=product_name + # ) + settings: ApplicationSettings = app.state.settings + got: PaymentsAutorechargeDB = await repo.replace_wallet_autorecharge( + user_id=user_id, + wallet_id=wallet_id, + new=_from_api_to_db_model(wallet_id, new), + ) + + return _from_db_to_api_model( + got, + min_balance_in_credits=settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS, + ) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index 8480063b19b..7f0225d0148 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -1,8 +1,17 @@ import logging +from decimal import Decimal from fastapi import FastAPI +from models_library.api_schemas_webserver.wallets import GetWalletAutoRecharge from models_library.rabbitmq_messages import WalletCreditsMessage from pydantic import parse_raw_as +from simcore_service_payments.db.auto_recharge_repo import AutoRechargeRepo +from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo +from simcore_service_payments.services.payments_gateway import PaymentsGatewayApi + +from ..core.settings import ApplicationSettings +from .auto_recharge import get_wallet_payment_autorecharge +from .payments_methods import get_payment_method _logger = logging.getLogger(__name__) @@ -12,9 +21,38 @@ async def process_message(app: FastAPI, data: bytes) -> bool: rabbit_message = parse_raw_as(WalletCreditsMessage, data) _logger.debug("Process msg: %s", rabbit_message) - # 1. Check if auto-recharge functionality is ON for wallet_id - # 2. Check if wallet credits are bellow the threshold + settings: ApplicationSettings = app.state.settings + auto_recharge_repo: AutoRechargeRepo = AutoRechargeRepo(db_engine=app.state.engine) + + # 1. Check if wallet credits are bellow the threshold + if rabbit_message.credits > Decimal(100): # TODO: from settings MIN CREDITS + return True + + # 2. Check if auto-recharge functionality is ON for wallet_id + payment_autorecharge: GetWalletAutoRecharge | None = ( + await get_wallet_payment_autorecharge( + settings, auto_recharge_repo, wallet_id=rabbit_message.wallet_id + ) + ) + if ( + payment_autorecharge is None + or payment_autorecharge.enabled is False + or payment_autorecharge.payment_method_id is None + ): + return True + # 3. Get Payment method + _payments_gateway = PaymentsGatewayApi.get_from_app_state(app) + _payments_repo = PaymentsMethodsRepo(db_engine=app.state.engine) + await get_payment_method( + _payments_gateway, + _payments_repo, + payment_method_id=payment_autorecharge.payment_method_id, + user_id=1, # TODO: Why user_id/wallet_id is here? I would query purly based on payment_method_id + wallet_id=rabbit_message.wallet_id, + ) + # 4. Pay with payment method + # TODO: I will need to query webserver for user email & wallet name? return True diff --git a/services/payments/src/simcore_service_payments/services/payments_methods.py b/services/payments/src/simcore_service_payments/services/payments_methods.py index bbfef17dab2..a4c1b980810 100644 --- a/services/payments/src/simcore_service_payments/services/payments_methods.py +++ b/services/payments/src/simcore_service_payments/services/payments_methods.py @@ -148,14 +148,14 @@ async def create_payment_method( ) -async def list_payments_methods( +async def list_successful_payment_methods( gateway: PaymentsGatewayApi, repo: PaymentsMethodsRepo, *, user_id: UserID, wallet_id: WalletID, ) -> list[PaymentMethodGet]: - acked_many = await repo.list_user_payment_methods( + acked_many = await repo.list_user_successful_payment_methods( user_id=user_id, wallet_id=wallet_id ) assert not any(acked.completed_at is None for acked in acked_many) # nosec diff --git a/services/payments/tests/unit/test_db_payments_methods_repo.py b/services/payments/tests/unit/test_db_payments_methods_repo.py index 76166c5d0be..ef876845b54 100644 --- a/services/payments/tests/unit/test_db_payments_methods_repo.py +++ b/services/payments/tests/unit/test_db_payments_methods_repo.py @@ -65,7 +65,7 @@ async def test_create_payments_method_annotations_workflow(app: FastAPI): ) # list - listed = await repo.list_user_payment_methods( + listed = await repo.list_user_successful_payment_methods( user_id=fake.user_id, wallet_id=fake.wallet_id, ) @@ -88,7 +88,7 @@ async def test_create_payments_method_annotations_workflow(app: FastAPI): ) assert deleted == got - listed = await repo.list_user_payment_methods( + listed = await repo.list_user_successful_payment_methods( user_id=fake.user_id, wallet_id=fake.wallet_id, ) From 83f05669a97e4d5bf157146d4922ae2ac809bfc8 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 13 Nov 2023 18:40:56 +0100 Subject: [PATCH 55/78] daily work --- services/docker-compose.yml | 3 +++ .../simcore_service_payments/core/settings.py | 16 ++++++++++++ .../db/payments_methods_repo.py | 20 ++++++++++++-- .../services/auto_recharge.py | 15 +++++------ .../services/payments_methods.py | 13 ++++++++++ .../services/rabbitmq.py | 11 ++++++++ services/web/server/setup.cfg | 2 +- .../products/_api.py | 9 +++++++ .../products/_rpc.py | 26 +++++++++++++++++++ .../products/plugin.py | 8 +++++- 10 files changed, 110 insertions(+), 13 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/products/_rpc.py diff --git a/services/docker-compose.yml b/services/docker-compose.yml index e8f63bbaaf7..4e0e9b23c23 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -238,6 +238,9 @@ services: - LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED} - PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES=${PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES} - PAYMENTS_ACCESS_TOKEN_SECRET_KEY=${PAYMENTS_ACCESS_TOKEN_SECRET_KEY} + - PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT=${PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT} + - PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT=${PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT} + - PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS=${PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS} - PAYMENTS_GATEWAY_API_SECRET=${PAYMENTS_GATEWAY_API_SECRET} - PAYMENTS_GATEWAY_URL=${PAYMENTS_GATEWAY_URL} - PAYMENTS_LOGLEVEL=${PAYMENTS_LOGLEVEL} diff --git a/services/payments/src/simcore_service_payments/core/settings.py b/services/payments/src/simcore_service_payments/core/settings.py index f0632fef47c..aaf70383e5c 100644 --- a/services/payments/src/simcore_service_payments/core/settings.py +++ b/services/payments/src/simcore_service_payments/core/settings.py @@ -1,6 +1,7 @@ from functools import cached_property from typing import cast +from models_library.basic_types import NonNegativeDecimal from pydantic import Field, HttpUrl, PositiveFloat, SecretStr, parse_obj_as, validator from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, VersionTag @@ -77,6 +78,21 @@ class ApplicationSettings(_BaseApplicationSettings): ) PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES: PositiveFloat = Field(default=30) + PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: NonNegativeDecimal = Field( + default=100, + description="Minimum balance in credits to top-up for auto-recharge", + ) + + PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT: NonNegativeDecimal = Field( + default=100, + description="Default value in USD on the amount to top-up for auto-recharge (`top_up_amount_in_usd`)", + ) + + PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT: NonNegativeDecimal | None = Field( + default=10000, + description="Default value in USD for the montly limit for auto-recharge (`monthly_limit_in_usd`)", + ) + PAYMENTS_RABBITMQ: RabbitSettings = Field( auto_default_from_env=True, description="settings for service/rabbitmq" ) diff --git a/services/payments/src/simcore_service_payments/db/payments_methods_repo.py b/services/payments/src/simcore_service_payments/db/payments_methods_repo.py index 105a6a704e5..71fcfd350ef 100644 --- a/services/payments/src/simcore_service_payments/db/payments_methods_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_methods_repo.py @@ -102,7 +102,6 @@ async def insert_payment_method( completion_state: InitPromptAckFlowState, state_message: str | None, ) -> PaymentsMethodsDB: - await self.insert_init_payment_method( payment_method_id, user_id=user_id, @@ -135,7 +134,24 @@ async def list_user_successful_payment_methods( rows = result.fetchall() or [] return parse_obj_as(list[PaymentsMethodsDB], rows) - async def get_successful_payment_method_( + async def get_successful_payment_method_by_id( + self, + payment_method_id: PaymentMethodID, + ) -> PaymentsMethodsDB: + async with self.db_engine.begin() as conn: + result = await conn.execute( + payments_methods.select().where( + (payments_methods.c.payment_method_id == payment_method_id) + & (payments_methods.c.state == InitPromptAckFlowState.SUCCESS) + ) + ) + row = result.first() + if row is None: + raise PaymentMethodNotFoundError(payment_method_id=payment_method_id) + + return PaymentsMethodsDB.from_orm(row) + + async def get_successful_payment_method( self, payment_method_id: PaymentMethodID, *, diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge.py b/services/payments/src/simcore_service_payments/services/auto_recharge.py index 3cc53c1b8b3..71bf61a352a 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge.py @@ -50,7 +50,7 @@ def _from_api_to_db_model( _NEWEST = 0 -async def get_wallet_payment_autorecharge( +async def get_wallet_auto_recharge( settings: ApplicationSettings, auto_recharge_repo: AutoRechargeRepo, *, @@ -64,7 +64,7 @@ async def get_wallet_payment_autorecharge( return GetWalletAutoRecharge( enabled=payments_autorecharge_db.enabled, payment_method_id=payments_autorecharge_db.primary_payment_method_id, - min_balance_in_credits=100.0, # TODO: take from settings + min_balance_in_credits=settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS, top_up_amount_in_usd=payments_autorecharge_db.top_up_amount_in_usd, monthly_limit_in_usd=payments_autorecharge_db.monthly_limit_in_usd, ) @@ -80,12 +80,10 @@ async def get_wallet_payment_autorecharge_with_default( user_id: UserID, wallet_id: WalletID, ) -> GetWalletAutoRecharge: - # TODO: Ask Pedro - # await raise_for_wallet_payments_permissions( - # app, user_id=user_id, wallet_id=wallet_id, product_name=product_name - # ) - wallet_autorecharge = await get_wallet_payment_autorecharge( - app, + settings: ApplicationSettings = app.state.settings + + wallet_autorecharge = await get_wallet_auto_recharge( + settings, auto_recharge_repo, wallet_id=wallet_id, ) @@ -107,7 +105,6 @@ async def get_wallet_payment_autorecharge_with_default( top_up_amount_in_usd=settings.PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT, monthly_limit_in_usd=settings.PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT, ) - return wallet_autorecharge diff --git a/services/payments/src/simcore_service_payments/services/payments_methods.py b/services/payments/src/simcore_service_payments/services/payments_methods.py index a4c1b980810..e4cc5821501 100644 --- a/services/payments/src/simcore_service_payments/services/payments_methods.py +++ b/services/payments/src/simcore_service_payments/services/payments_methods.py @@ -170,6 +170,19 @@ async def list_successful_payment_methods( ] +async def get_payment_method_by_id( + gateway: PaymentsGatewayApi, + repo: PaymentsMethodsRepo, + *, + payment_method_id: PaymentMethodID, +) -> PaymentMethodGet: + acked = await repo.get_successful_payment_method_by_id(payment_method_id) + assert acked.state == InitPromptAckFlowState.SUCCESS # nosec + + got: GetPaymentMethod = await gateway.get_payment_method(acked.payment_method_id) + return merge_models(got, acked) + + async def get_payment_method( gateway: PaymentsGatewayApi, repo: PaymentsMethodsRepo, diff --git a/services/payments/src/simcore_service_payments/services/rabbitmq.py b/services/payments/src/simcore_service_payments/services/rabbitmq.py index 58facd39868..8938b52cbc2 100644 --- a/services/payments/src/simcore_service_payments/services/rabbitmq.py +++ b/services/payments/src/simcore_service_payments/services/rabbitmq.py @@ -27,6 +27,9 @@ async def _on_startup() -> None: app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( client_name="payments_rpc_server", settings=settings ) + app.state.rabbitmq_rpc_client = await RabbitMQRPCClient.create( + client_name="payments_rpc_client", settings=settings + ) async def _on_shutdown() -> None: if app.state.rabbitmq_client: @@ -35,6 +38,9 @@ async def _on_shutdown() -> None: if app.state.rabbitmq_rpc_server: await app.state.rabbitmq_rpc_server.close() app.state.rabbitmq_rpc_server = None + if app.state.rabbitmq_rpc_client: + await app.state.rabbitmq_rpc_client.close() + app.state.rabbitmq_rpc_client = None app.add_event_handler("startup", _on_startup) app.add_event_handler("shutdown", _on_shutdown) @@ -50,5 +56,10 @@ def get_rabbitmq_rpc_server(app: FastAPI) -> RabbitMQRPCClient: return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_server) +def get_rabbitmq_rpc_client(app: FastAPI) -> RabbitMQRPCClient: + assert app.state.rabbitmq_rpc_client # nosec + return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_client) + + async def post_message(app: FastAPI, message: RabbitMessageBase) -> None: await get_rabbitmq_client(app).publish(message.channel_name, message) diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 91dfe674a06..9e00e568cd6 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -12,7 +12,7 @@ commit_args = --no-verify [tool:pytest] addopts = --strict-markers asyncio_mode = auto -markers = +markers = slow: marks tests as slow (deselect with '-m "not slow"') acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." testit: "marks test to run during development" diff --git a/services/web/server/src/simcore_service_webserver/products/_api.py b/services/web/server/src/simcore_service_webserver/products/_api.py index 9368627c17a..6e0498dc2df 100644 --- a/services/web/server/src/simcore_service_webserver/products/_api.py +++ b/services/web/server/src/simcore_service_webserver/products/_api.py @@ -49,6 +49,15 @@ async def get_current_product_credit_price( return await repo.get_product_latest_credit_price_or_none(current_product_name) +async def get_product_credit_price_by_app_and_product( + app: web.Application, *, product_name: ProductName +): + repo = ProductRepository.create_from_app(app) + return await repo.get_product_latest_credit_price_or_none(product_name) + + +# Get conversion! send already money -> get credits + # # helpers for get_product_template_path # diff --git a/services/web/server/src/simcore_service_webserver/products/_rpc.py b/services/web/server/src/simcore_service_webserver/products/_rpc.py new file mode 100644 index 00000000000..75f92ccbe2a --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/products/_rpc.py @@ -0,0 +1,26 @@ +from aiohttp import web +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.basic_types import NonNegativeDecimal +from models_library.products import ProductName +from servicelib.rabbitmq import RPCRouter + +from ..rabbitmq import get_rabbitmq_rpc_server +from . import _api + +router = RPCRouter() + + +@router.expose() +async def get_product_credit_price_by_app_and_product( + app: web.Application, + *, + product_name: ProductName, +) -> NonNegativeDecimal | None: + return await _api.get_product_credit_price_by_app_and_product( + app, product_name=product_name + ) + + +async def register_rpc_routes_on_startup(app: web.Application): + rpc_server = get_rabbitmq_rpc_server(app) + await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/products/plugin.py b/services/web/server/src/simcore_service_webserver/products/plugin.py index 823c8fb2718..70483623419 100644 --- a/services/web/server/src/simcore_service_webserver/products/plugin.py +++ b/services/web/server/src/simcore_service_webserver/products/plugin.py @@ -15,7 +15,8 @@ from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup from .._constants import APP_SETTINGS_KEY -from . import _handlers, _invitations_handlers +from ..rabbitmq import setup_rabbitmq +from . import _handlers, _invitations_handlers, _rpc from ._events import ( auto_create_products_groups, load_products_on_startup, @@ -43,6 +44,11 @@ def setup_products(app: web.Application): app.router.add_routes(_handlers.routes) app.router.add_routes(_invitations_handlers.routes) + # rpc api + setup_rabbitmq(app) + if app[APP_SETTINGS_KEY].WEBSERVER_RABBITMQ: + app.on_startup.append(_rpc.register_rpc_routes_on_startup) + # events app.on_startup.append( # NOTE: must go BEFORE load_products_on_startup From 3acb1dba9cd3bae6b9218b3de990d480ba3e00f3 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 13 Nov 2023 19:07:52 +0100 Subject: [PATCH 56/78] daily work --- .../services/auto_recharge_process_message.py | 83 +++++++++++++------ 1 file changed, 59 insertions(+), 24 deletions(-) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index 7f0225d0148..23af7eb0821 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -2,16 +2,22 @@ from decimal import Decimal from fastapi import FastAPI +from simcore_service_payments.db.payments_transactions_repo import ( + PaymentsTransactionsRepo, +) from models_library.api_schemas_webserver.wallets import GetWalletAutoRecharge +from models_library.rabbitmq_basic_types import RPCMethodName from models_library.rabbitmq_messages import WalletCreditsMessage -from pydantic import parse_raw_as +from pydantic import EmailStr, parse_obj_as, parse_raw_as from simcore_service_payments.db.auto_recharge_repo import AutoRechargeRepo from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo from simcore_service_payments.services.payments_gateway import PaymentsGatewayApi - +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE from ..core.settings import ApplicationSettings -from .auto_recharge import get_wallet_payment_autorecharge -from .payments_methods import get_payment_method +from .auto_recharge import get_wallet_auto_recharge +from .payments_methods import get_payment_method_by_id +from .payments import init_payment_with_payment_method +from .rabbitmq import get_rabbitmq_rpc_client _logger = logging.getLogger(__name__) @@ -22,37 +28,66 @@ async def process_message(app: FastAPI, data: bytes) -> bool: _logger.debug("Process msg: %s", rabbit_message) settings: ApplicationSettings = app.state.settings - auto_recharge_repo: AutoRechargeRepo = AutoRechargeRepo(db_engine=app.state.engine) # 1. Check if wallet credits are bellow the threshold - if rabbit_message.credits > Decimal(100): # TODO: from settings MIN CREDITS - return True + if rabbit_message.credits > settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: + return True # --> We do not auto recharge - # 2. Check if auto-recharge functionality is ON for wallet_id - payment_autorecharge: GetWalletAutoRecharge | None = ( - await get_wallet_payment_autorecharge( - settings, auto_recharge_repo, wallet_id=rabbit_message.wallet_id - ) + # 2. Check if auto-recharge functionality is enabled for wallet_id + _auto_recharge_repo: AutoRechargeRepo = AutoRechargeRepo(db_engine=app.state.engine) + wallet_auto_recharge: GetWalletAutoRecharge | None = await get_wallet_auto_recharge( + settings, _auto_recharge_repo, wallet_id=rabbit_message.wallet_id ) if ( - payment_autorecharge is None - or payment_autorecharge.enabled is False - or payment_autorecharge.payment_method_id is None + wallet_auto_recharge is None + or wallet_auto_recharge.enabled is False + or wallet_auto_recharge.payment_method_id is None ): - return True + return True # --> We do not auto recharge # 3. Get Payment method _payments_gateway = PaymentsGatewayApi.get_from_app_state(app) _payments_repo = PaymentsMethodsRepo(db_engine=app.state.engine) - await get_payment_method( - _payments_gateway, - _payments_repo, - payment_method_id=payment_autorecharge.payment_method_id, - user_id=1, # TODO: Why user_id/wallet_id is here? I would query purly based on payment_method_id - wallet_id=rabbit_message.wallet_id, + payment_method = ( + await get_payment_method_by_id( # Maybe add user to the payment_method? + _payments_gateway, + _payments_repo, + payment_method_id=wallet_auto_recharge.payment_method_id, + ) + ) + + # 4. Check whether number of topups is still in the limit + assert settings.PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT + + # 5. Protective measure: check whether there was not already top up made in the last minutes? + + # 6. Pay with payment method + ## 6.1 Ask webserver to compute credits with current dollar/credit ratio + rabbitmq_rpc_client = get_rabbitmq_rpc_client(app) + result = await rabbitmq_rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "get_product_credit_price_by_app_and_product"), + product_name="osparc", ) + parse_obj_as(ConvertedCreditsGet | None, result) + target_credits = Decimal(1000) - # 4. Pay with payment method - # TODO: I will need to query webserver for user email & wallet name? + ## 6.2 Make payment + _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + await init_payment_with_payment_method( + gateway=_payments_gateway, + repo_transactions=_payments_transactions_repo, + repo_methods=_payments_repo, + payment_method_id=payment_method.idr, # equals to wallet_auto_recharge.payment_method_id + amount_dollars=wallet_auto_recharge.top_up_amount_in_usd, + target_credits=target_credits, + product_name="osparc", # I need to know the product probably will add to the message + wallet_id=rabbit_message.wallet_id, + wallet_name=f"id={rabbit_message.wallet_id}", + user_id=1, # payment_method.user_id ? + user_name=f"id={1}", # payment_method.user_id ? + user_email=EmailStr("unknown@unknown.itis"), # Get + comment="Payment generated by auto recharge", + ) return True From 0e18c88d16aa2dc3e655662e05098617f55a781b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 14 Nov 2023 15:05:18 +0100 Subject: [PATCH 57/78] introducing rpc for product prices --- .../src/models_library/products.py | 17 ++++++++- .../src/models_library/rabbitmq_messages.py | 1 + .../services/auto_recharge_process_message.py | 26 +++++++------ .../test_services_auto_recharge_listener.py | 2 +- .../resource_tracker_utils.py | 1 + .../products/_api.py | 37 +++++++++++++++---- .../products/_rpc.py | 9 +++-- .../simcore_service_webserver/products/api.py | 2 + .../products/errors.py | 14 +++++++ .../wallets/_handlers.py | 5 +++ .../wallets/_payments_handlers.py | 17 ++++++++- 11 files changed, 105 insertions(+), 26 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/products/errors.py diff --git a/packages/models-library/src/models_library/products.py b/packages/models-library/src/models_library/products.py index 5fdb2a59209..db95a680d9b 100644 --- a/packages/models-library/src/models_library/products.py +++ b/packages/models-library/src/models_library/products.py @@ -1,3 +1,18 @@ -from typing import TypeAlias +from decimal import Decimal +from typing import Any, ClassVar, TypeAlias + +from pydantic import BaseModel, Field ProductName: TypeAlias = str + + +class CreditResultGet(BaseModel): + product_name: str + credit_amount: Decimal = Field(..., description="") + + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + {"product_name": "osparc", "credit_amount": Decimal(15.5)}, + ] + } diff --git a/packages/models-library/src/models_library/rabbitmq_messages.py b/packages/models-library/src/models_library/rabbitmq_messages.py index a32a0b8dbe3..6f43202b385 100644 --- a/packages/models-library/src/models_library/rabbitmq_messages.py +++ b/packages/models-library/src/models_library/rabbitmq_messages.py @@ -264,6 +264,7 @@ class WalletCreditsMessage(RabbitMessageBase): ) wallet_id: WalletID credits: Decimal + product_name: str def routing_key(self) -> str | None: return f"{self.wallet_id}" diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index 23af7eb0821..f73be3dcb2e 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -1,22 +1,23 @@ import logging -from decimal import Decimal from fastapi import FastAPI -from simcore_service_payments.db.payments_transactions_repo import ( - PaymentsTransactionsRepo, -) +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE from models_library.api_schemas_webserver.wallets import GetWalletAutoRecharge +from models_library.products import CreditResultGet from models_library.rabbitmq_basic_types import RPCMethodName from models_library.rabbitmq_messages import WalletCreditsMessage from pydantic import EmailStr, parse_obj_as, parse_raw_as from simcore_service_payments.db.auto_recharge_repo import AutoRechargeRepo from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo +from simcore_service_payments.db.payments_transactions_repo import ( + PaymentsTransactionsRepo, +) from simcore_service_payments.services.payments_gateway import PaymentsGatewayApi -from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE + from ..core.settings import ApplicationSettings from .auto_recharge import get_wallet_auto_recharge -from .payments_methods import get_payment_method_by_id from .payments import init_payment_with_payment_method +from .payments_methods import get_payment_method_by_id from .rabbitmq import get_rabbitmq_rpc_client _logger = logging.getLogger(__name__) @@ -60,17 +61,18 @@ async def process_message(app: FastAPI, data: bytes) -> bool: assert settings.PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT # 5. Protective measure: check whether there was not already top up made in the last minutes? + # TODO: Matus add protective measure # 6. Pay with payment method ## 6.1 Ask webserver to compute credits with current dollar/credit ratio rabbitmq_rpc_client = get_rabbitmq_rpc_client(app) result = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, - parse_obj_as(RPCMethodName, "get_product_credit_price_by_app_and_product"), - product_name="osparc", + parse_obj_as(RPCMethodName, "get_credit_amount"), + dollar_amount=wallet_auto_recharge.top_up_amount_in_usd, + product_name=rabbit_message.product_name, ) - parse_obj_as(ConvertedCreditsGet | None, result) - target_credits = Decimal(1000) + credit_result = parse_obj_as(CreditResultGet, result) ## 6.2 Make payment _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) @@ -80,8 +82,8 @@ async def process_message(app: FastAPI, data: bytes) -> bool: repo_methods=_payments_repo, payment_method_id=payment_method.idr, # equals to wallet_auto_recharge.payment_method_id amount_dollars=wallet_auto_recharge.top_up_amount_in_usd, - target_credits=target_credits, - product_name="osparc", # I need to know the product probably will add to the message + target_credits=credit_result.credit_amount, + product_name=rabbit_message.product_name, wallet_id=rabbit_message.wallet_id, wallet_name=f"id={rabbit_message.wallet_id}", user_id=1, # payment_method.user_id ? diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 9557dba382d..01c571fb5dd 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -67,7 +67,7 @@ async def test_process_event_functions( rabbitmq_client: Callable[[str], RabbitMQClient], ): publisher = rabbitmq_client("publisher") - msg = WalletCreditsMessage(wallet_id=1, credits=Decimal(120.5)) + msg = WalletCreditsMessage(wallet_id=1, credits=Decimal(120.5), product_name="s4l") await publisher.publish(WalletCreditsMessage.get_channel_name(), msg) async for attempt in AsyncRetrying( diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py index 4c54c8b0628..d2ffc4ad425 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_utils.py @@ -45,6 +45,7 @@ async def sum_credit_transactions_and_publish_to_rabbitmq( wallet_id=wallet_id, created_at=datetime.now(tz=timezone.utc), credits=wallet_total_credits.available_osparc_credits, + product_name=product_name, ) await rabbitmq_client.publish(publish_message.channel_name, publish_message) return wallet_total_credits diff --git a/services/web/server/src/simcore_service_webserver/products/_api.py b/services/web/server/src/simcore_service_webserver/products/_api.py index 6e0498dc2df..931534c457c 100644 --- a/services/web/server/src/simcore_service_webserver/products/_api.py +++ b/services/web/server/src/simcore_service_webserver/products/_api.py @@ -1,15 +1,18 @@ +from decimal import Decimal from pathlib import Path +from typing import Final import aiofiles from aiohttp import web from models_library.basic_types import NonNegativeDecimal -from models_library.products import ProductName +from models_library.products import CreditResultGet, ProductName from .._constants import APP_PRODUCTS_KEY, RQ_PRODUCT_KEY from .._resources import webserver_resources from ._db import ProductRepository from ._events import APP_PRODUCTS_TEMPLATES_DIR_KEY from ._model import Product +from .errors import ProductPriceNotDefinedError def get_product_name(request: web.Request) -> str: @@ -49,14 +52,34 @@ async def get_current_product_credit_price( return await repo.get_product_latest_credit_price_or_none(current_product_name) -async def get_product_credit_price_by_app_and_product( - app: web.Application, *, product_name: ProductName -): - repo = ProductRepository.create_from_app(app) - return await repo.get_product_latest_credit_price_or_none(product_name) +MSG_PRICE_NOT_DEFINED_ERROR: Final[ + str +] = "No payments are accepted until this product has a price" + + +async def get_credit_amount( + app: web.Application, *, dollar_amount: Decimal, product_name: ProductName +) -> CreditResultGet: + """For provided dollars and product gets credit amount. + NOTE: Contrary to other product api functions (e.g. get_current_product) this function + gets the latest update from the database. Otherwise, products are loaded + on startup and cached therefore in those cases would require a restart + of the service for the latest changes to take effect. + """ + repo = ProductRepository.create_from_app(app) + usd_per_credit: NonNegativeDecimal | None = ( + await repo.get_product_latest_credit_price_or_none(product_name) + ) + if not usd_per_credit: + # '0 or None' should raise + raise ProductPriceNotDefinedError( + reason=f"Product {product_name} usd_per_credit is either not defined or zero" + ) + + credit_amount = dollar_amount / usd_per_credit + return CreditResultGet(product_name=product_name, credit_amount=credit_amount) -# Get conversion! send already money -> get credits # # helpers for get_product_template_path diff --git a/services/web/server/src/simcore_service_webserver/products/_rpc.py b/services/web/server/src/simcore_service_webserver/products/_rpc.py index 75f92ccbe2a..1a9333b76c3 100644 --- a/services/web/server/src/simcore_service_webserver/products/_rpc.py +++ b/services/web/server/src/simcore_service_webserver/products/_rpc.py @@ -1,3 +1,5 @@ +from decimal import Decimal + from aiohttp import web from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE from models_library.basic_types import NonNegativeDecimal @@ -11,13 +13,14 @@ @router.expose() -async def get_product_credit_price_by_app_and_product( +async def get_credit_amount( app: web.Application, *, + dollar_amount: Decimal, product_name: ProductName, ) -> NonNegativeDecimal | None: - return await _api.get_product_credit_price_by_app_and_product( - app, product_name=product_name + await _api.get_credit_amount( + app, dollar_amount=dollar_amount, product_name=product_name ) diff --git a/services/web/server/src/simcore_service_webserver/products/api.py b/services/web/server/src/simcore_service_webserver/products/api.py index 58d82f2a872..64532082b16 100644 --- a/services/web/server/src/simcore_service_webserver/products/api.py +++ b/services/web/server/src/simcore_service_webserver/products/api.py @@ -1,6 +1,7 @@ from models_library.products import ProductName from ._api import ( + get_credit_amount, get_current_product, get_current_product_credit_price, get_product, @@ -11,6 +12,7 @@ from ._model import Product __all__: tuple[str, ...] = ( + "get_credit_amount", "get_current_product_credit_price", "get_current_product", "get_product_name", diff --git a/services/web/server/src/simcore_service_webserver/products/errors.py b/services/web/server/src/simcore_service_webserver/products/errors.py new file mode 100644 index 00000000000..a4696b01dc3 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/products/errors.py @@ -0,0 +1,14 @@ +""" + API plugin errors +""" + + +from pydantic.errors import PydanticErrorMixin + + +class ProductError(PydanticErrorMixin, ValueError): + ... + + +class ProductPriceNotDefinedError(ProductError): + msg_template = "Product price not defined. {reason}" diff --git a/services/web/server/src/simcore_service_webserver/wallets/_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_handlers.py index beaba7fc4f7..c1d4d8bf154 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_handlers.py @@ -33,10 +33,12 @@ PaymentNotFoundError, PaymentUniqueViolationError, ) +from ..products.errors import ProductPriceNotDefinedError from ..security.decorators import permission_required from ..users.exceptions import UserDefaultWalletNotFoundError from ..utils_aiohttp import envelope_json_response from . import _api +from ._constants import MSG_PRICE_NOT_DEFINED_ERROR from .errors import WalletAccessForbiddenError, WalletNotFoundError _logger = logging.getLogger(__name__) @@ -68,6 +70,9 @@ async def wrapper(request: web.Request) -> web.StreamResponse: except WalletAccessForbiddenError as exc: raise web.HTTPForbidden(reason=f"{exc}") from exc + except ProductPriceNotDefinedError as exc: + raise web.HTTPConflict(reason=MSG_PRICE_NOT_DEFINED_ERROR) + return wrapper diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index c4cac0680f2..097cdf5baae 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -41,7 +41,7 @@ pay_with_payment_method, replace_wallet_payment_autorecharge, ) -from ..products.api import get_current_product_credit_price +from ..products.api import get_credit_amount from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from ._constants import MSG_PRICE_NOT_DEFINED_ERROR @@ -63,7 +63,20 @@ async def _eval_total_credits_or_raise( # '0 or None' should raise raise web.HTTPConflict(reason=MSG_PRICE_NOT_DEFINED_ERROR) - return parse_obj_as(NonNegativeDecimal, amount_dollars / usd_per_credit) + credit_result = await get_credit_amount( + request.app, dollar_amount=init.price_dollars, product_name=product_name + ) + + return await init_creation_of_wallet_payment( + request.app, + user_id=user_id, + product_name=product_name, + wallet_id=wallet_id, + osparc_credits=credit_result.credit_amount, + comment=init.comment, + price_dollars=init.price_dollars, + payment_method_id=payment_method_id, + ) routes = web.RouteTableDef() From 8db2fece4f3d041c8b5f83bdace1090654dba924 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Nov 2023 18:50:52 +0100 Subject: [PATCH 58/78] daily work --- .../src/models_library/products.py | 2 +- .../pytest_simcore/helpers/rawdata_fakers.py | 23 +++ .../db/auto_recharge_repo.py | 4 +- .../db/payments_transactions_repo.py | 44 +++++- .../models/payments_gateway.py | 20 ++- .../services/auto_recharge.py | 10 -- .../services/auto_recharge_process_message.py | 71 ++++++--- .../services/payments.py | 2 +- .../test_services_auto_recharge_listener.py | 137 +++++++++++++++++- .../products/_api.py | 6 - .../products/_rpc.py | 8 +- .../wallets/_payments_handlers.py | 3 +- 12 files changed, 275 insertions(+), 55 deletions(-) diff --git a/packages/models-library/src/models_library/products.py b/packages/models-library/src/models_library/products.py index db95a680d9b..60e5297b203 100644 --- a/packages/models-library/src/models_library/products.py +++ b/packages/models-library/src/models_library/products.py @@ -13,6 +13,6 @@ class CreditResultGet(BaseModel): class Config: schema_extra: ClassVar[dict[str, Any]] = { "examples": [ - {"product_name": "osparc", "credit_amount": Decimal(15.5)}, + {"product_name": "s4l", "credit_amount": Decimal(15.5)}, ] } diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py index 036d0a9c126..f39d691bf2c 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py @@ -199,6 +199,8 @@ def random_payment_method( "user_id": FAKE.pyint(), "wallet_id": FAKE.pyint(), "initiated_at": utcnow(), + "state": "PENDING", + "completed_at": None, } # state is not added on purpose assert set(data.keys()).issubset({c.name for c in payments_methods.columns}) @@ -234,6 +236,27 @@ def random_payment_transaction( return data +def random_payment_autorecharge( + primary_payment_method_id: str = FAKE.uuid4(), + **overrides, +) -> dict[str, Any]: + from simcore_postgres_database.models.payments_autorecharge import ( + payments_autorecharge, + ) + + data = { + "wallet_id": FAKE.pyint(), + "enabled": True, + "primary_payment_method_id": primary_payment_method_id, + "top_up_amount_in_usd": 100, + "monthly_limit_in_usd": 1000, + } + assert set(data.keys()).issubset({c.name for c in payments_autorecharge.columns}) + + data.update(overrides) + return data + + def random_api_key(product_name: str, user_id: int, **overrides) -> dict[str, Any]: data = { "display_name": FAKE.word(), diff --git a/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py b/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py index 53743a8fa8d..d4bc5634e54 100644 --- a/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py +++ b/services/payments/src/simcore_service_payments/db/auto_recharge_repo.py @@ -37,7 +37,7 @@ async def get_wallet_autorecharge( async with self.db_engine.begin() as conn: stmt = AutoRechargeStmts.get_wallet_autorecharge(wallet_id) result = await conn.execute(stmt) - row = await result.first() + row = result.first() return PaymentsAutorechargeDB.from_orm(row) if row else None async def replace_wallet_autorecharge( @@ -71,6 +71,6 @@ async def replace_wallet_autorecharge( monthly_limit_in_usd=new.monthly_limit_in_usd, ) result = await conn.execute(stmt) - row = await result.first() + row = result.first() assert row # nosec return PaymentsAutorechargeDB.from_orm(row) diff --git a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py index 8cfde93980d..84ee8b8543c 100644 --- a/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_transactions_repo.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime, timezone from decimal import Decimal import sqlalchemy as sa @@ -33,7 +33,7 @@ async def insert_init_payment_transaction( user_email: str, wallet_id: WalletID, comment: str | None, - initiated_at: datetime.datetime, + initiated_at: datetime, ) -> PaymentID: """Annotates init-payment transaction @@ -177,3 +177,43 @@ async def get_payment_transaction( ) row = result.fetchone() return PaymentsTransactionsDB.from_orm(row) if row else None + + async def sum_current_month_dollars(self, *, wallet_id: WalletID) -> Decimal: + _current_timestamp = datetime.now(tz=timezone.utc) + _current_month_start_timestamp = _current_timestamp.replace( + day=1, hour=0, minute=0, second=0, microsecond=0 + ) + + async with self.db_engine.begin() as conn: + sum_stmt = sa.select( + sa.func.sum(payments_transactions.c.price_dollars) + ).where( + (payments_transactions.c.wallet_id == wallet_id) + & ( + payments_transactions.c.state.in_( + [ + PaymentTransactionState.SUCCESS, + ] + ) + ) + & ( + payments_transactions.c.completed_at + >= _current_month_start_timestamp + ) + ) + result = await conn.execute(sum_stmt) + row = result.first() + return Decimal(0) if row is None or row[0] is None else Decimal(row[0]) + + async def get_last_payment_transaction_for_wallet( + self, *, wallet_id: WalletID + ) -> PaymentsTransactionsDB | None: + async with self.db_engine.begin() as connection: + result = await connection.execute( + payments_transactions.select() + .where(payments_transactions.c.wallet_id == wallet_id) + .order_by(payments_transactions.c.initiated_at.desc()) + .limit(1) + ) + row = result.fetchone() + return PaymentsTransactionsDB.from_orm(row) if row else None diff --git a/services/payments/src/simcore_service_payments/models/payments_gateway.py b/services/payments/src/simcore_service_payments/models/payments_gateway.py index 6614f67d303..6cd2d753949 100644 --- a/services/payments/src/simcore_service_payments/models/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/models/payments_gateway.py @@ -1,6 +1,6 @@ from datetime import datetime from pathlib import Path -from typing import Literal +from typing import Any, ClassVar, Literal from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID from models_library.basic_types import AmountDecimal, IDStr @@ -59,6 +59,24 @@ class GetPaymentMethod(BaseModel): expiration_year: int | None = None created: datetime + class Config: + schema_extra: ClassVar[dict[str, Any]] = { + "examples": [ + { + "idr": "pm_1234567890", + "card_holder_name": "John Doe", + "card_number_masked": "**** **** **** 1234", + "card_type": "Visa", + "expiration_month": 10, + "expiration_year": 2025, + "street_address": "123 Main St", + "zipcode": "12345", + "country": "United States", + "created": "2023-09-13T15:30:00Z", + }, + ], + } + class BatchGetPaymentMethods(BaseModel): payment_methods_ids: list[PaymentMethodID] diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge.py b/services/payments/src/simcore_service_payments/services/auto_recharge.py index 71bf61a352a..162ed68740e 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge.py @@ -6,12 +6,9 @@ ReplaceWalletAutoRecharge, ) from models_library.basic_types import NonNegativeDecimal -from models_library.products import ProductName from models_library.users import UserID from models_library.wallets import WalletID -# from ._methods_db import list_successful_payment_methods -# from ._onetime_api import raise_for_wallet_payments_permissions from ..core.settings import ApplicationSettings from ..db.auto_recharge_repo import AutoRechargeRepo, PaymentsAutorechargeDB from ..db.payments_methods_repo import PaymentsMethodsRepo @@ -56,7 +53,6 @@ async def get_wallet_auto_recharge( *, wallet_id: WalletID, ) -> GetWalletAutoRecharge | None: - # TODO: Permissions will be checked in webserver! payments_autorecharge_db: PaymentsAutorechargeDB | None = ( await auto_recharge_repo.get_wallet_autorecharge(wallet_id=wallet_id) ) @@ -76,7 +72,6 @@ async def get_wallet_payment_autorecharge_with_default( auto_recharge_repo: AutoRechargeRepo, payments_method_repo: PaymentsMethodsRepo, *, - product_name: ProductName, user_id: UserID, wallet_id: WalletID, ) -> GetWalletAutoRecharge: @@ -112,15 +107,10 @@ async def replace_wallet_payment_autorecharge( app: FastAPI, repo: AutoRechargeRepo, *, - product_name: ProductName, user_id: UserID, wallet_id: WalletID, new: ReplaceWalletAutoRecharge, ) -> GetWalletAutoRecharge: - # TODO: Ask Pedro - # await raise_for_wallet_payments_permissions( - # app, user_id=user_id, wallet_id=wallet_id, product_name=product_name - # ) settings: ApplicationSettings = app.state.settings got: PaymentsAutorechargeDB = await repo.replace_wallet_autorecharge( user_id=user_id, diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index f73be3dcb2e..cdb57e1ddb0 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -1,4 +1,5 @@ import logging +from datetime import datetime, timedelta, timezone from fastapi import FastAPI from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE @@ -6,7 +7,7 @@ from models_library.products import CreditResultGet from models_library.rabbitmq_basic_types import RPCMethodName from models_library.rabbitmq_messages import WalletCreditsMessage -from pydantic import EmailStr, parse_obj_as, parse_raw_as +from pydantic import parse_obj_as, parse_raw_as from simcore_service_payments.db.auto_recharge_repo import AutoRechargeRepo from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo from simcore_service_payments.db.payments_transactions_repo import ( @@ -16,7 +17,6 @@ from ..core.settings import ApplicationSettings from .auto_recharge import get_wallet_auto_recharge -from .payments import init_payment_with_payment_method from .payments_methods import get_payment_method_by_id from .rabbitmq import get_rabbitmq_rpc_client @@ -24,7 +24,6 @@ async def process_message(app: FastAPI, data: bytes) -> bool: - assert app # nosec rabbit_message = parse_raw_as(WalletCreditsMessage, data) _logger.debug("Process msg: %s", rabbit_message) @@ -57,11 +56,37 @@ async def process_message(app: FastAPI, data: bytes) -> bool: ) ) - # 4. Check whether number of topups is still in the limit - assert settings.PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT + # 4. Check whether number of top-ups is still in the limit + _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + cumulative_current_month_spending = ( + await _payments_transactions_repo.sum_current_month_dollars( + wallet_id=rabbit_message.wallet_id + ) + ) + if ( + wallet_auto_recharge.monthly_limit_in_usd + is not None # User had setup monthly limit + and cumulative_current_month_spending + + wallet_auto_recharge.top_up_amount_in_usd + > wallet_auto_recharge.monthly_limit_in_usd + ): + _logger.warning("Current month spending") + return True # --> We do not auto recharge - # 5. Protective measure: check whether there was not already top up made in the last minutes? - # TODO: Matus add protective measure + # 5. Protective measure: check whether there was not already top up made in the last 5 minutes + _last_wallet_transaction = ( + await _payments_transactions_repo.get_last_payment_transaction_for_wallet( + wallet_id=rabbit_message.wallet_id + ) + ) + _current_timestamp = datetime.now(tz=timezone.utc) + _current_timestamp_minus_5_minutes = _current_timestamp - timedelta(minutes=5) + if ( + _last_wallet_transaction + and _last_wallet_transaction.initiated_at > _current_timestamp_minus_5_minutes + ): + _logger.warning("blabla") + return True # --> We do not auto recharge # 6. Pay with payment method ## 6.1 Ask webserver to compute credits with current dollar/credit ratio @@ -75,21 +100,21 @@ async def process_message(app: FastAPI, data: bytes) -> bool: credit_result = parse_obj_as(CreditResultGet, result) ## 6.2 Make payment - _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) - await init_payment_with_payment_method( - gateway=_payments_gateway, - repo_transactions=_payments_transactions_repo, - repo_methods=_payments_repo, - payment_method_id=payment_method.idr, # equals to wallet_auto_recharge.payment_method_id - amount_dollars=wallet_auto_recharge.top_up_amount_in_usd, - target_credits=credit_result.credit_amount, - product_name=rabbit_message.product_name, - wallet_id=rabbit_message.wallet_id, - wallet_name=f"id={rabbit_message.wallet_id}", - user_id=1, # payment_method.user_id ? - user_name=f"id={1}", # payment_method.user_id ? - user_email=EmailStr("unknown@unknown.itis"), # Get - comment="Payment generated by auto recharge", - ) + assert _payments_transactions_repo + # await init_payment_with_payment_method( + # gateway=_payments_gateway, + # repo_transactions=_payments_transactions_repo, + # repo_methods=_payments_repo, + # payment_method_id=payment_method.idr, # equals to wallet_auto_recharge.payment_method_id + # amount_dollars=wallet_auto_recharge.top_up_amount_in_usd, + # target_credits=Decimal(10),#credit_result.credit_amount, + # product_name=rabbit_message.product_name, + # wallet_id=rabbit_message.wallet_id, + # wallet_name=f"id={rabbit_message.wallet_id}", + # user_id=1, # payment_method.user_id ? + # user_name=f"id={1}", # payment_method.user_id ? + # user_email=EmailStr("unknown@unknown.itis"), + # comment="Payment generated by auto recharge", + # ) return True diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index 5fa4f374c33..11a860dbabb 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -198,7 +198,7 @@ async def pay_with_payment_method( # noqa: PLR0913 ) -> PaymentTransaction: initiated_at = arrow.utcnow().datetime - acked = await repo_methods.get_payment_method( + acked = await repo_methods.get_successful_payment_method( payment_method_id, user_id=user_id, wallet_id=wallet_id ) diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 01c571fb5dd..874dab21204 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -4,17 +4,36 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable +import asyncio from collections.abc import Callable +from datetime import datetime, timedelta, timezone from decimal import Decimal +from typing import Awaitable, Iterator from unittest import mock import pytest +import sqlalchemy as sa +from faker import Faker from fastapi import FastAPI +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.products import CreditResultGet, ProductName from models_library.rabbitmq_messages import WalletCreditsMessage from pytest_mock.plugin import MockerFixture + +# I need autorecharge DB +# I need payment method +# Mock get_credit_amount +# +from pytest_simcore.helpers.rawdata_fakers import ( + random_payment_autorecharge, + random_payment_method, +) from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict -from servicelib.rabbitmq import RabbitMQClient +from servicelib.rabbitmq import RabbitMQClient, RabbitMQRPCClient, RPCRouter +from simcore_postgres_database.models.payments_autorecharge import payments_autorecharge +from simcore_postgres_database.models.payments_methods import payments_methods +from simcore_service_payments.models.payments_gateway import GetPaymentMethod from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay @@ -33,7 +52,7 @@ def app_environment( monkeypatch: pytest.MonkeyPatch, app_environment: EnvVarsDict, - rabbit_env_vars_dict: EnvVarsDict, # rabbitMQ settings from 'rabbit' service + rabbit_env_vars_dict: EnvVarsDict, postgres_env_vars_dict: EnvVarsDict, wait_for_postgres_ready_and_db_migrated: None, external_environment: EnvVarsDict, @@ -61,13 +80,13 @@ async def mocked_message_parser(mocker: MockerFixture) -> mock.AsyncMock: ) -async def test_process_event_functions( +async def test_process_event_function_called( mocked_message_parser: mock.AsyncMock, app: FastAPI, rabbitmq_client: Callable[[str], RabbitMQClient], ): publisher = rabbitmq_client("publisher") - msg = WalletCreditsMessage(wallet_id=1, credits=Decimal(120.5), product_name="s4l") + msg = WalletCreditsMessage(wallet_id=1, credits=Decimal(80.5), product_name="s4l") await publisher.publish(WalletCreditsMessage.get_channel_name(), msg) async for attempt in AsyncRetrying( @@ -78,3 +97,113 @@ async def test_process_event_functions( ): with attempt: mocked_message_parser.assert_called_once() + + +@pytest.fixture() +def populate_test_db( + postgres_db: sa.engine.Engine, faker: Faker, wallet_id: int +) -> Iterator[None]: + with postgres_db.connect() as con: + _primary_payment_method_id = faker.uuid4() + # _wallet_id = faker.pyint() + _completed_at = datetime.now(tz=timezone.utc) + timedelta(minutes=1) + + con.execute( + payments_methods.insert().values( + **random_payment_method( + payment_method_id=_primary_payment_method_id, + wallet_id=wallet_id, + state="SUCCESS", + completed_at=_completed_at, + ) + ) + ) + con.execute( + payments_autorecharge.insert().values( + **random_payment_autorecharge( + primary_payment_method_id=_primary_payment_method_id, + wallet_id=wallet_id, + ) + ) + ) + + yield + + con.execute(payments_methods.delete()) + con.execute(payments_autorecharge.delete()) + + +@pytest.fixture() +def wallet_id(): + return 10 + + +# @pytest.fixture() +# def mock_(): +# GetPaymentMethod + + +@pytest.fixture +async def mocked_get_payment_method(mocker: MockerFixture) -> mock.AsyncMock: + return mocker.patch( + "simcore_service_payments.services.payments_methods.PaymentsGatewayApi.get_payment_method", + return_value=GetPaymentMethod.construct( + **GetPaymentMethod.Config.schema_extra["examples"][0] + ), + ) + + +@pytest.fixture +async def mock_rpc_server( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], + mocker: MockerFixture, +) -> RabbitMQRPCClient: + rpc_client = await rabbitmq_rpc_client("client") + rpc_server = await rabbitmq_rpc_client("mock_server") + + router = RPCRouter() + + # mocks the interface defined in the webserver + + @router.expose() + async def get_credit_amount( + dollar_amount: ProductName, product_name: ProductName + ) -> CreditResultGet: + return CreditResultGet.parse_obj( + CreditResultGet.Config.schema_extra["examples"][0] + ) + + await rpc_server.register_router(router, namespace=WEBSERVER_RPC_NAMESPACE) + + # mock returned client + mocker.patch( + "simcore_service_payments.services.auto_recharge_process_message.get_rabbitmq_rpc_client", + return_value=rpc_client, + ) + + return rpc_client + + +async def test_process_event_function_autorecharge_flow( + app: FastAPI, + rabbitmq_client: Callable[[str], RabbitMQClient], + wallet_id: int, + populate_test_db: None, + mocked_get_payment_method: mock.AsyncMock, + mock_rpc_server: RabbitMQRPCClient, +): + publisher = rabbitmq_client("publisher") + msg = WalletCreditsMessage( + wallet_id=wallet_id, credits=Decimal(80.5), product_name="s4l" + ) + await publisher.publish(WalletCreditsMessage.get_channel_name(), msg) + + await asyncio.sleep(20) + # async for attempt in AsyncRetrying( + # wait=wait_fixed(0.1), + # stop=stop_after_delay(5), + # retry=retry_if_exception_type(AssertionError), + # reraise=True, + # ): + # with attempt: + # assert 2 == 1 diff --git a/services/web/server/src/simcore_service_webserver/products/_api.py b/services/web/server/src/simcore_service_webserver/products/_api.py index 931534c457c..1239b648d3e 100644 --- a/services/web/server/src/simcore_service_webserver/products/_api.py +++ b/services/web/server/src/simcore_service_webserver/products/_api.py @@ -1,6 +1,5 @@ from decimal import Decimal from pathlib import Path -from typing import Final import aiofiles from aiohttp import web @@ -52,11 +51,6 @@ async def get_current_product_credit_price( return await repo.get_product_latest_credit_price_or_none(current_product_name) -MSG_PRICE_NOT_DEFINED_ERROR: Final[ - str -] = "No payments are accepted until this product has a price" - - async def get_credit_amount( app: web.Application, *, dollar_amount: Decimal, product_name: ProductName ) -> CreditResultGet: diff --git a/services/web/server/src/simcore_service_webserver/products/_rpc.py b/services/web/server/src/simcore_service_webserver/products/_rpc.py index 1a9333b76c3..4a4ee46a655 100644 --- a/services/web/server/src/simcore_service_webserver/products/_rpc.py +++ b/services/web/server/src/simcore_service_webserver/products/_rpc.py @@ -2,8 +2,7 @@ from aiohttp import web from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE -from models_library.basic_types import NonNegativeDecimal -from models_library.products import ProductName +from models_library.products import CreditResultGet, ProductName from servicelib.rabbitmq import RPCRouter from ..rabbitmq import get_rabbitmq_rpc_server @@ -18,10 +17,11 @@ async def get_credit_amount( *, dollar_amount: Decimal, product_name: ProductName, -) -> NonNegativeDecimal | None: - await _api.get_credit_amount( +) -> CreditResultGet: + credit_result_get: CreditResultGet = await _api.get_credit_amount( app, dollar_amount=dollar_amount, product_name=product_name ) + return credit_result_get async def register_rpc_routes_on_startup(app: web.Application): diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 097cdf5baae..e3804da74fc 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -12,6 +12,7 @@ WalletPaymentInitiated, ) from models_library.basic_types import AmountDecimal, NonNegativeDecimal +from models_library.products import CreditResultGet from models_library.rest_pagination import Page, PageQueryParameters from models_library.rest_pagination_utils import paginate_data from pydantic import parse_obj_as @@ -63,7 +64,7 @@ async def _eval_total_credits_or_raise( # '0 or None' should raise raise web.HTTPConflict(reason=MSG_PRICE_NOT_DEFINED_ERROR) - credit_result = await get_credit_amount( + credit_result: CreditResultGet = await get_credit_amount( request.app, dollar_amount=init.price_dollars, product_name=product_name ) From e9099ce6325469bba2a41d6d9a27272f5827fa1f Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Nov 2023 19:41:14 +0100 Subject: [PATCH 59/78] daily work --- .../unit/test_services_auto_recharge_listener.py | 15 ++------------- .../src/simcore_service_webserver/products/api.py | 2 -- .../wallets/_payments_handlers.py | 7 ------- 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 874dab21204..1f9427f6477 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -19,11 +19,6 @@ from models_library.products import CreditResultGet, ProductName from models_library.rabbitmq_messages import WalletCreditsMessage from pytest_mock.plugin import MockerFixture - -# I need autorecharge DB -# I need payment method -# Mock get_credit_amount -# from pytest_simcore.helpers.rawdata_fakers import ( random_payment_autorecharge, random_payment_method, @@ -105,7 +100,6 @@ def populate_test_db( ) -> Iterator[None]: with postgres_db.connect() as con: _primary_payment_method_id = faker.uuid4() - # _wallet_id = faker.pyint() _completed_at = datetime.now(tz=timezone.utc) + timedelta(minutes=1) con.execute( @@ -134,13 +128,8 @@ def populate_test_db( @pytest.fixture() -def wallet_id(): - return 10 - - -# @pytest.fixture() -# def mock_(): -# GetPaymentMethod +def wallet_id(faker: Faker): + return faker.pyint() @pytest.fixture diff --git a/services/web/server/src/simcore_service_webserver/products/api.py b/services/web/server/src/simcore_service_webserver/products/api.py index 64532082b16..eb4b2bab505 100644 --- a/services/web/server/src/simcore_service_webserver/products/api.py +++ b/services/web/server/src/simcore_service_webserver/products/api.py @@ -3,7 +3,6 @@ from ._api import ( get_credit_amount, get_current_product, - get_current_product_credit_price, get_product, get_product_name, get_product_template_path, @@ -13,7 +12,6 @@ __all__: tuple[str, ...] = ( "get_credit_amount", - "get_current_product_credit_price", "get_current_product", "get_product_name", "get_product_template_path", diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index e3804da74fc..a303229738b 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -45,7 +45,6 @@ from ..products.api import get_credit_amount from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from ._constants import MSG_PRICE_NOT_DEFINED_ERROR from ._handlers import ( WalletsPathParams, WalletsRequestContext, @@ -58,12 +57,6 @@ async def _eval_total_credits_or_raise( request: web.Request, amount_dollars: AmountDecimal ) -> NonNegativeDecimal: - # Conversion - usd_per_credit = await get_current_product_credit_price(request) - if not usd_per_credit: - # '0 or None' should raise - raise web.HTTPConflict(reason=MSG_PRICE_NOT_DEFINED_ERROR) - credit_result: CreditResultGet = await get_credit_amount( request.app, dollar_amount=init.price_dollars, product_name=product_name ) From f0d1e4e78476d094a959a976828fcf340826b7e2 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 08:45:29 +0100 Subject: [PATCH 60/78] small modification, before merging Pedro's PR --- .../services/auto_recharge_process_message.py | 25 +++++++++---------- .../services/payments_methods.py | 4 +-- .../unit/test_db_payments_methods_repo.py | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index cdb57e1ddb0..cbb626775fa 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -17,7 +17,6 @@ from ..core.settings import ApplicationSettings from .auto_recharge import get_wallet_auto_recharge -from .payments_methods import get_payment_method_by_id from .rabbitmq import get_rabbitmq_rpc_client _logger = logging.getLogger(__name__) @@ -46,14 +45,9 @@ async def process_message(app: FastAPI, data: bytes) -> bool: return True # --> We do not auto recharge # 3. Get Payment method - _payments_gateway = PaymentsGatewayApi.get_from_app_state(app) _payments_repo = PaymentsMethodsRepo(db_engine=app.state.engine) - payment_method = ( - await get_payment_method_by_id( # Maybe add user to the payment_method? - _payments_gateway, - _payments_repo, - payment_method_id=wallet_auto_recharge.payment_method_id, - ) + payment_method_db = await _payments_repo.get_successful_payment_method_by_id( + payment_method_id=wallet_auto_recharge.payment_method_id ) # 4. Check whether number of top-ups is still in the limit @@ -70,7 +64,10 @@ async def process_message(app: FastAPI, data: bytes) -> bool: + wallet_auto_recharge.top_up_amount_in_usd > wallet_auto_recharge.monthly_limit_in_usd ): - _logger.warning("Current month spending") + _logger.warning( + "Current month spending would go over the limit %s", + wallet_auto_recharge.monthly_limit_in_usd, + ) return True # --> We do not auto recharge # 5. Protective measure: check whether there was not already top up made in the last 5 minutes @@ -85,7 +82,9 @@ async def process_message(app: FastAPI, data: bytes) -> bool: _last_wallet_transaction and _last_wallet_transaction.initiated_at > _current_timestamp_minus_5_minutes ): - _logger.warning("blabla") + _logger.warning( + "There was already made top up in last 5 minutes, therefore we will pass" + ) return True # --> We do not auto recharge # 6. Pay with payment method @@ -100,7 +99,7 @@ async def process_message(app: FastAPI, data: bytes) -> bool: credit_result = parse_obj_as(CreditResultGet, result) ## 6.2 Make payment - assert _payments_transactions_repo + _payments_gateway = PaymentsGatewayApi.get_from_app_state(app) # await init_payment_with_payment_method( # gateway=_payments_gateway, # repo_transactions=_payments_transactions_repo, @@ -111,8 +110,8 @@ async def process_message(app: FastAPI, data: bytes) -> bool: # product_name=rabbit_message.product_name, # wallet_id=rabbit_message.wallet_id, # wallet_name=f"id={rabbit_message.wallet_id}", - # user_id=1, # payment_method.user_id ? - # user_name=f"id={1}", # payment_method.user_id ? + # user_id=payment_method_db.user_id, + # user_name=f"id={payment_method_db.user_id}", # user_email=EmailStr("unknown@unknown.itis"), # comment="Payment generated by auto recharge", # ) diff --git a/services/payments/src/simcore_service_payments/services/payments_methods.py b/services/payments/src/simcore_service_payments/services/payments_methods.py index e4cc5821501..487dfb539cf 100644 --- a/services/payments/src/simcore_service_payments/services/payments_methods.py +++ b/services/payments/src/simcore_service_payments/services/payments_methods.py @@ -191,7 +191,7 @@ async def get_payment_method( user_id: UserID, wallet_id: WalletID, ) -> PaymentMethodGet: - acked = await repo.get_payment_method( + acked = await repo.get_successful_payment_method( payment_method_id, user_id=user_id, wallet_id=wallet_id ) assert acked.state == InitPromptAckFlowState.SUCCESS # nosec @@ -208,7 +208,7 @@ async def delete_payment_method( user_id: UserID, wallet_id: WalletID, ): - acked = await repo.get_payment_method( + acked = await repo.get_successful_payment_method( payment_method_id, user_id=user_id, wallet_id=wallet_id ) diff --git a/services/payments/tests/unit/test_db_payments_methods_repo.py b/services/payments/tests/unit/test_db_payments_methods_repo.py index ef876845b54..2a7831a79b1 100644 --- a/services/payments/tests/unit/test_db_payments_methods_repo.py +++ b/services/payments/tests/unit/test_db_payments_methods_repo.py @@ -73,7 +73,7 @@ async def test_create_payments_method_annotations_workflow(app: FastAPI): assert listed[0] == acked # get - got = await repo.get_payment_method( + got = await repo.get_successful_payment_method( payment_method_id, user_id=fake.user_id, wallet_id=fake.wallet_id, From d9adbab099d4da9049bfe382be492c4436324d26 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 10:00:28 +0100 Subject: [PATCH 61/78] modification --- .../models/payments_gateway.py | 5 +-- .../services/auto_recharge_process_message.py | 38 +++++++++---------- .../wallets/_payments_handlers.py | 31 +++------------ 3 files changed, 25 insertions(+), 49 deletions(-) diff --git a/services/payments/src/simcore_service_payments/models/payments_gateway.py b/services/payments/src/simcore_service_payments/models/payments_gateway.py index 6cd2d753949..c8347475f8a 100644 --- a/services/payments/src/simcore_service_payments/models/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/models/payments_gateway.py @@ -63,15 +63,12 @@ class Config: schema_extra: ClassVar[dict[str, Any]] = { "examples": [ { - "idr": "pm_1234567890", + "id": "pm_1234567890", "card_holder_name": "John Doe", "card_number_masked": "**** **** **** 1234", "card_type": "Visa", "expiration_month": 10, "expiration_year": 2025, - "street_address": "123 Main St", - "zipcode": "12345", - "country": "United States", "created": "2023-09-13T15:30:00Z", }, ], diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index cbb626775fa..cbfa9bbbdf7 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -7,16 +7,17 @@ from models_library.products import CreditResultGet from models_library.rabbitmq_basic_types import RPCMethodName from models_library.rabbitmq_messages import WalletCreditsMessage -from pydantic import parse_obj_as, parse_raw_as +from pydantic import EmailStr, parse_obj_as, parse_raw_as from simcore_service_payments.db.auto_recharge_repo import AutoRechargeRepo from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo from simcore_service_payments.db.payments_transactions_repo import ( PaymentsTransactionsRepo, ) -from simcore_service_payments.services.payments_gateway import PaymentsGatewayApi from ..core.settings import ApplicationSettings from .auto_recharge import get_wallet_auto_recharge +from .payments import pay_with_payment_method +from .payments_gateway import PaymentsGatewayApi from .rabbitmq import get_rabbitmq_rpc_client _logger = logging.getLogger(__name__) @@ -50,7 +51,7 @@ async def process_message(app: FastAPI, data: bytes) -> bool: payment_method_id=wallet_auto_recharge.payment_method_id ) - # 4. Check whether number of top-ups is still in the limit + # 4. Check whether number of US dollar top-ups is still in the limit _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) cumulative_current_month_spending = ( await _payments_transactions_repo.sum_current_month_dollars( @@ -100,20 +101,19 @@ async def process_message(app: FastAPI, data: bytes) -> bool: ## 6.2 Make payment _payments_gateway = PaymentsGatewayApi.get_from_app_state(app) - # await init_payment_with_payment_method( - # gateway=_payments_gateway, - # repo_transactions=_payments_transactions_repo, - # repo_methods=_payments_repo, - # payment_method_id=payment_method.idr, # equals to wallet_auto_recharge.payment_method_id - # amount_dollars=wallet_auto_recharge.top_up_amount_in_usd, - # target_credits=Decimal(10),#credit_result.credit_amount, - # product_name=rabbit_message.product_name, - # wallet_id=rabbit_message.wallet_id, - # wallet_name=f"id={rabbit_message.wallet_id}", - # user_id=payment_method_db.user_id, - # user_name=f"id={payment_method_db.user_id}", - # user_email=EmailStr("unknown@unknown.itis"), - # comment="Payment generated by auto recharge", - # ) - + await pay_with_payment_method( + gateway=_payments_gateway, + repo_transactions=_payments_transactions_repo, + repo_methods=_payments_repo, + payment_method_id=wallet_auto_recharge.payment_method_id, + amount_dollars=wallet_auto_recharge.top_up_amount_in_usd, + target_credits=credit_result.credit_amount, + product_name=rabbit_message.product_name, + wallet_id=rabbit_message.wallet_id, + wallet_name=f"id={rabbit_message.wallet_id}", + user_id=payment_method_db.user_id, + user_name=f"id={payment_method_db.user_id}", + user_email=EmailStr(f"placeholder_{payment_method_db.user_id}@example.itis"), + comment="Payment generated by auto recharge", + ) return True diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index a303229738b..7cf16c0620e 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -11,11 +11,9 @@ ReplaceWalletAutoRecharge, WalletPaymentInitiated, ) -from models_library.basic_types import AmountDecimal, NonNegativeDecimal from models_library.products import CreditResultGet from models_library.rest_pagination import Page, PageQueryParameters from models_library.rest_pagination_utils import paginate_data -from pydantic import parse_obj_as from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -54,25 +52,6 @@ _logger = logging.getLogger(__name__) -async def _eval_total_credits_or_raise( - request: web.Request, amount_dollars: AmountDecimal -) -> NonNegativeDecimal: - credit_result: CreditResultGet = await get_credit_amount( - request.app, dollar_amount=init.price_dollars, product_name=product_name - ) - - return await init_creation_of_wallet_payment( - request.app, - user_id=user_id, - product_name=product_name, - wallet_id=wallet_id, - osparc_credits=credit_result.credit_amount, - comment=init.comment, - price_dollars=init.price_dollars, - payment_method_id=payment_method_id, - ) - - routes = web.RouteTableDef() @@ -98,9 +77,10 @@ async def _create_payment(request: web.Request): log_duration=True, extra=get_log_record_extra(user_id=req_ctx.user_id), ): - - osparc_credits = await _eval_total_credits_or_raise( - request, body_params.price_dollars + credit_result: CreditResultGet = await get_credit_amount( + request.app, + dollar_amount=body_params.price_dollars, + product_name=req_ctx.product_name, ) payment: WalletPaymentInitiated = await init_creation_of_wallet_payment( @@ -108,7 +88,7 @@ async def _create_payment(request: web.Request): user_id=req_ctx.user_id, product_name=req_ctx.product_name, wallet_id=wallet_id, - osparc_credits=osparc_credits, + osparc_credits=credit_result.credit_amount, comment=body_params.comment, price_dollars=body_params.price_dollars, ) @@ -341,7 +321,6 @@ async def _pay_with_payment_method(request: web.Request): log_duration=True, extra=get_log_record_extra(user_id=req_ctx.user_id), ): - osparc_credits = await _eval_total_credits_or_raise( request, body_params.price_dollars ) From 7183323e6024a82b85d9329afd80dfe1f95cf239 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 10:08:21 +0100 Subject: [PATCH 62/78] fixing after conflicts --- .../wallets/_payments_handlers.py | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py index 4328987eb54..0efde369a95 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_payments_handlers.py @@ -321,8 +321,10 @@ async def _pay_with_payment_method(request: web.Request): log_duration=True, extra=get_log_record_extra(user_id=req_ctx.user_id), ): - osparc_credits = await _eval_total_credits_or_raise( - request, body_params.price_dollars + credit_result: CreditResultGet = await get_credit_amount( + request.app, + dollar_amount=body_params.price_dollars, + product_name=req_ctx.product_name, ) payment: PaymentTransaction = await pay_with_payment_method( @@ -331,10 +333,7 @@ async def _pay_with_payment_method(request: web.Request): product_name=req_ctx.product_name, wallet_id=wallet_id, payment_method_id=path_params.payment_method_id, - osparc_credits=osparc_credits, - comment=body_params.comment, - price_dollars=body_params.price_dollars, - osparc_credits=osparc_credits, + osparc_credits=credit_result.credit_amount, comment=body_params.comment, price_dollars=body_params.price_dollars, ) @@ -358,25 +357,6 @@ async def _pay_with_payment_method(request: web.Request): ), web.HTTPAccepted, ) - # NOTE: Due to the design change in https://github.com/ITISFoundation/osparc-simcore/pull/5017 - # we decided not to change the return value to avoid changing the front-end logic - # instead we emulate a init-prompt-ack workflow by firing a background task that acks payment - - fire_and_forget_task( - notify_payment_completed( - request.app, user_id=req_ctx.user_id, payment=payment - ), - task_suffix_name=f"{__name__}._pay_with_payment_method", - fire_and_forget_tasks_collection=request.app[APP_FIRE_AND_FORGET_TASKS_KEY], - ) - - return envelope_json_response( - WalletPaymentInitiated( - payment_id=payment.payment_id, - payment_form_url=None, - ), - web.HTTPAccepted, - ) # From a34ed545c1e00dd38dc25bf1f858d670b550523f Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 10:24:33 +0100 Subject: [PATCH 63/78] minor cleanup --- .../services/auto_recharge_process_message.py | 3 ++- .../services/payments_methods.py | 13 ------------- .../unit/test_services_auto_recharge_listener.py | 2 +- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index cbfa9bbbdf7..7e6f8c2f827 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -66,8 +66,9 @@ async def process_message(app: FastAPI, data: bytes) -> bool: > wallet_auto_recharge.monthly_limit_in_usd ): _logger.warning( - "Current month spending would go over the limit %s", + "Current month spending would go over the limit %s for payment method %s", wallet_auto_recharge.monthly_limit_in_usd, + wallet_auto_recharge.payment_method_id, ) return True # --> We do not auto recharge diff --git a/services/payments/src/simcore_service_payments/services/payments_methods.py b/services/payments/src/simcore_service_payments/services/payments_methods.py index 487dfb539cf..ac6a47bee51 100644 --- a/services/payments/src/simcore_service_payments/services/payments_methods.py +++ b/services/payments/src/simcore_service_payments/services/payments_methods.py @@ -170,19 +170,6 @@ async def list_successful_payment_methods( ] -async def get_payment_method_by_id( - gateway: PaymentsGatewayApi, - repo: PaymentsMethodsRepo, - *, - payment_method_id: PaymentMethodID, -) -> PaymentMethodGet: - acked = await repo.get_successful_payment_method_by_id(payment_method_id) - assert acked.state == InitPromptAckFlowState.SUCCESS # nosec - - got: GetPaymentMethod = await gateway.get_payment_method(acked.payment_method_id) - return merge_models(got, acked) - - async def get_payment_method( gateway: PaymentsGatewayApi, repo: PaymentsMethodsRepo, diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 1f9427f6477..3ecc1518493 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -47,7 +47,7 @@ def app_environment( monkeypatch: pytest.MonkeyPatch, app_environment: EnvVarsDict, - rabbit_env_vars_dict: EnvVarsDict, + rabbit_env_vars_dict: EnvVarsDict, # rabbitMQ settings from 'rabbit' service postgres_env_vars_dict: EnvVarsDict, wait_for_postgres_ready_and_db_migrated: None, external_environment: EnvVarsDict, From 537d58b695716bc5844c0da91aa71102db84bf33 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 11:06:11 +0100 Subject: [PATCH 64/78] refactor process message --- .../services/auto_recharge_process_message.py | 111 +++++++++++------- 1 file changed, 68 insertions(+), 43 deletions(-) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index 7e6f8c2f827..554ea2922ce 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -1,18 +1,24 @@ import logging from datetime import datetime, timedelta, timezone +from typing import cast from fastapi import FastAPI from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE -from models_library.api_schemas_webserver.wallets import GetWalletAutoRecharge +from models_library.api_schemas_webserver.wallets import ( + GetWalletAutoRecharge, + PaymentMethodID, +) from models_library.products import CreditResultGet from models_library.rabbitmq_basic_types import RPCMethodName from models_library.rabbitmq_messages import WalletCreditsMessage +from models_library.wallets import WalletID from pydantic import EmailStr, parse_obj_as, parse_raw_as from simcore_service_payments.db.auto_recharge_repo import AutoRechargeRepo from simcore_service_payments.db.payments_methods_repo import PaymentsMethodsRepo from simcore_service_payments.db.payments_transactions_repo import ( PaymentsTransactionsRepo, ) +from simcore_service_payments.models.db import PaymentsMethodsDB from ..core.settings import ApplicationSettings from .auto_recharge import get_wallet_auto_recharge @@ -29,11 +35,11 @@ async def process_message(app: FastAPI, data: bytes) -> bool: settings: ApplicationSettings = app.state.settings - # 1. Check if wallet credits are bellow the threshold + # Step 1: Check if wallet credits are below the threshold if rabbit_message.credits > settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: - return True # --> We do not auto recharge + return True # We do not auto recharge - # 2. Check if auto-recharge functionality is enabled for wallet_id + # Step 2: Check auto-recharge conditions _auto_recharge_repo: AutoRechargeRepo = AutoRechargeRepo(db_engine=app.state.engine) wallet_auto_recharge: GetWalletAutoRecharge | None = await get_wallet_auto_recharge( settings, _auto_recharge_repo, wallet_id=rabbit_message.wallet_id @@ -43,54 +49,73 @@ async def process_message(app: FastAPI, data: bytes) -> bool: or wallet_auto_recharge.enabled is False or wallet_auto_recharge.payment_method_id is None ): - return True # --> We do not auto recharge + return True # We do not auto recharge - # 3. Get Payment method + # Step 3: Get Payment method _payments_repo = PaymentsMethodsRepo(db_engine=app.state.engine) payment_method_db = await _payments_repo.get_successful_payment_method_by_id( payment_method_id=wallet_auto_recharge.payment_method_id ) - # 4. Check whether number of US dollar top-ups is still in the limit + # Step 4: Check spending limits _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + if _exceeds_monthly_limit( + _payments_transactions_repo, rabbit_message.wallet_id, wallet_auto_recharge + ): + return True # We do not auto recharge + + # Step 5: Check last top-up time + if _recently_topped_up(_payments_transactions_repo, rabbit_message.wallet_id): + return True # We do not auto recharge + + # Step 6: Perform auto-recharge + await _perform_auto_recharge( + app, rabbit_message, payment_method_db, wallet_auto_recharge + ) + return True + + +async def _exceeds_monthly_limit( + payments_transactions_repo: PaymentsTransactionsRepo, + wallet_id: WalletID, + wallet_auto_recharge: GetWalletAutoRecharge, +): cumulative_current_month_spending = ( - await _payments_transactions_repo.sum_current_month_dollars( - wallet_id=rabbit_message.wallet_id - ) + await payments_transactions_repo.sum_current_month_dollars(wallet_id=wallet_id) ) - if ( - wallet_auto_recharge.monthly_limit_in_usd - is not None # User had setup monthly limit + + return ( + wallet_auto_recharge.monthly_limit_in_usd is not None and cumulative_current_month_spending + wallet_auto_recharge.top_up_amount_in_usd > wallet_auto_recharge.monthly_limit_in_usd - ): - _logger.warning( - "Current month spending would go over the limit %s for payment method %s", - wallet_auto_recharge.monthly_limit_in_usd, - wallet_auto_recharge.payment_method_id, - ) - return True # --> We do not auto recharge + ) + - # 5. Protective measure: check whether there was not already top up made in the last 5 minutes - _last_wallet_transaction = ( - await _payments_transactions_repo.get_last_payment_transaction_for_wallet( - wallet_id=rabbit_message.wallet_id +async def _recently_topped_up( + payments_transactions_repo: PaymentsTransactionsRepo, wallet_id: WalletID +): + last_wallet_transaction = ( + await payments_transactions_repo.get_last_payment_transaction_for_wallet( + wallet_id=wallet_id ) ) - _current_timestamp = datetime.now(tz=timezone.utc) - _current_timestamp_minus_5_minutes = _current_timestamp - timedelta(minutes=5) - if ( - _last_wallet_transaction - and _last_wallet_transaction.initiated_at > _current_timestamp_minus_5_minutes - ): - _logger.warning( - "There was already made top up in last 5 minutes, therefore we will pass" - ) - return True # --> We do not auto recharge - # 6. Pay with payment method - ## 6.1 Ask webserver to compute credits with current dollar/credit ratio + current_timestamp = datetime.now(tz=timezone.utc) + current_timestamp_minus_5_minutes = current_timestamp - timedelta(minutes=5) + + return ( + last_wallet_transaction + and last_wallet_transaction.initiated_at > current_timestamp_minus_5_minutes + ) + + +async def _perform_auto_recharge( + app: FastAPI, + rabbit_message: WalletCreditsMessage, + payment_method_db: PaymentsMethodsDB, + wallet_auto_recharge: GetWalletAutoRecharge, +): rabbitmq_rpc_client = get_rabbitmq_rpc_client(app) result = await rabbitmq_rpc_client.request( WEBSERVER_RPC_NAMESPACE, @@ -100,13 +125,14 @@ async def process_message(app: FastAPI, data: bytes) -> bool: ) credit_result = parse_obj_as(CreditResultGet, result) - ## 6.2 Make payment - _payments_gateway = PaymentsGatewayApi.get_from_app_state(app) + payments_gateway = PaymentsGatewayApi.get_from_app_state(app) + payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + await pay_with_payment_method( - gateway=_payments_gateway, - repo_transactions=_payments_transactions_repo, - repo_methods=_payments_repo, - payment_method_id=wallet_auto_recharge.payment_method_id, + gateway=payments_gateway, + repo_transactions=payments_transactions_repo, + repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine), + payment_method_id=cast(PaymentMethodID, wallet_auto_recharge.payment_method_id), amount_dollars=wallet_auto_recharge.top_up_amount_in_usd, target_credits=credit_result.credit_amount, product_name=rabbit_message.product_name, @@ -117,4 +143,3 @@ async def process_message(app: FastAPI, data: bytes) -> bool: user_email=EmailStr(f"placeholder_{payment_method_db.user_id}@example.itis"), comment="Payment generated by auto recharge", ) - return True From a86fa7cbaeb1c11659eef3fd7869a022807ccba9 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 14:28:01 +0100 Subject: [PATCH 65/78] adding test --- .../api/rpc/_payments_methods.py | 2 + .../models/payments_gateway.py | 17 +---- .../services/auto_recharge.py | 2 +- .../services/auto_recharge_process_message.py | 10 ++- .../services/payments.py | 3 + services/payments/tests/unit/conftest.py | 63 ++++++++++++++++++- .../test_services_auto_recharge_listener.py | 54 ++++++++++------ .../test_services_resource_usage_tracker.py | 62 +----------------- 8 files changed, 113 insertions(+), 100 deletions(-) diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py index 5e44f2ac917..e2af796a89e 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py @@ -17,6 +17,7 @@ from ...db.payments_transactions_repo import PaymentsTransactionsRepo from ...services import payments, payments_methods from ...services.payments_gateway import PaymentsGatewayApi +from ...services.resource_usage_tracker import ResourceUsageTrackerApi _logger = logging.getLogger(__name__) @@ -128,6 +129,7 @@ async def pay_with_payment_method( # noqa: PLR0913 # pylint: disable=too-many-a ): return await payments.pay_with_payment_method( gateway=PaymentsGatewayApi.get_from_app_state(app), + rut=ResourceUsageTrackerApi.get_from_app_state(app), repo_transactions=PaymentsTransactionsRepo(db_engine=app.state.engine), repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine), payment_method_id=payment_method_id, diff --git a/services/payments/src/simcore_service_payments/models/payments_gateway.py b/services/payments/src/simcore_service_payments/models/payments_gateway.py index c8347475f8a..6614f67d303 100644 --- a/services/payments/src/simcore_service_payments/models/payments_gateway.py +++ b/services/payments/src/simcore_service_payments/models/payments_gateway.py @@ -1,6 +1,6 @@ from datetime import datetime from pathlib import Path -from typing import Any, ClassVar, Literal +from typing import Literal from models_library.api_schemas_webserver.wallets import PaymentID, PaymentMethodID from models_library.basic_types import AmountDecimal, IDStr @@ -59,21 +59,6 @@ class GetPaymentMethod(BaseModel): expiration_year: int | None = None created: datetime - class Config: - schema_extra: ClassVar[dict[str, Any]] = { - "examples": [ - { - "id": "pm_1234567890", - "card_holder_name": "John Doe", - "card_number_masked": "**** **** **** 1234", - "card_type": "Visa", - "expiration_month": 10, - "expiration_year": 2025, - "created": "2023-09-13T15:30:00Z", - }, - ], - } - class BatchGetPaymentMethods(BaseModel): payment_methods_ids: list[PaymentMethodID] diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge.py b/services/payments/src/simcore_service_payments/services/auto_recharge.py index 162ed68740e..64ccda92691 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge.py @@ -67,7 +67,7 @@ async def get_wallet_auto_recharge( return None -async def get_wallet_payment_autorecharge_with_default( +async def get_user_wallet_payment_autorecharge_with_default( app: FastAPI, auto_recharge_repo: AutoRechargeRepo, payments_method_repo: PaymentsMethodsRepo, diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index 554ea2922ce..71261ac2e8c 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -19,6 +19,9 @@ PaymentsTransactionsRepo, ) from simcore_service_payments.models.db import PaymentsMethodsDB +from simcore_service_payments.services.resource_usage_tracker import ( + ResourceUsageTrackerApi, +) from ..core.settings import ApplicationSettings from .auto_recharge import get_wallet_auto_recharge @@ -59,13 +62,13 @@ async def process_message(app: FastAPI, data: bytes) -> bool: # Step 4: Check spending limits _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) - if _exceeds_monthly_limit( + if await _exceeds_monthly_limit( _payments_transactions_repo, rabbit_message.wallet_id, wallet_auto_recharge ): return True # We do not auto recharge # Step 5: Check last top-up time - if _recently_topped_up(_payments_transactions_repo, rabbit_message.wallet_id): + if await _recently_topped_up(_payments_transactions_repo, rabbit_message.wallet_id): return True # We do not auto recharge # Step 6: Perform auto-recharge @@ -83,7 +86,6 @@ async def _exceeds_monthly_limit( cumulative_current_month_spending = ( await payments_transactions_repo.sum_current_month_dollars(wallet_id=wallet_id) ) - return ( wallet_auto_recharge.monthly_limit_in_usd is not None and cumulative_current_month_spending @@ -127,9 +129,11 @@ async def _perform_auto_recharge( payments_gateway = PaymentsGatewayApi.get_from_app_state(app) payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + rut_api = ResourceUsageTrackerApi.get_from_app_state(app) await pay_with_payment_method( gateway=payments_gateway, + rut=rut_api, repo_transactions=payments_transactions_repo, repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine), payment_method_id=cast(PaymentMethodID, wallet_auto_recharge.payment_method_id), diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index 11a860dbabb..78209f52dc3 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -182,6 +182,7 @@ async def on_payment_completed( async def pay_with_payment_method( # noqa: PLR0913 gateway: PaymentsGatewayApi, + rut: ResourceUsageTrackerApi, repo_transactions: PaymentsTransactionsRepo, repo_methods: PaymentsMethodsRepo, *, @@ -246,6 +247,8 @@ async def pay_with_payment_method( # noqa: PLR0913 invoice_url=ack.invoice_url, ) + await on_payment_completed(transaction, rut) + return transaction.to_api_model() diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index 086f6052d98..f7ba2055acc 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -7,10 +7,11 @@ from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from pathlib import Path -from typing import NamedTuple +from typing import Any, NamedTuple from unittest.mock import Mock import httpx +import jsonref import pytest import respx import sqlalchemy as sa @@ -40,6 +41,7 @@ from simcore_service_payments.models.schemas.acknowledgements import ( AckPaymentWithPaymentMethod, ) +from toolz.dicttoolz import get_in @pytest.fixture(scope="session") @@ -361,3 +363,62 @@ def mock_payments_gateway_service_or_none( mock_payments_routes(mock_payments_gateway_service_api_base) mock_payments_methods_routes(mock_payments_gateway_service_api_base) return mock_payments_gateway_service_api_base + + +# +# mock resource-usage-tracker API +# + + +@pytest.fixture +def rut_service_openapi_specs( + osparc_simcore_services_dir: Path, +) -> dict[str, Any]: + openapi_path = ( + osparc_simcore_services_dir / "resource-usage-tracker" / "openapi.json" + ) + return jsonref.loads(openapi_path.read_text()) + + +@pytest.fixture +def mock_resource_usage_tracker_service_api_base( + app: FastAPI, rut_service_openapi_specs: dict[str, Any] +) -> Iterator[MockRouter]: + settings: ApplicationSettings = app.state.settings + with respx.mock( + base_url=settings.PAYMENTS_RESOURCE_USAGE_TRACKER.base_url, + assert_all_called=False, + assert_all_mocked=True, # IMPORTANT: KEEP always True! + ) as respx_mock: + assert "healthcheck" in get_in( + ["paths", "/", "get", "operationId"], + rut_service_openapi_specs, + no_default=True, + ) # type: ignore + respx_mock.get( + path="/", + name="healthcheck", + ).respond(status.HTTP_200_OK) + + yield respx_mock + + +@pytest.fixture +def mock_rut_service_api( + faker: Faker, + mock_resource_usage_tracker_service_api_base: MockRouter, + rut_service_openapi_specs: dict[str, Any], +) -> MockRouter: + # check it exists + get_in( + ["paths", "/v1/credit-transactions", "post", "operationId"], + rut_service_openapi_specs, + no_default=True, + ) + + # fake successful response + mock_resource_usage_tracker_service_api_base.post( + "/v1/credit-transactions" + ).respond(json={"credit_transaction_id": faker.pyint()}) + + return mock_resource_usage_tracker_service_api_base diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 3ecc1518493..9c62f700c4e 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -4,7 +4,6 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -import asyncio from collections.abc import Callable from datetime import datetime, timedelta, timezone from decimal import Decimal @@ -25,10 +24,18 @@ ) from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from respx import MockRouter from servicelib.rabbitmq import RabbitMQClient, RabbitMQRPCClient, RPCRouter from simcore_postgres_database.models.payments_autorecharge import payments_autorecharge from simcore_postgres_database.models.payments_methods import payments_methods -from simcore_service_payments.models.payments_gateway import GetPaymentMethod +from simcore_postgres_database.models.payments_transactions import ( + PaymentTransactionState, + payments_transactions, +) +from simcore_service_payments.models.db import PaymentsTransactionsDB +from simcore_service_payments.models.schemas.acknowledgements import ( + AckPaymentWithPaymentMethod, +) from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay @@ -75,7 +82,7 @@ async def mocked_message_parser(mocker: MockerFixture) -> mock.AsyncMock: ) -async def test_process_event_function_called( +async def test_process_message__called( mocked_message_parser: mock.AsyncMock, app: FastAPI, rabbitmq_client: Callable[[str], RabbitMQClient], @@ -133,11 +140,11 @@ def wallet_id(faker: Faker): @pytest.fixture -async def mocked_get_payment_method(mocker: MockerFixture) -> mock.AsyncMock: +async def mocked_pay_with_payment_method(mocker: MockerFixture) -> mock.AsyncMock: return mocker.patch( - "simcore_service_payments.services.payments_methods.PaymentsGatewayApi.get_payment_method", - return_value=GetPaymentMethod.construct( - **GetPaymentMethod.Config.schema_extra["examples"][0] + "simcore_service_payments.services.payments.PaymentsGatewayApi.pay_with_payment_method", + return_value=AckPaymentWithPaymentMethod.construct( + **AckPaymentWithPaymentMethod.Config.schema_extra["example"] ), ) @@ -173,13 +180,29 @@ async def get_credit_amount( return rpc_client -async def test_process_event_function_autorecharge_flow( +async def _assert_payments_transactions_db_row(postgres_db) -> PaymentsTransactionsDB: + async for attempt in AsyncRetrying( + wait=wait_fixed(0.2), + stop=stop_after_delay(10), + retry=retry_if_exception_type(AssertionError), + reraise=True, + ): + with attempt, postgres_db.connect() as con: + result = con.execute(sa.select(payments_transactions)) + row = result.first() + assert row + return PaymentsTransactionsDB.from_orm(row) + + +async def test_process_message__topup_with_autorecharge_flow( app: FastAPI, rabbitmq_client: Callable[[str], RabbitMQClient], wallet_id: int, populate_test_db: None, - mocked_get_payment_method: mock.AsyncMock, + mocked_pay_with_payment_method: mock.AsyncMock, mock_rpc_server: RabbitMQRPCClient, + mock_rut_service_api: MockRouter, + postgres_db: sa.engine.Engine, ): publisher = rabbitmq_client("publisher") msg = WalletCreditsMessage( @@ -187,12 +210,7 @@ async def test_process_event_function_autorecharge_flow( ) await publisher.publish(WalletCreditsMessage.get_channel_name(), msg) - await asyncio.sleep(20) - # async for attempt in AsyncRetrying( - # wait=wait_fixed(0.1), - # stop=stop_after_delay(5), - # retry=retry_if_exception_type(AssertionError), - # reraise=True, - # ): - # with attempt: - # assert 2 == 1 + row = await _assert_payments_transactions_db_row(postgres_db) + assert row.wallet_id == wallet_id + assert row.state == PaymentTransactionState.SUCCESS + assert row.comment == "Payment generated by auto recharge" diff --git a/services/payments/tests/unit/test_services_resource_usage_tracker.py b/services/payments/tests/unit/test_services_resource_usage_tracker.py index 26aa9a8a015..022dfc35973 100644 --- a/services/payments/tests/unit/test_services_resource_usage_tracker.py +++ b/services/payments/tests/unit/test_services_resource_usage_tracker.py @@ -5,17 +5,12 @@ # pylint: disable=unused-variable -from collections.abc import Iterator from datetime import datetime, timezone -from pathlib import Path -from typing import Any -import jsonref import pytest -import respx from asgi_lifespan import LifespanManager from faker import Faker -from fastapi import FastAPI, status +from fastapi import FastAPI from pytest_simcore.helpers.utils_envs import EnvVarsDict from respx import MockRouter from simcore_service_payments.core.application import create_app @@ -24,7 +19,6 @@ ResourceUsageTrackerApi, setup_resource_usage_tracker, ) -from toolz.dicttoolz import get_in async def test_setup_rut_api(app_environment: EnvVarsDict): @@ -61,60 +55,6 @@ def app( return create_app() -@pytest.fixture -def rut_service_openapi_specs( - osparc_simcore_services_dir: Path, -) -> dict[str, Any]: - openapi_path = ( - osparc_simcore_services_dir / "resource-usage-tracker" / "openapi.json" - ) - return jsonref.loads(openapi_path.read_text()) - - -@pytest.fixture -def mock_resource_usage_tracker_service_api_base( - app: FastAPI, rut_service_openapi_specs: dict[str, Any] -) -> Iterator[MockRouter]: - settings: ApplicationSettings = app.state.settings - with respx.mock( - base_url=settings.PAYMENTS_RESOURCE_USAGE_TRACKER.base_url, - assert_all_called=False, - assert_all_mocked=True, # IMPORTANT: KEEP always True! - ) as respx_mock: - assert "healthcheck" in get_in( - ["paths", "/", "get", "operationId"], - rut_service_openapi_specs, - no_default=True, - ) # type: ignore - respx_mock.get( - path="/", - name="healthcheck", - ).respond(status.HTTP_200_OK) - - yield respx_mock - - -@pytest.fixture -def mock_rut_service_api( - faker: Faker, - mock_resource_usage_tracker_service_api_base: MockRouter, - rut_service_openapi_specs: dict[str, Any], -): - # check it exists - get_in( - ["paths", "/v1/credit-transactions", "post", "operationId"], - rut_service_openapi_specs, - no_default=True, - ) - - # fake successful response - mock_resource_usage_tracker_service_api_base.post( - "/v1/credit-transactions" - ).respond(json={"credit_transaction_id": faker.pyint()}) - - return mock_resource_usage_tracker_service_api_base - - async def test_add_credits_to_wallet( app: FastAPI, faker: Faker, mock_rut_service_api: MockRouter ): From 9664b1e35acd55b3f8f5917b4273363a091e6352 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 17:57:21 +0100 Subject: [PATCH 66/78] adding test --- .../pytest_simcore/helpers/rawdata_fakers.py | 2 + .../services/auto_recharge_process_message.py | 30 ++- .../test_services_auto_recharge_listener.py | 182 +++++++++++++++++- 3 files changed, 207 insertions(+), 7 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py index f39d691bf2c..f092321b56d 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py @@ -228,6 +228,8 @@ def random_payment_transaction( "wallet_id": 1, "comment": "Free starting credits", "initiated_at": utcnow(), + "state": "PENDING", + "completed_at": None, } # state is not added on purpose assert set(data.keys()).issubset({c.name for c in payments_transactions.columns}) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index 71261ac2e8c..dbe0522c534 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -1,5 +1,6 @@ import logging from datetime import datetime, timedelta, timezone +from decimal import Decimal from typing import cast from fastapi import FastAPI @@ -8,6 +9,7 @@ GetWalletAutoRecharge, PaymentMethodID, ) +from models_library.basic_types import NonNegativeDecimal from models_library.products import CreditResultGet from models_library.rabbitmq_basic_types import RPCMethodName from models_library.rabbitmq_messages import WalletCreditsMessage @@ -39,7 +41,9 @@ async def process_message(app: FastAPI, data: bytes) -> bool: settings: ApplicationSettings = app.state.settings # Step 1: Check if wallet credits are below the threshold - if rabbit_message.credits > settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: + if await _check_wallet_credits_above_threshold( + settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS, rabbit_message.credits + ): return True # We do not auto recharge # Step 2: Check auto-recharge conditions @@ -47,12 +51,10 @@ async def process_message(app: FastAPI, data: bytes) -> bool: wallet_auto_recharge: GetWalletAutoRecharge | None = await get_wallet_auto_recharge( settings, _auto_recharge_repo, wallet_id=rabbit_message.wallet_id ) - if ( - wallet_auto_recharge is None - or wallet_auto_recharge.enabled is False - or wallet_auto_recharge.payment_method_id is None - ): + if await _check_autorecharge_conditions_not_met(wallet_auto_recharge): return True # We do not auto recharge + assert wallet_auto_recharge is not None # nosec + assert wallet_auto_recharge.payment_method_id is not None # nosec # Step 3: Get Payment method _payments_repo = PaymentsMethodsRepo(db_engine=app.state.engine) @@ -78,6 +80,22 @@ async def process_message(app: FastAPI, data: bytes) -> bool: return True +async def _check_wallet_credits_above_threshold( + threshold_in_credits: NonNegativeDecimal, _credits: Decimal +) -> bool: + return _credits > threshold_in_credits + + +async def _check_autorecharge_conditions_not_met( + wallet_auto_recharge: GetWalletAutoRecharge | None, +) -> bool: + return ( + wallet_auto_recharge is None + or wallet_auto_recharge.enabled is False + or wallet_auto_recharge.payment_method_id is None + ) + + async def _exceeds_monthly_limit( payments_transactions_repo: PaymentsTransactionsRepo, wallet_id: WalletID, diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 9c62f700c4e..326235e067c 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -15,12 +15,18 @@ from faker import Faker from fastapi import FastAPI from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.api_schemas_webserver.wallets import ( + GetWalletAutoRecharge, + PaymentMethodID, +) +from models_library.basic_types import NonNegativeDecimal from models_library.products import CreditResultGet, ProductName from models_library.rabbitmq_messages import WalletCreditsMessage from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.rawdata_fakers import ( random_payment_autorecharge, random_payment_method, + random_payment_transaction, ) from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.utils_envs import setenvs_from_dict @@ -32,10 +38,20 @@ PaymentTransactionState, payments_transactions, ) +from simcore_service_payments.core.settings import ApplicationSettings +from simcore_service_payments.db.payments_transactions_repo import ( + PaymentsTransactionsRepo, +) from simcore_service_payments.models.db import PaymentsTransactionsDB from simcore_service_payments.models.schemas.acknowledgements import ( AckPaymentWithPaymentMethod, ) +from simcore_service_payments.services.auto_recharge_process_message import ( + _check_autorecharge_conditions_not_met, + _check_wallet_credits_above_threshold, + _exceeds_monthly_limit, + _recently_topped_up, +) from tenacity._asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay @@ -194,7 +210,7 @@ async def _assert_payments_transactions_db_row(postgres_db) -> PaymentsTransacti return PaymentsTransactionsDB.from_orm(row) -async def test_process_message__topup_with_autorecharge_flow( +async def test_process_message__whole_autorecharge_flow_success( app: FastAPI, rabbitmq_client: Callable[[str], RabbitMQClient], wallet_id: int, @@ -214,3 +230,167 @@ async def test_process_message__topup_with_autorecharge_flow( assert row.wallet_id == wallet_id assert row.state == PaymentTransactionState.SUCCESS assert row.comment == "Payment generated by auto recharge" + assert len(mock_rut_service_api.calls) == 1 + + +@pytest.mark.parametrize( + "_credits,expected", [(Decimal(10001), True), (Decimal(9999), False)] +) +async def test_check_wallet_credits_above_threshold( + app: FastAPI, _credits: Decimal, expected: bool +): + settings: ApplicationSettings = app.state.settings + assert settings.PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT + + assert expected == await _check_wallet_credits_above_threshold( + settings.PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT, _credits + ) + + +@pytest.mark.parametrize( + "get_wallet_auto_recharge,expected", + [ + ( + GetWalletAutoRecharge( + enabled=True, + payment_method_id=PaymentMethodID("123"), + min_balance_in_credits=NonNegativeDecimal(10), + top_up_amount_in_usd=NonNegativeDecimal(10), + monthly_limit_in_usd=NonNegativeDecimal(10), + ), + False, + ), + ( + GetWalletAutoRecharge( + enabled=False, + payment_method_id=PaymentMethodID("123"), + min_balance_in_credits=NonNegativeDecimal(10), + top_up_amount_in_usd=NonNegativeDecimal(10), + monthly_limit_in_usd=NonNegativeDecimal(10), + ), + True, + ), + ( + GetWalletAutoRecharge( + enabled=True, + payment_method_id=None, + min_balance_in_credits=NonNegativeDecimal(10), + top_up_amount_in_usd=NonNegativeDecimal(10), + monthly_limit_in_usd=NonNegativeDecimal(10), + ), + True, + ), + (None, True), + ], +) +async def test_check_autorecharge_conditions_not_met( + app: FastAPI, get_wallet_auto_recharge: GetWalletAutoRecharge, expected: bool +): + assert expected == await _check_autorecharge_conditions_not_met( + get_wallet_auto_recharge + ) + + +@pytest.fixture() +def populate_payment_transaction_db( + postgres_db: sa.engine.Engine, wallet_id: int +) -> Iterator[None]: + with postgres_db.connect() as con: + con.execute( + payments_transactions.insert().values( + **random_payment_transaction( + price_dollars=Decimal(9500), + wallet_id=wallet_id, + state=PaymentTransactionState.SUCCESS, + completed_at=datetime.now(tz=timezone.utc), + initiated_at=datetime.now(tz=timezone.utc) - timedelta(seconds=10), + ) + ) + ) + + yield + + con.execute(payments_transactions.delete()) + + +@pytest.mark.parametrize( + "get_wallet_auto_recharge,expected", + [ + ( + GetWalletAutoRecharge( + enabled=True, + payment_method_id=PaymentMethodID("123"), + min_balance_in_credits=NonNegativeDecimal(10), + top_up_amount_in_usd=NonNegativeDecimal(300), + monthly_limit_in_usd=NonNegativeDecimal(10000), + ), + False, + ), + ( + GetWalletAutoRecharge( + enabled=False, + payment_method_id=PaymentMethodID("123"), + min_balance_in_credits=NonNegativeDecimal(10), + top_up_amount_in_usd=NonNegativeDecimal(1000), + monthly_limit_in_usd=NonNegativeDecimal(10000), + ), + True, + ), + ], +) +async def test_exceeds_monthly_limit( + app: FastAPI, + wallet_id: int, + populate_payment_transaction_db: None, + get_wallet_auto_recharge: GetWalletAutoRecharge, + expected: bool, +): + _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + + assert expected == await _exceeds_monthly_limit( + _payments_transactions_repo, wallet_id, get_wallet_auto_recharge + ) + + +async def test_recently_topped_up_true( + app: FastAPI, + wallet_id: int, + populate_payment_transaction_db: None, +): + _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + + assert await _recently_topped_up(_payments_transactions_repo, wallet_id) is True + + +@pytest.fixture() +def populate_payment_transaction_db_with_older_trans( + postgres_db: sa.engine.Engine, wallet_id: int +) -> Iterator[None]: + with postgres_db.connect() as con: + current_timestamp = datetime.now(tz=timezone.utc) + current_timestamp_minus_10_minutes = current_timestamp - timedelta(minutes=10) + + con.execute( + payments_transactions.insert().values( + **random_payment_transaction( + price_dollars=Decimal(9500), + wallet_id=wallet_id, + state=PaymentTransactionState.SUCCESS, + initiated_at=current_timestamp_minus_10_minutes, + ) + ) + ) + + yield + + con.execute(payments_transactions.delete()) + + +async def test_recently_topped_up_false( + app: FastAPI, + wallet_id: int, + populate_payment_transaction_db_with_older_trans: None, +): + _payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine) + + assert await _recently_topped_up(_payments_transactions_repo, wallet_id) is False From 3e6aecb9b4fd9b6dc314547b21119ec8e80c45c8 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 18:05:01 +0100 Subject: [PATCH 67/78] adding mock --- services/payments/tests/unit/test_rpc_payments_methods.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/payments/tests/unit/test_rpc_payments_methods.py b/services/payments/tests/unit/test_rpc_payments_methods.py index 7cce4ae6f95..0220858d442 100644 --- a/services/payments/tests/unit/test_rpc_payments_methods.py +++ b/services/payments/tests/unit/test_rpc_payments_methods.py @@ -201,6 +201,7 @@ async def test_webserver_pay_with_payment_method_workflow( is_pdb_enabled: bool, app: FastAPI, rpc_client: RabbitMQRPCClient, + mock_rut_service_api: None, mock_payments_gateway_service_or_none: MockRouter | None, faker: Faker, product_name: ProductName, From 7e615b36e11f67f3b394240570ae9a366d0bedd5 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 18:43:46 +0100 Subject: [PATCH 68/78] fix --- .../tests/test_models_payments_transactions.py | 7 ++++++- .../services/auto_recharge_process_message.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/postgres-database/tests/test_models_payments_transactions.py b/packages/postgres-database/tests/test_models_payments_transactions.py index df6adee83bb..278dbb62fa5 100644 --- a/packages/postgres-database/tests/test_models_payments_transactions.py +++ b/packages/postgres-database/tests/test_models_payments_transactions.py @@ -47,7 +47,9 @@ def init_transaction(connection: SAConnection): async def _init(payment_id: str): # get payment_id from payment-gateway values = random_payment_transaction(payment_id=payment_id) - + # remove states + values.pop("state") + values.pop("completed_at") # init successful: set timestamp values["initiated_at"] = utcnow() @@ -178,6 +180,9 @@ async def _go(expected_total=5): payment_ids = [] for _ in range(expected_total): values = random_payment_transaction(user_id=user_id) + # remove states + values.pop("state") + values.pop("completed_at") payment_id = await insert_init_payment_transaction(connection, **values) assert payment_id payment_ids.append(payment_id) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index dbe0522c534..7bf9b03172d 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -83,7 +83,7 @@ async def process_message(app: FastAPI, data: bytes) -> bool: async def _check_wallet_credits_above_threshold( threshold_in_credits: NonNegativeDecimal, _credits: Decimal ) -> bool: - return _credits > threshold_in_credits + return bool(_credits > threshold_in_credits) async def _check_autorecharge_conditions_not_met( From 4d495b04b13d4673a0a2f36b18e509477544c9a3 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 19:27:00 +0100 Subject: [PATCH 69/78] adding PAYMENTS_AUTORECHARGE_ENABLED --- services/docker-compose.yml | 1 + .../payments/src/simcore_service_payments/core/settings.py | 4 ++++ .../services/auto_recharge_process_message.py | 7 ++++--- .../tests/unit/test_services_auto_recharge_listener.py | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 1b8d6baa573..54db38e2c8e 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -481,6 +481,7 @@ services: PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT: ${PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT} PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT: ${PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT} PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: ${PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS} + PAYMENTS_AUTORECHARGE_ENABLED: ${PAYMENTS_AUTORECHARGE_ENABLED} PAYMENTS_FAKE_COMPLETION_DELAY_SEC: ${PAYMENTS_FAKE_COMPLETION_DELAY_SEC} PAYMENTS_FAKE_COMPLETION: ${PAYMENTS_FAKE_COMPLETION} PAYMENTS_FAKE_GATEWAY_URL: ${PAYMENTS_GATEWAY_URL} diff --git a/services/payments/src/simcore_service_payments/core/settings.py b/services/payments/src/simcore_service_payments/core/settings.py index aaf70383e5c..ada7260d2bb 100644 --- a/services/payments/src/simcore_service_payments/core/settings.py +++ b/services/payments/src/simcore_service_payments/core/settings.py @@ -92,6 +92,10 @@ class ApplicationSettings(_BaseApplicationSettings): default=10000, description="Default value in USD for the montly limit for auto-recharge (`monthly_limit_in_usd`)", ) + PAYMENTS_AUTORECHARGE_ENABLED: bool = Field( + default=False, + description="Based on this variable is the auto recharge functionality in Payment service enabled", + ) PAYMENTS_RABBITMQ: RabbitSettings = Field( auto_default_from_env=True, description="settings for service/rabbitmq" diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index 7bf9b03172d..947b9dec5eb 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -74,9 +74,10 @@ async def process_message(app: FastAPI, data: bytes) -> bool: return True # We do not auto recharge # Step 6: Perform auto-recharge - await _perform_auto_recharge( - app, rabbit_message, payment_method_db, wallet_auto_recharge - ) + if settings.PAYMENTS_AUTORECHARGE_ENABLED: + await _perform_auto_recharge( + app, rabbit_message, payment_method_db, wallet_auto_recharge + ) return True diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 326235e067c..1af545f2b7f 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -87,6 +87,7 @@ def app_environment( **postgres_env_vars_dict, **external_environment, "POSTGRES_CLIENT_NAME": "payments-service-pg-client", + "PAYMENTS_AUTORECHARGE_ENABLED": "1", }, ) From bfd067a9da7295c343e5d041e7e16769d457626c Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 19:29:10 +0100 Subject: [PATCH 70/78] adding PAYMENTS_AUTORECHARGE_ENABLED --- services/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 54db38e2c8e..35f8529b826 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -273,6 +273,7 @@ services: - PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT=${PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT} - PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT=${PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT} - PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS=${PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS} + - PAYMENTS_AUTORECHARGE_ENABLED=${PAYMENTS_AUTORECHARGE_ENABLED} - PAYMENTS_GATEWAY_API_SECRET=${PAYMENTS_GATEWAY_API_SECRET} - PAYMENTS_GATEWAY_URL=${PAYMENTS_GATEWAY_URL} - PAYMENTS_LOGLEVEL=${PAYMENTS_LOGLEVEL} @@ -481,7 +482,6 @@ services: PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT: ${PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT} PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT: ${PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT} PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: ${PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS} - PAYMENTS_AUTORECHARGE_ENABLED: ${PAYMENTS_AUTORECHARGE_ENABLED} PAYMENTS_FAKE_COMPLETION_DELAY_SEC: ${PAYMENTS_FAKE_COMPLETION_DELAY_SEC} PAYMENTS_FAKE_COMPLETION: ${PAYMENTS_FAKE_COMPLETION} PAYMENTS_FAKE_GATEWAY_URL: ${PAYMENTS_GATEWAY_URL} From 6144dae6cee0cd768daeb59feb46c15022ddb1aa Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 20:09:19 +0100 Subject: [PATCH 71/78] add var to .env-devel --- .env-devel | 1 + 1 file changed, 1 insertion(+) diff --git a/.env-devel b/.env-devel index a2fe30524ce..d4b58c5796d 100644 --- a/.env-devel +++ b/.env-devel @@ -76,6 +76,7 @@ PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT=10000 PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT=100.0 PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS=100 +PAYMENTS_AUTORECHARGE_ENABLED=1 PAYMENTS_FAKE_COMPLETION_DELAY_SEC=10 PAYMENTS_FAKE_COMPLETION=0 PAYMENTS_GATEWAY_API_SECRET=replace-with-api-secret From e5b50f844ce6ed78113e8152c79dffb80ec5f61a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Nov 2023 20:49:29 +0100 Subject: [PATCH 72/78] adding webserver unit tests --- .../with_dbs/03/products/test_products_rpc.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py diff --git a/services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py b/services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py new file mode 100644 index 00000000000..f7e67518121 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py @@ -0,0 +1,98 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +from collections.abc import Awaitable, Callable +from decimal import Decimal + +import pytest +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.products import CreditResultGet, ProductName +from models_library.rabbitmq_basic_types import RPCMethodName +from pydantic import parse_obj_as +from pytest_mock import MockerFixture +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.utils_envs import setenvs_from_dict +from pytest_simcore.helpers.utils_login import UserInfoDict +from servicelib.rabbitmq import RabbitMQRPCClient, RPCServerError +from settings_library.rabbit import RabbitSettings +from simcore_postgres_database.models.users import UserRole +from simcore_service_webserver.application_settings import ApplicationSettings + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +@pytest.fixture +def app_environment( + rabbit_service: RabbitSettings, + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, +): + new_envs = setenvs_from_dict( + monkeypatch, + { + **app_environment, + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + }, + ) + + settings = ApplicationSettings.create_from_envs() + assert settings.WEBSERVER_RABBITMQ + + return new_envs + + +@pytest.fixture +def user_role() -> UserRole: + return UserRole.USER + + +@pytest.fixture +async def rpc_client( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], + mocker: MockerFixture, + all_product_prices, +) -> RabbitMQRPCClient: + return await rabbitmq_rpc_client("client") + + +async def test_api_key_get( + rpc_client: RabbitMQRPCClient, + osparc_product_name: ProductName, + logged_user: UserInfoDict, +): + result = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "get_credit_amount"), + dollar_amount=Decimal(900), + product_name="s4l", + ) + credit_result = parse_obj_as(CreditResultGet, result) + assert credit_result.credit_amount == 100 + + result = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "get_credit_amount"), + dollar_amount=Decimal(900), + product_name="tis", + ) + credit_result = parse_obj_as(CreditResultGet, result) + assert credit_result.credit_amount == 180 + + with pytest.raises(RPCServerError) as exc_info: + await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + parse_obj_as(RPCMethodName, "get_credit_amount"), + dollar_amount=Decimal(900), + product_name="osparc", + ) + exc = exc_info.value + assert exc.method_name == "get_credit_amount" + assert exc.msg From b10ae8e532f38db529e7e0e6159edf1fc416e068 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Nov 2023 10:12:32 +0100 Subject: [PATCH 73/78] @pcrespov review --- packages/models-library/src/models_library/products.py | 2 +- .../src/models_library/rabbitmq_messages.py | 3 ++- .../simcore_service_payments/api/rpc/_payments_methods.py | 2 +- .../simcore_service_payments/db/payments_methods_repo.py | 6 +++--- .../simcore_service_payments/services/auto_recharge.py | 8 +++----- .../services/auto_recharge_process_message.py | 4 ++-- .../src/simcore_service_payments/services/payments.py | 2 +- .../simcore_service_payments/services/payments_methods.py | 8 ++++---- .../tests/unit/with_dbs/03/products/test_products_rpc.py | 2 +- 9 files changed, 18 insertions(+), 19 deletions(-) diff --git a/packages/models-library/src/models_library/products.py b/packages/models-library/src/models_library/products.py index 60e5297b203..fdd18ff3f3b 100644 --- a/packages/models-library/src/models_library/products.py +++ b/packages/models-library/src/models_library/products.py @@ -7,7 +7,7 @@ class CreditResultGet(BaseModel): - product_name: str + product_name: ProductName credit_amount: Decimal = Field(..., description="") class Config: diff --git a/packages/models-library/src/models_library/rabbitmq_messages.py b/packages/models-library/src/models_library/rabbitmq_messages.py index 6f43202b385..9d1731731d5 100644 --- a/packages/models-library/src/models_library/rabbitmq_messages.py +++ b/packages/models-library/src/models_library/rabbitmq_messages.py @@ -6,6 +6,7 @@ from typing import Any, Literal, TypeAlias import arrow +from models_library.products import ProductName from pydantic import BaseModel, Field from pydantic.types import NonNegativeFloat @@ -264,7 +265,7 @@ class WalletCreditsMessage(RabbitMessageBase): ) wallet_id: WalletID credits: Decimal - product_name: str + product_name: ProductName def routing_key(self) -> str | None: return f"{self.wallet_id}" diff --git a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py index e2af796a89e..18262bacae1 100644 --- a/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py +++ b/services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py @@ -70,7 +70,7 @@ async def list_payment_methods( user_id: UserID, wallet_id: WalletID, ) -> list[PaymentMethodGet]: - return await payments_methods.list_successful_payment_methods( + return await payments_methods.list_payment_methods( gateway=PaymentsGatewayApi.get_from_app_state(app), repo=PaymentsMethodsRepo(db_engine=app.state.engine), user_id=user_id, diff --git a/services/payments/src/simcore_service_payments/db/payments_methods_repo.py b/services/payments/src/simcore_service_payments/db/payments_methods_repo.py index 71fcfd350ef..f1c3f791efd 100644 --- a/services/payments/src/simcore_service_payments/db/payments_methods_repo.py +++ b/services/payments/src/simcore_service_payments/db/payments_methods_repo.py @@ -114,7 +114,7 @@ async def insert_payment_method( state_message=state_message, ) - async def list_user_successful_payment_methods( + async def list_user_payment_methods( self, *, user_id: UserID, @@ -134,7 +134,7 @@ async def list_user_successful_payment_methods( rows = result.fetchall() or [] return parse_obj_as(list[PaymentsMethodsDB], rows) - async def get_successful_payment_method_by_id( + async def get_payment_method_by_id( self, payment_method_id: PaymentMethodID, ) -> PaymentsMethodsDB: @@ -151,7 +151,7 @@ async def get_successful_payment_method_by_id( return PaymentsMethodsDB.from_orm(row) - async def get_successful_payment_method( + async def get_payment_method( self, payment_method_id: PaymentMethodID, *, diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge.py b/services/payments/src/simcore_service_payments/services/auto_recharge.py index 64ccda92691..aa72715816d 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge.py @@ -84,11 +84,9 @@ async def get_user_wallet_payment_autorecharge_with_default( ) if not wallet_autorecharge: payment_method_id = None - wallet_payment_methods = ( - await payments_method_repo.list_user_successful_payment_methods( - user_id=user_id, - wallet_id=wallet_id, - ) + wallet_payment_methods = await payments_method_repo.list_user_payment_methods( + user_id=user_id, + wallet_id=wallet_id, ) if wallet_payment_methods: payment_method_id = wallet_payment_methods[_NEWEST].payment_method_id diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py index 947b9dec5eb..a4dae4eefe7 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py @@ -40,7 +40,7 @@ async def process_message(app: FastAPI, data: bytes) -> bool: settings: ApplicationSettings = app.state.settings - # Step 1: Check if wallet credits are below the threshold + # Step 1: Check if wallet credits are above the threshold if await _check_wallet_credits_above_threshold( settings.PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS, rabbit_message.credits ): @@ -58,7 +58,7 @@ async def process_message(app: FastAPI, data: bytes) -> bool: # Step 3: Get Payment method _payments_repo = PaymentsMethodsRepo(db_engine=app.state.engine) - payment_method_db = await _payments_repo.get_successful_payment_method_by_id( + payment_method_db = await _payments_repo.get_payment_method_by_id( payment_method_id=wallet_auto_recharge.payment_method_id ) diff --git a/services/payments/src/simcore_service_payments/services/payments.py b/services/payments/src/simcore_service_payments/services/payments.py index 78209f52dc3..c4ed342afc6 100644 --- a/services/payments/src/simcore_service_payments/services/payments.py +++ b/services/payments/src/simcore_service_payments/services/payments.py @@ -199,7 +199,7 @@ async def pay_with_payment_method( # noqa: PLR0913 ) -> PaymentTransaction: initiated_at = arrow.utcnow().datetime - acked = await repo_methods.get_successful_payment_method( + acked = await repo_methods.get_payment_method( payment_method_id, user_id=user_id, wallet_id=wallet_id ) diff --git a/services/payments/src/simcore_service_payments/services/payments_methods.py b/services/payments/src/simcore_service_payments/services/payments_methods.py index ac6a47bee51..24b9988177b 100644 --- a/services/payments/src/simcore_service_payments/services/payments_methods.py +++ b/services/payments/src/simcore_service_payments/services/payments_methods.py @@ -148,14 +148,14 @@ async def create_payment_method( ) -async def list_successful_payment_methods( +async def list_payment_methods( gateway: PaymentsGatewayApi, repo: PaymentsMethodsRepo, *, user_id: UserID, wallet_id: WalletID, ) -> list[PaymentMethodGet]: - acked_many = await repo.list_user_successful_payment_methods( + acked_many = await repo.list_user_payment_methods( user_id=user_id, wallet_id=wallet_id ) assert not any(acked.completed_at is None for acked in acked_many) # nosec @@ -178,7 +178,7 @@ async def get_payment_method( user_id: UserID, wallet_id: WalletID, ) -> PaymentMethodGet: - acked = await repo.get_successful_payment_method( + acked = await repo.get_payment_method( payment_method_id, user_id=user_id, wallet_id=wallet_id ) assert acked.state == InitPromptAckFlowState.SUCCESS # nosec @@ -195,7 +195,7 @@ async def delete_payment_method( user_id: UserID, wallet_id: WalletID, ): - acked = await repo.get_successful_payment_method( + acked = await repo.get_payment_method( payment_method_id, user_id=user_id, wallet_id=wallet_id ) diff --git a/services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py b/services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py index f7e67518121..c359d07a7e7 100644 --- a/services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py +++ b/services/web/server/tests/unit/with_dbs/03/products/test_products_rpc.py @@ -63,7 +63,7 @@ async def rpc_client( return await rabbitmq_rpc_client("client") -async def test_api_key_get( +async def test_get_credit_amount( rpc_client: RabbitMQRPCClient, osparc_product_name: ProductName, logged_user: UserInfoDict, From be46fcf71a0e63facb9f71e44b78721c805b184d Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Nov 2023 10:25:03 +0100 Subject: [PATCH 74/78] @GitHK @sanderegg review --- .../pytest_simcore/helpers/rawdata_fakers.py | 8 +++-- services/payments/tests/unit/conftest.py | 2 +- .../tests/unit/test_rpc_payments_methods.py | 2 +- .../test_services_auto_recharge_listener.py | 30 ++++++++++++------- .../test_services_resource_usage_tracker.py | 2 +- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py index f092321b56d..22fb55a620d 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/rawdata_fakers.py @@ -23,6 +23,10 @@ from faker import Faker from simcore_postgres_database.models.api_keys import api_keys from simcore_postgres_database.models.comp_pipeline import StateType +from simcore_postgres_database.models.payments_methods import InitPromptAckFlowState +from simcore_postgres_database.models.payments_transactions import ( + PaymentTransactionState, +) from simcore_postgres_database.models.products import products from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.users import users @@ -199,7 +203,7 @@ def random_payment_method( "user_id": FAKE.pyint(), "wallet_id": FAKE.pyint(), "initiated_at": utcnow(), - "state": "PENDING", + "state": InitPromptAckFlowState.PENDING, "completed_at": None, } # state is not added on purpose @@ -228,7 +232,7 @@ def random_payment_transaction( "wallet_id": 1, "comment": "Free starting credits", "initiated_at": utcnow(), - "state": "PENDING", + "state": PaymentTransactionState.PENDING, "completed_at": None, } # state is not added on purpose diff --git a/services/payments/tests/unit/conftest.py b/services/payments/tests/unit/conftest.py index f7ba2055acc..79bfe37ef7f 100644 --- a/services/payments/tests/unit/conftest.py +++ b/services/payments/tests/unit/conftest.py @@ -404,7 +404,7 @@ def mock_resource_usage_tracker_service_api_base( @pytest.fixture -def mock_rut_service_api( +def mock_resoruce_usage_tracker_service_api( faker: Faker, mock_resource_usage_tracker_service_api_base: MockRouter, rut_service_openapi_specs: dict[str, Any], diff --git a/services/payments/tests/unit/test_rpc_payments_methods.py b/services/payments/tests/unit/test_rpc_payments_methods.py index 0220858d442..49a3a59a1ef 100644 --- a/services/payments/tests/unit/test_rpc_payments_methods.py +++ b/services/payments/tests/unit/test_rpc_payments_methods.py @@ -201,7 +201,7 @@ async def test_webserver_pay_with_payment_method_workflow( is_pdb_enabled: bool, app: FastAPI, rpc_client: RabbitMQRPCClient, - mock_rut_service_api: None, + mock_resoruce_usage_tracker_service_api: None, mock_payments_gateway_service_or_none: MockRouter | None, faker: Faker, product_name: ProductName, diff --git a/services/payments/tests/unit/test_services_auto_recharge_listener.py b/services/payments/tests/unit/test_services_auto_recharge_listener.py index 1af545f2b7f..7b3098ddf63 100644 --- a/services/payments/tests/unit/test_services_auto_recharge_listener.py +++ b/services/payments/tests/unit/test_services_auto_recharge_listener.py @@ -167,11 +167,26 @@ async def mocked_pay_with_payment_method(mocker: MockerFixture) -> mock.AsyncMoc @pytest.fixture -async def mock_rpc_server( +async def mock_rpc_client( rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], mocker: MockerFixture, ) -> RabbitMQRPCClient: rpc_client = await rabbitmq_rpc_client("client") + + # mock returned client + mocker.patch( + "simcore_service_payments.services.auto_recharge_process_message.get_rabbitmq_rpc_client", + return_value=rpc_client, + ) + + return rpc_client + + +@pytest.fixture +async def mock_rpc_server( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], + mocker: MockerFixture, +) -> RabbitMQRPCClient: rpc_server = await rabbitmq_rpc_client("mock_server") router = RPCRouter() @@ -188,13 +203,7 @@ async def get_credit_amount( await rpc_server.register_router(router, namespace=WEBSERVER_RPC_NAMESPACE) - # mock returned client - mocker.patch( - "simcore_service_payments.services.auto_recharge_process_message.get_rabbitmq_rpc_client", - return_value=rpc_client, - ) - - return rpc_client + return rpc_server async def _assert_payments_transactions_db_row(postgres_db) -> PaymentsTransactionsDB: @@ -218,7 +227,8 @@ async def test_process_message__whole_autorecharge_flow_success( populate_test_db: None, mocked_pay_with_payment_method: mock.AsyncMock, mock_rpc_server: RabbitMQRPCClient, - mock_rut_service_api: MockRouter, + mock_rpc_client: RabbitMQRPCClient, + mock_resoruce_usage_tracker_service_api: MockRouter, postgres_db: sa.engine.Engine, ): publisher = rabbitmq_client("publisher") @@ -231,7 +241,7 @@ async def test_process_message__whole_autorecharge_flow_success( assert row.wallet_id == wallet_id assert row.state == PaymentTransactionState.SUCCESS assert row.comment == "Payment generated by auto recharge" - assert len(mock_rut_service_api.calls) == 1 + assert len(mock_resoruce_usage_tracker_service_api.calls) == 1 @pytest.mark.parametrize( diff --git a/services/payments/tests/unit/test_services_resource_usage_tracker.py b/services/payments/tests/unit/test_services_resource_usage_tracker.py index 022dfc35973..868cdf65661 100644 --- a/services/payments/tests/unit/test_services_resource_usage_tracker.py +++ b/services/payments/tests/unit/test_services_resource_usage_tracker.py @@ -56,7 +56,7 @@ def app( async def test_add_credits_to_wallet( - app: FastAPI, faker: Faker, mock_rut_service_api: MockRouter + app: FastAPI, faker: Faker, mock_resoruce_usage_tracker_service_api: MockRouter ): # test rut_api = ResourceUsageTrackerApi.get_from_app_state(app) From 9c860de280a8a57374869d3222cb78f433f2ce5c Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Nov 2023 11:43:22 +0100 Subject: [PATCH 75/78] minor fix --- .../payments/tests/unit/test_db_payments_methods_repo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/payments/tests/unit/test_db_payments_methods_repo.py b/services/payments/tests/unit/test_db_payments_methods_repo.py index 2a7831a79b1..76166c5d0be 100644 --- a/services/payments/tests/unit/test_db_payments_methods_repo.py +++ b/services/payments/tests/unit/test_db_payments_methods_repo.py @@ -65,7 +65,7 @@ async def test_create_payments_method_annotations_workflow(app: FastAPI): ) # list - listed = await repo.list_user_successful_payment_methods( + listed = await repo.list_user_payment_methods( user_id=fake.user_id, wallet_id=fake.wallet_id, ) @@ -73,7 +73,7 @@ async def test_create_payments_method_annotations_workflow(app: FastAPI): assert listed[0] == acked # get - got = await repo.get_successful_payment_method( + got = await repo.get_payment_method( payment_method_id, user_id=fake.user_id, wallet_id=fake.wallet_id, @@ -88,7 +88,7 @@ async def test_create_payments_method_annotations_workflow(app: FastAPI): ) assert deleted == got - listed = await repo.list_user_successful_payment_methods( + listed = await repo.list_user_payment_methods( user_id=fake.user_id, wallet_id=fake.wallet_id, ) From abb3dc44f77a324f793fd9fd060a8ff36cd43b73 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Nov 2023 14:36:05 +0100 Subject: [PATCH 76/78] exclude web server payments from codeclimate --- .codeclimate.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.codeclimate.yml b/.codeclimate.yml index a0936fee04a..983a132586b 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -85,3 +85,4 @@ exclude_patterns: - services/web/server/src/simcore_service_webserver/exporter/formatters/sds/xlsx/templates/code_description.py - services/web/server/src/simcore_service_webserver/projects/db.py - "scripts/" + - services/web/server/src/simcore_service_webserver/payments/_autorecharge* From 4a7fb73dae9ae9e9d6bfb098294ef2ff2b1be06e Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Nov 2023 14:45:29 +0100 Subject: [PATCH 77/78] no sonar --- .codeclimate.yml | 1 - .../src/simcore_service_payments/services/auto_recharge.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index 983a132586b..a0936fee04a 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -85,4 +85,3 @@ exclude_patterns: - services/web/server/src/simcore_service_webserver/exporter/formatters/sds/xlsx/templates/code_description.py - services/web/server/src/simcore_service_webserver/projects/db.py - "scripts/" - - services/web/server/src/simcore_service_webserver/payments/_autorecharge* diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge.py b/services/payments/src/simcore_service_payments/services/auto_recharge.py index aa72715816d..1a28fdaa870 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge.py @@ -16,7 +16,7 @@ _logger = logging.getLogger(__name__) -def _from_db_to_api_model( +def _from_db_to_api_model( # NOSONAR db_model: PaymentsAutorechargeDB, min_balance_in_credits: NonNegativeDecimal ) -> GetWalletAutoRecharge: return GetWalletAutoRecharge( @@ -28,7 +28,7 @@ def _from_db_to_api_model( ) -def _from_api_to_db_model( +def _from_api_to_db_model( # NOSONAR wallet_id: WalletID, api_model: ReplaceWalletAutoRecharge ) -> PaymentsAutorechargeDB: return PaymentsAutorechargeDB( From 1d46a7d4a0ee77d94aae3bd8c5551d9cfbf906df Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Nov 2023 15:33:05 +0100 Subject: [PATCH 78/78] no sonar --- .../src/simcore_service_payments/services/auto_recharge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/payments/src/simcore_service_payments/services/auto_recharge.py b/services/payments/src/simcore_service_payments/services/auto_recharge.py index 1a28fdaa870..aa72715816d 100644 --- a/services/payments/src/simcore_service_payments/services/auto_recharge.py +++ b/services/payments/src/simcore_service_payments/services/auto_recharge.py @@ -16,7 +16,7 @@ _logger = logging.getLogger(__name__) -def _from_db_to_api_model( # NOSONAR +def _from_db_to_api_model( db_model: PaymentsAutorechargeDB, min_balance_in_credits: NonNegativeDecimal ) -> GetWalletAutoRecharge: return GetWalletAutoRecharge( @@ -28,7 +28,7 @@ def _from_db_to_api_model( # NOSONAR ) -def _from_api_to_db_model( # NOSONAR +def _from_api_to_db_model( wallet_id: WalletID, api_model: ReplaceWalletAutoRecharge ) -> PaymentsAutorechargeDB: return PaymentsAutorechargeDB(