From b6991bd6311709010ad12db5e5ef36b71f1ea42d Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sat, 16 Nov 2024 22:59:07 +0100 Subject: [PATCH 001/113] First test for scheduler --- app/core/endpoints_core.py | 7 +++++++ app/core/log.py | 1 + app/dependencies.py | 19 +++++++++++++++---- app/types/scheduler.py | 27 +++++++++++++++++++++++++++ app/worker.py | 14 ++++++++++++++ docker-compose.yaml | 9 +++++++++ requirements-common.txt | 6 +++--- worker.sh | 5 +++++ 8 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 app/types/scheduler.py create mode 100644 app/worker.py create mode 100644 worker.sh diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index dab29ef5e..a9b1e4dbf 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -11,6 +11,7 @@ from app.core.groups.groups_type import AccountType, GroupType from app.dependencies import ( get_db, + get_scheduler, get_settings, is_user, is_user_in, @@ -32,6 +33,12 @@ async def read_information(settings: Settings = Depends(get_settings)): """ Return information about Hyperion. This endpoint can be used to check if the API is up. """ + scheduler = await get_scheduler() + + def testing_print(): + print("Super, ça marche") + + await scheduler.queue_job(testing_print, "testingprint10", 10) return schemas_core.CoreInformation( ready=True, diff --git a/app/core/log.py b/app/core/log.py index 6756dad04..2b62d3d4f 100644 --- a/app/core/log.py +++ b/app/core/log.py @@ -234,6 +234,7 @@ def get_config_dict(self, settings: Settings): ], "level": MINIMUM_LOG_LEVEL, }, + "scheduler": {"handlers": ["console"], "level": MINIMUM_LOG_LEVEL}, # We disable "uvicorn.access" to replace it with our custom "hyperion.access" which add custom information like the request_id "uvicorn.access": {"handlers": []}, "uvicorn.error": { diff --git a/app/dependencies.py b/app/dependencies.py index b7fb7d934..eb10521d5 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -13,6 +13,8 @@ async def get_users(db: AsyncSession = Depends(get_db)): from typing import Any, cast import redis +from arq import ArqRedis, create_pool +from arq.connections import RedisSettings from fastapi import BackgroundTasks, Depends, HTTPException, Request, status from sqlalchemy.ext.asyncio import ( AsyncEngine, @@ -27,15 +29,13 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.core.groups.groups_type import AccountType, GroupType, get_ecl_account_types from app.core.payment.payment_tool import PaymentTool from app.modules.raid.utils.drive.drive_file_manager import DriveFileManager +from app.types.scheduler import Scheduler from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils from app.utils.communication.notifications import NotificationManager, NotificationTool from app.utils.redis import connect -from app.utils.tools import ( - is_user_external, - is_user_member_of_an_allowed_group, -) +from app.utils.tools import is_user_external, is_user_member_of_an_allowed_group # We could maybe use hyperion.security hyperion_access_logger = logging.getLogger("hyperion.access") @@ -44,6 +44,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): redis_client: redis.Redis | bool | None = ( None # Create a global variable for the redis client, so that it can be instancied in the startup event ) +scheduler: ArqRedis | None = None # Is None if the redis client is not instantiated, is False if the redis client is instancied but not connected, is a redis.Redis object if the redis client is connected websocket_connection_manager: WebsocketConnectionManager | None = None @@ -164,6 +165,16 @@ def get_redis_client( return redis_client +async def get_scheduler() -> ArqRedis: + global scheduler + if scheduler is None: + settings = get_settings() + arq_settings = RedisSettings(host=settings.REDIS_HOST, port=settings.REDIS_PORT) + scheduler = Scheduler(await create_pool(arq_settings)) + + return scheduler + + def get_websocket_connection_manager( settings: Settings = Depends(get_settings), ): diff --git a/app/types/scheduler.py b/app/types/scheduler.py new file mode 100644 index 000000000..9a4dcc5e0 --- /dev/null +++ b/app/types/scheduler.py @@ -0,0 +1,27 @@ +import logging +from datetime import timedelta + +from arq import ArqRedis +from arq.jobs import Job + +scheduler_logger = logging.getLogger("scheduler") + + +class Scheduler: + def __init__(self, redis_pool: ArqRedis): + self.redis_pool = redis_pool + + async def queue_job(self, job_function, job_id, defer_time): + job = await self.redis_pool.enqueue_job( + "run_task", + job_function=job_function, + _job_id=job_id, + _defer_by=timedelta(seconds=defer_time), + ) + scheduler_logger.debug(f"Job queued {job}") + return job + + async def cancel_job(self, job_id): + job = Job(job_id, redis=self.redis_pool) + scheduler_logger.debug(f"Job aborted {job}") + await job.abort() diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 000000000..3573f6472 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,14 @@ +import logging + +scheduler_logger = logging.getLogger("scheduler") + + +async def run_task(ctx, job_function): + scheduler_logger.debug(f"Job function consumed {job_function}") + return await job_function() + + +class WorkerSettings: + functions = [run_task] + allow_abort_jobs = True + keep_result = 0 diff --git a/docker-compose.yaml b/docker-compose.yaml index f78b1a8d8..3200546f1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -23,6 +23,15 @@ services: command: redis-server --requirepass ${REDIS_PASSWORD} environment: - REDIS_PASSWORD=${REDIS_PASSWORD} + + hyperion-scheduler-worker: + image: hyperion-scheduler-worker + container_name: hyperion-scheduler-worker + depends_on: + - hyperion-db + - hyperion-redis + entrypoint: "./worker.sh" + env_file: .env hyperion-app: image: hyperion-app diff --git a/requirements-common.txt b/requirements-common.txt index 068b7dddb..448d12c92 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,5 +1,6 @@ aiofiles==24.1.0 # Asynchronous file manipulation alembic==1.13.2 # database migrations +arq==0.26.1 # Scheduler asyncpg==0.29.0 # PostgreSQL adapter for asynchronous operations bcrypt==4.1.3 # password hashing broadcaster==0.3.1 # Working with websockets with multiple workers. @@ -7,9 +8,8 @@ calypsso==1.2.0 fastapi[standard]==0.115.6 firebase-admin==6.5.0 # Firebase is used for push notification fpdf2==2.7.8 -HelloAssoAPIWrapper==1.0.0 -httpx==0.27.0 google-auth-oauthlib==1.2.1 +HelloAssoAPIWrapper==1.0.0 icalendar==5.0.13 jellyfish==1.0.4 # String Matching Jinja2==3.1.4 # template engine for html files @@ -24,8 +24,8 @@ python-dotenv==1.0.1 # load environment variables from .env file python-multipart==0.0.18 # a form data parser, as oauth flow requires form-data parameters redis==5.0.8 requests==2.32.3 -SQLAlchemy[asyncio]==2.0.32 # [asyncio] allows greenlet to be installed on Apple M1 devices. sqlalchemy-utils == 0.41.2 +SQLAlchemy[asyncio]==2.0.32 # [asyncio] allows greenlet to be installed on Apple M1 devices. unidecode==1.3.8 uvicorn[standard]==0.30.6 xlsxwriter==3.2.0 diff --git a/worker.sh b/worker.sh new file mode 100644 index 000000000..e3c6bb331 --- /dev/null +++ b/worker.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -e + +echo "Starting Scheduler Worker..." +exec arq app.worker.WorkerSettings From ad97e65dbda091a27085cb3981ef5f9bbc0487b7 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sat, 16 Nov 2024 23:34:04 +0100 Subject: [PATCH 002/113] Fix docker files --- Dockerfile | 3 +++ docker-compose.yaml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5c419e628..ee09330c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,4 +19,7 @@ COPY app app/ COPY start.sh . RUN chmod +x start.sh +COPY worker.sh . +RUN chmod +x worker.sh + ENTRYPOINT ["./start.sh"] diff --git a/docker-compose.yaml b/docker-compose.yaml index 3200546f1..44e570e96 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -25,7 +25,7 @@ services: - REDIS_PASSWORD=${REDIS_PASSWORD} hyperion-scheduler-worker: - image: hyperion-scheduler-worker + image: hyperion-app container_name: hyperion-scheduler-worker depends_on: - hyperion-db From bd61a1acfbfff218a40da76976335a6b07a3c281 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:04:30 +0100 Subject: [PATCH 003/113] Worker redis settings configuration --- app/worker.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/worker.py b/app/worker.py index 3573f6472..432dd98fd 100644 --- a/app/worker.py +++ b/app/worker.py @@ -1,5 +1,9 @@ import logging +from arq.connections import RedisSettings + +from app.dependencies import get_settings + scheduler_logger = logging.getLogger("scheduler") @@ -8,7 +12,15 @@ async def run_task(ctx, job_function): return await job_function() +settings = get_settings() # Ce fichier ne doit être lancé que par la prod + + class WorkerSettings: functions = [run_task] allow_abort_jobs = True keep_result = 0 + settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) From e25aa96373b55155c2bc94e13a145a3b82c4f0ea Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:10:21 +0100 Subject: [PATCH 004/113] Fixed stupid mistake --- app/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/worker.py b/app/worker.py index 432dd98fd..bdc6b4da2 100644 --- a/app/worker.py +++ b/app/worker.py @@ -19,7 +19,7 @@ class WorkerSettings: functions = [run_task] allow_abort_jobs = True keep_result = 0 - settings = RedisSettings( + redis_settings = RedisSettings( host=settings.REDIS_HOST, port=settings.REDIS_PORT, password=settings.REDIS_PASSWORD, From bad5a7ee5a745d86cffb988466df5bea84ed00ff Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:14:48 +0100 Subject: [PATCH 005/113] Fix depency scheduler --- app/dependencies.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/dependencies.py b/app/dependencies.py index eb10521d5..118905669 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -169,7 +169,11 @@ async def get_scheduler() -> ArqRedis: global scheduler if scheduler is None: settings = get_settings() - arq_settings = RedisSettings(host=settings.REDIS_HOST, port=settings.REDIS_PORT) + arq_settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) scheduler = Scheduler(await create_pool(arq_settings)) return scheduler From 6e80232125f28d9109ebbf69c45d523589c9754e Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:18:27 +0100 Subject: [PATCH 006/113] Denesting test function --- app/core/endpoints_core.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index a9b1e4dbf..1b4dbfd58 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -24,6 +24,10 @@ hyperion_error_logger = logging.getLogger("hyperion.error") +def testing_print(): + print("Super, ça marche") + + @router.get( "/information", response_model=schemas_core.CoreInformation, @@ -35,9 +39,6 @@ async def read_information(settings: Settings = Depends(get_settings)): """ scheduler = await get_scheduler() - def testing_print(): - print("Super, ça marche") - await scheduler.queue_job(testing_print, "testingprint10", 10) return schemas_core.CoreInformation( From effd99f36bc47701082eed9cf9e55ec986178ebf Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:26:40 +0100 Subject: [PATCH 007/113] Slight testing changes --- app/core/endpoints_core.py | 3 ++- app/dependencies.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index 1b4dbfd58..b2b64945f 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -1,6 +1,7 @@ import logging from os import path from pathlib import Path +from uuid import uuid4 from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import FileResponse @@ -39,7 +40,7 @@ async def read_information(settings: Settings = Depends(get_settings)): """ scheduler = await get_scheduler() - await scheduler.queue_job(testing_print, "testingprint10", 10) + await scheduler.queue_job(testing_print, uuid4(), 10) return schemas_core.CoreInformation( ready=True, diff --git a/app/dependencies.py b/app/dependencies.py index 118905669..2164cb575 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -169,6 +169,8 @@ async def get_scheduler() -> ArqRedis: global scheduler if scheduler is None: settings = get_settings() + scheduler_logger = logging.getLogger("scheduler") + scheduler_logger.debug("Initializing Scheduler") arq_settings = RedisSettings( host=settings.REDIS_HOST, port=settings.REDIS_PORT, From bfe5970154b0a54787f83a018ea473ff9227af5f Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:27:10 +0100 Subject: [PATCH 008/113] Slight testing changes --- app/types/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 9a4dcc5e0..6546ea381 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -18,7 +18,7 @@ async def queue_job(self, job_function, job_id, defer_time): _job_id=job_id, _defer_by=timedelta(seconds=defer_time), ) - scheduler_logger.debug(f"Job queued {job}") + scheduler_logger.debug(f"Job {job_id} queued {job}") return job async def cancel_job(self, job_id): From 5c78773a1b656ef962e79d7fa2d3a7bc97dc7830 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:30:07 +0100 Subject: [PATCH 009/113] Fixed uuid type --- app/core/endpoints_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index b2b64945f..f298f1407 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -40,7 +40,7 @@ async def read_information(settings: Settings = Depends(get_settings)): """ scheduler = await get_scheduler() - await scheduler.queue_job(testing_print, uuid4(), 10) + await scheduler.queue_job(testing_print, str(uuid4()), 10) return schemas_core.CoreInformation( ready=True, From c6fd66965e84d8ed3abdf1f706584f37d1718d0e Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:39:53 +0100 Subject: [PATCH 010/113] Used dependency injection --- app/core/endpoints_core.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index f298f1407..55bb0caf2 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -18,6 +18,7 @@ is_user_in, ) from app.modules.module_list import module_list +from app.types.scheduler import Scheduler from app.utils.tools import is_group_id_valid router = APIRouter(tags=["Core"]) @@ -34,12 +35,13 @@ def testing_print(): response_model=schemas_core.CoreInformation, status_code=200, ) -async def read_information(settings: Settings = Depends(get_settings)): +async def read_information( + settings: Settings = Depends(get_settings), + scheduler: Scheduler = Depends(get_scheduler), +): """ Return information about Hyperion. This endpoint can be used to check if the API is up. """ - scheduler = await get_scheduler() - await scheduler.queue_job(testing_print, str(uuid4()), 10) return schemas_core.CoreInformation( From ecf8130d65486705267d80acc37cba771ab71093 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:52:13 +0100 Subject: [PATCH 011/113] More logging --- app/dependencies.py | 1 + app/types/scheduler.py | 1 + 2 files changed, 2 insertions(+) diff --git a/app/dependencies.py b/app/dependencies.py index 2164cb575..e0e0c1ffb 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -177,6 +177,7 @@ async def get_scheduler() -> ArqRedis: password=settings.REDIS_PASSWORD, ) scheduler = Scheduler(await create_pool(arq_settings)) + scheduler_logger.debug(f"Scheduler {scheduler} initialized") return scheduler diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 6546ea381..0df60c44a 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -10,6 +10,7 @@ class Scheduler: def __init__(self, redis_pool: ArqRedis): self.redis_pool = redis_pool + scheduler_logger.debug(f"Pool in init {self.redis_pool}") async def queue_job(self, job_function, job_id, defer_time): job = await self.redis_pool.enqueue_job( From 610bc2c7e613df887d6bff6e33a08d783f9421a2 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 01:11:14 +0100 Subject: [PATCH 012/113] test --- app/core/endpoints_core.py | 1 + app/worker.py | 10 +++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index 55bb0caf2..29541053f 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -28,6 +28,7 @@ def testing_print(): print("Super, ça marche") + return 42 @router.get( diff --git a/app/worker.py b/app/worker.py index bdc6b4da2..e7efd431e 100644 --- a/app/worker.py +++ b/app/worker.py @@ -2,7 +2,7 @@ from arq.connections import RedisSettings -from app.dependencies import get_settings +from app.dependencies import get_scheduler, get_settings scheduler_logger = logging.getLogger("scheduler") @@ -18,9 +18,5 @@ async def run_task(ctx, job_function): class WorkerSettings: functions = [run_task] allow_abort_jobs = True - keep_result = 0 - redis_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) + keep_result_forever = True + redis_pool = get_scheduler().redis_pool From 2475ee5fda119e0d79c7aab22ad977350a9a25c9 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 01:21:45 +0100 Subject: [PATCH 013/113] test 2 --- app/dependencies.py | 4 +++- app/worker.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index e0e0c1ffb..42ec53656 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -165,11 +165,13 @@ def get_redis_client( return redis_client +scheduler_logger = logging.getLogger("scheduler") + + async def get_scheduler() -> ArqRedis: global scheduler if scheduler is None: settings = get_settings() - scheduler_logger = logging.getLogger("scheduler") scheduler_logger.debug("Initializing Scheduler") arq_settings = RedisSettings( host=settings.REDIS_HOST, diff --git a/app/worker.py b/app/worker.py index e7efd431e..78be2af1c 100644 --- a/app/worker.py +++ b/app/worker.py @@ -2,7 +2,7 @@ from arq.connections import RedisSettings -from app.dependencies import get_scheduler, get_settings +from app.dependencies import get_settings scheduler_logger = logging.getLogger("scheduler") @@ -19,4 +19,8 @@ class WorkerSettings: functions = [run_task] allow_abort_jobs = True keep_result_forever = True - redis_pool = get_scheduler().redis_pool + redis_settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) From 5f8516d479c79b45842df22e93eda60e632765f5 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 01:27:47 +0100 Subject: [PATCH 014/113] test 3 --- app/core/endpoints_core.py | 2 +- app/dependencies.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index 29541053f..bde7826f2 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -26,7 +26,7 @@ hyperion_error_logger = logging.getLogger("hyperion.error") -def testing_print(): +async def testing_print(): print("Super, ça marche") return 42 diff --git a/app/dependencies.py b/app/dependencies.py index 42ec53656..cfb60bda6 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -168,10 +168,9 @@ def get_redis_client( scheduler_logger = logging.getLogger("scheduler") -async def get_scheduler() -> ArqRedis: +async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: global scheduler if scheduler is None: - settings = get_settings() scheduler_logger.debug("Initializing Scheduler") arq_settings = RedisSettings( host=settings.REDIS_HOST, From 4676374b940f1f76cb676055ef93a10f35988367 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 01:55:34 +0100 Subject: [PATCH 015/113] test 4 --- app/core/endpoints_core.py | 1 + app/dependencies.py | 3 ++- app/types/scheduler.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index bde7826f2..c6a09a31c 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -44,6 +44,7 @@ async def read_information( Return information about Hyperion. This endpoint can be used to check if the API is up. """ await scheduler.queue_job(testing_print, str(uuid4()), 10) + print(scheduler) return schemas_core.CoreInformation( ready=True, diff --git a/app/dependencies.py b/app/dependencies.py index cfb60bda6..c64011ed0 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -170,6 +170,7 @@ def get_redis_client( async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: global scheduler + scheduler_logger.debug(f"Current scheduler {scheduler.name}") if scheduler is None: scheduler_logger.debug("Initializing Scheduler") arq_settings = RedisSettings( @@ -178,7 +179,7 @@ async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: password=settings.REDIS_PASSWORD, ) scheduler = Scheduler(await create_pool(arq_settings)) - scheduler_logger.debug(f"Scheduler {scheduler} initialized") + scheduler_logger.debug(f"Scheduler {scheduler.name} initialized") return scheduler diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 0df60c44a..f33ff71ae 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -1,5 +1,6 @@ import logging from datetime import timedelta +from uuid import uuid4 from arq import ArqRedis from arq.jobs import Job @@ -11,6 +12,7 @@ class Scheduler: def __init__(self, redis_pool: ArqRedis): self.redis_pool = redis_pool scheduler_logger.debug(f"Pool in init {self.redis_pool}") + name = str(uuid4()) async def queue_job(self, job_function, job_id, defer_time): job = await self.redis_pool.enqueue_job( From 05b48f4edda593a669e0087f8d6d8825ff7da300 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 01:56:55 +0100 Subject: [PATCH 016/113] test 4 fixed --- app/dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dependencies.py b/app/dependencies.py index c64011ed0..4278edf63 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -170,7 +170,7 @@ def get_redis_client( async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: global scheduler - scheduler_logger.debug(f"Current scheduler {scheduler.name}") + scheduler_logger.debug(f"Current scheduler {scheduler}") if scheduler is None: scheduler_logger.debug("Initializing Scheduler") arq_settings = RedisSettings( From 0ffda1e8780029669916083630f0bd8a3567a4ad Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 02:10:12 +0100 Subject: [PATCH 017/113] test 4 fixed 2 --- app/dependencies.py | 1 + app/types/scheduler.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/dependencies.py b/app/dependencies.py index 4278edf63..560ca010e 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -7,6 +7,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): ``` """ +import asyncio import logging from collections.abc import AsyncGenerator, Callable, Coroutine from functools import lru_cache diff --git a/app/types/scheduler.py b/app/types/scheduler.py index f33ff71ae..4bd294f08 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -12,7 +12,7 @@ class Scheduler: def __init__(self, redis_pool: ArqRedis): self.redis_pool = redis_pool scheduler_logger.debug(f"Pool in init {self.redis_pool}") - name = str(uuid4()) + self.name = str(uuid4()) async def queue_job(self, job_function, job_id, defer_time): job = await self.redis_pool.enqueue_job( From 4943520afdcc9cb04bf1907faa956a87081271d8 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 02:23:12 +0100 Subject: [PATCH 018/113] test 5 with lock --- app/dependencies.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index 560ca010e..ca8801444 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -173,14 +173,16 @@ async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: global scheduler scheduler_logger.debug(f"Current scheduler {scheduler}") if scheduler is None: - scheduler_logger.debug("Initializing Scheduler") - arq_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) - scheduler = Scheduler(await create_pool(arq_settings)) - scheduler_logger.debug(f"Scheduler {scheduler.name} initialized") + lock = asyncio.Lock() + async with lock: + scheduler_logger.debug("Initializing Scheduler") + arq_settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) + scheduler = Scheduler(await create_pool(arq_settings)) + scheduler_logger.debug(f"Scheduler {scheduler.name} initialized") return scheduler From 79af08cf5e7775f8246bb669a660037bd4587b96 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 02:32:31 +0100 Subject: [PATCH 019/113] test 6 with enter --- app/types/scheduler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 4bd294f08..6f1e35c1f 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -14,6 +14,9 @@ def __init__(self, redis_pool: ArqRedis): scheduler_logger.debug(f"Pool in init {self.redis_pool}") self.name = str(uuid4()) + def __enter__(self): + return self + async def queue_job(self, job_function, job_id, defer_time): job = await self.redis_pool.enqueue_job( "run_task", From 91625ec6ea206cc2152957eb0cea3f9e9ed9a857 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 02:48:41 +0100 Subject: [PATCH 020/113] Test 6 --- app/dependencies.py | 20 +++++++++----------- app/types/scheduler.py | 20 ++++++++++++++------ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index ca8801444..6d0245b9c 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -30,7 +30,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.core.groups.groups_type import AccountType, GroupType, get_ecl_account_types from app.core.payment.payment_tool import PaymentTool from app.modules.raid.utils.drive.drive_file_manager import DriveFileManager -from app.types.scheduler import Scheduler +from app.types.scheduler import Scheduler, create_scheduler from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils @@ -173,16 +173,14 @@ async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: global scheduler scheduler_logger.debug(f"Current scheduler {scheduler}") if scheduler is None: - lock = asyncio.Lock() - async with lock: - scheduler_logger.debug("Initializing Scheduler") - arq_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) - scheduler = Scheduler(await create_pool(arq_settings)) - scheduler_logger.debug(f"Scheduler {scheduler.name} initialized") + scheduler_logger.debug("Initializing Scheduler") + arq_settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) + scheduler = await create_scheduler(arq_settings) + scheduler_logger.debug(f"Scheduler {scheduler} initialized") return scheduler diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 6f1e35c1f..cc56ac611 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -2,20 +2,28 @@ from datetime import timedelta from uuid import uuid4 -from arq import ArqRedis +from arq import create_pool from arq.jobs import Job scheduler_logger = logging.getLogger("scheduler") +async def create_scheduler(settings): + scheduler = Scheduler(settings) + await scheduler.async_init() + return scheduler + + class Scheduler: - def __init__(self, redis_pool: ArqRedis): - self.redis_pool = redis_pool + def __init__(self, redis_settings): + self.settings = redis_settings + + async def async_init(self): + self.redis_pool = await create_pool(self.settings) scheduler_logger.debug(f"Pool in init {self.redis_pool}") - self.name = str(uuid4()) - def __enter__(self): - return self + def __del__(self): + scheduler_logger.debug(f"Del here with {self.redis_pool}") async def queue_job(self, job_function, job_id, defer_time): job = await self.redis_pool.enqueue_job( From b071bf0a0e2a37de0a8d19cb3447d93118acf179 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 03:01:44 +0100 Subject: [PATCH 021/113] Test_6 --- app/types/scheduler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index cc56ac611..c39b52fbb 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -15,8 +15,11 @@ async def create_scheduler(settings): class Scheduler: + current_instance = None # Trying to fool the garbage collector + def __init__(self, redis_settings): self.settings = redis_settings + self.current_instance = self async def async_init(self): self.redis_pool = await create_pool(self.settings) From f5e74d7467037582465cce3ed97496328560ec48 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 03:28:12 +0100 Subject: [PATCH 022/113] Working scheduler functions --- app/core/endpoints_core.py | 2 +- app/types/scheduler.py | 42 ++++++++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index c6a09a31c..6a5c3eba9 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -43,7 +43,7 @@ async def read_information( """ Return information about Hyperion. This endpoint can be used to check if the API is up. """ - await scheduler.queue_job(testing_print, str(uuid4()), 10) + await scheduler.queue_job_time_defer(testing_print, str(uuid4()), 10) print(scheduler) return schemas_core.CoreInformation( diff --git a/app/types/scheduler.py b/app/types/scheduler.py index c39b52fbb..d50e6cf3f 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -1,5 +1,6 @@ import logging -from datetime import timedelta +from datetime import datetime, timedelta +from typing import Any, Callable, Coroutine from uuid import uuid4 from arq import create_pool @@ -15,30 +16,55 @@ async def create_scheduler(settings): class Scheduler: - current_instance = None # Trying to fool the garbage collector - def __init__(self, redis_settings): self.settings = redis_settings - self.current_instance = self async def async_init(self): self.redis_pool = await create_pool(self.settings) scheduler_logger.debug(f"Pool in init {self.redis_pool}") - def __del__(self): - scheduler_logger.debug(f"Del here with {self.redis_pool}") + async def queue_job_time_defer( + self, + job_function: Callable[..., Coroutine[Any, Any, Any]], + job_id: str, + defer_seconds: float, + ): + """ + Queue a job to execute job_function in defer_seconds amount of seconds + job_id will allow to abort if needed + """ + job = await self.redis_pool.enqueue_job( + "run_task", + job_function=job_function, + _job_id=job_id, + _defer_by=timedelta(seconds=defer_seconds), + ) + scheduler_logger.debug(f"Job {job_id} queued {job}") + return job - async def queue_job(self, job_function, job_id, defer_time): + async def queue_job_defer_to( + self, + job_function: Callable[..., Coroutine[Any, Any, Any]], + job_id: str, + defer_date: datetime, + ): + """ + Queue a job to execute job_function at defer_date + job_id will allow to abort if needed + """ job = await self.redis_pool.enqueue_job( "run_task", job_function=job_function, _job_id=job_id, - _defer_by=timedelta(seconds=defer_time), + _defer_until=defer_date, ) scheduler_logger.debug(f"Job {job_id} queued {job}") return job async def cancel_job(self, job_id): + """ + cancel a queued job based on its job_id + """ job = Job(job_id, redis=self.redis_pool) scheduler_logger.debug(f"Job aborted {job}") await job.abort() From 58f7f172db53bbb491970be1f4ca452542c0ca95 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 03:28:17 +0100 Subject: [PATCH 023/113] Working scheduler functions --- app/types/scheduler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index d50e6cf3f..4e43a3dd5 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -16,6 +16,8 @@ async def create_scheduler(settings): class Scheduler: + """Disappears sometimes for no reason""" + def __init__(self, redis_settings): self.settings = redis_settings From c899e6a30df51ef6388b8aa17a6aa8fe9c74bc21 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 04:41:36 +0100 Subject: [PATCH 024/113] fun test --- app/types/scheduler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 4e43a3dd5..9b29c1e64 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -18,8 +18,12 @@ async def create_scheduler(settings): class Scheduler: """Disappears sometimes for no reason""" + all_instances = [] + def __init__(self, redis_settings): self.settings = redis_settings + self.all_instances.append(self) + scheduler_logger.debug(f"Other instances {self.all_instances}") async def async_init(self): self.redis_pool = await create_pool(self.settings) From d19479b44e039e1b7876b031a38a78d228feb7f7 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 05:57:59 +0100 Subject: [PATCH 025/113] Final working config for scheduler --- app/core/endpoints_core.py | 7 ------- app/dependencies.py | 23 +++++++++++------------ app/types/scheduler.py | 27 ++++++++------------------- app/utils/redis.py | 1 + app/worker.py | 4 ++-- 5 files changed, 22 insertions(+), 40 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index 6a5c3eba9..ea1bdd34a 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -26,11 +26,6 @@ hyperion_error_logger = logging.getLogger("hyperion.error") -async def testing_print(): - print("Super, ça marche") - return 42 - - @router.get( "/information", response_model=schemas_core.CoreInformation, @@ -43,8 +38,6 @@ async def read_information( """ Return information about Hyperion. This endpoint can be used to check if the API is up. """ - await scheduler.queue_job_time_defer(testing_print, str(uuid4()), 10) - print(scheduler) return schemas_core.CoreInformation( ready=True, diff --git a/app/dependencies.py b/app/dependencies.py index 6d0245b9c..2ead1cf55 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -7,7 +7,6 @@ async def get_users(db: AsyncSession = Depends(get_db)): ``` """ -import asyncio import logging from collections.abc import AsyncGenerator, Callable, Coroutine from functools import lru_cache @@ -30,7 +29,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.core.groups.groups_type import AccountType, GroupType, get_ecl_account_types from app.core.payment.payment_tool import PaymentTool from app.modules.raid.utils.drive.drive_file_manager import DriveFileManager -from app.types.scheduler import Scheduler, create_scheduler +from app.types.scheduler import Scheduler from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils @@ -169,18 +168,18 @@ def get_redis_client( scheduler_logger = logging.getLogger("scheduler") -async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: +async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis | None: global scheduler - scheduler_logger.debug(f"Current scheduler {scheduler}") if scheduler is None: - scheduler_logger.debug("Initializing Scheduler") - arq_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) - scheduler = await create_scheduler(arq_settings) - scheduler_logger.debug(f"Scheduler {scheduler} initialized") + if settings.REDIS_HOST != "": + scheduler_logger.debug("Initializing Scheduler") + arq_settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) + scheduler = Scheduler(await create_pool(arq_settings)) + scheduler_logger.debug(f"Scheduler {scheduler} initialized") return scheduler diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 9b29c1e64..aa72dba7a 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -1,39 +1,25 @@ import logging +from collections.abc import Callable, Coroutine from datetime import datetime, timedelta -from typing import Any, Callable, Coroutine -from uuid import uuid4 +from typing import Any -from arq import create_pool from arq.jobs import Job scheduler_logger = logging.getLogger("scheduler") -async def create_scheduler(settings): - scheduler = Scheduler(settings) - await scheduler.async_init() - return scheduler - - class Scheduler: """Disappears sometimes for no reason""" - all_instances = [] - - def __init__(self, redis_settings): - self.settings = redis_settings - self.all_instances.append(self) - scheduler_logger.debug(f"Other instances {self.all_instances}") - - async def async_init(self): - self.redis_pool = await create_pool(self.settings) - scheduler_logger.debug(f"Pool in init {self.redis_pool}") + def __init__(self, redis_pool): + self.redis_pool = redis_pool async def queue_job_time_defer( self, job_function: Callable[..., Coroutine[Any, Any, Any]], job_id: str, defer_seconds: float, + **kwargs, ): """ Queue a job to execute job_function in defer_seconds amount of seconds @@ -44,6 +30,7 @@ async def queue_job_time_defer( job_function=job_function, _job_id=job_id, _defer_by=timedelta(seconds=defer_seconds), + **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") return job @@ -53,6 +40,7 @@ async def queue_job_defer_to( job_function: Callable[..., Coroutine[Any, Any, Any]], job_id: str, defer_date: datetime, + **kwargs, ): """ Queue a job to execute job_function at defer_date @@ -63,6 +51,7 @@ async def queue_job_defer_to( job_function=job_function, _job_id=job_id, _defer_until=defer_date, + **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") return job diff --git a/app/utils/redis.py b/app/utils/redis.py index 44d78aa5d..ad6d98440 100644 --- a/app/utils/redis.py +++ b/app/utils/redis.py @@ -8,6 +8,7 @@ def connect(settings: Settings) -> redis.Redis | bool: host=settings.REDIS_HOST, port=settings.REDIS_PORT, password=settings.REDIS_PASSWORD, + socket_keepalive=True, ) redis_client.ping() # Test the connection diff --git a/app/worker.py b/app/worker.py index 78be2af1c..f9407882a 100644 --- a/app/worker.py +++ b/app/worker.py @@ -7,9 +7,9 @@ scheduler_logger = logging.getLogger("scheduler") -async def run_task(ctx, job_function): +async def run_task(ctx, job_function, **kwargs): scheduler_logger.debug(f"Job function consumed {job_function}") - return await job_function() + return await job_function(**kwargs) settings = get_settings() # Ce fichier ne doit être lancé que par la prod From 5390075cc178f1d97175e8fd88967fac909e4774 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sat, 26 Oct 2024 23:08:20 +0200 Subject: [PATCH 026/113] First clean --- app/core/notification/cruds_notification.py | 61 --------------------- app/utils/communication/notifications.py | 21 +++---- 2 files changed, 11 insertions(+), 71 deletions(-) diff --git a/app/core/notification/cruds_notification.py b/app/core/notification/cruds_notification.py index e59fe8f18..30d9edceb 100644 --- a/app/core/notification/cruds_notification.py +++ b/app/core/notification/cruds_notification.py @@ -9,16 +9,6 @@ from app.core.notification.notification_types import CustomTopic, Topic -async def create_message( - message: models_notification.Message, - db: AsyncSession, -) -> None: - db.add(message) - try: - await db.commit() - except IntegrityError: - await db.rollback() - raise async def create_batch_messages( @@ -33,57 +23,6 @@ async def create_batch_messages( raise -async def get_messages_by_firebase_token( - firebase_token: str, - db: AsyncSession, -) -> Sequence[models_notification.Message]: - result = await db.execute( - select(models_notification.Message).where( - models_notification.Message.firebase_device_token == firebase_token, - ), - ) - return result.scalars().all() - - -async def get_messages_by_context_and_firebase_tokens( - context: str, - firebase_tokens: list[str], - db: AsyncSession, -) -> Sequence[models_notification.Message]: - result = await db.execute( - select(models_notification.Message).where( - models_notification.Message.context == context, - models_notification.Message.firebase_device_token.in_(firebase_tokens), - ), - ) - return result.scalars().all() - - -async def remove_message_by_context_and_firebase_device_token( - context: str, - firebase_device_token: str, - db: AsyncSession, -): - await db.execute( - delete(models_notification.Message).where( - models_notification.Message.context == context, - models_notification.Message.firebase_device_token == firebase_device_token, - ), - ) - await db.commit() - - -async def remove_message_by_firebase_device_token( - firebase_device_token: str, - db: AsyncSession, -): - await db.execute( - delete(models_notification.Message).where( - models_notification.Message.firebase_device_token == firebase_device_token, - ), - ) - await db.commit() - async def remove_messages_by_context_and_firebase_device_tokens_list( context: str, diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index ad83015bb..5225a385d 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -75,6 +75,7 @@ async def _send_firebase_push_notification_by_tokens( self, db: AsyncSession, tokens: list[str], + message_content: Message, ): """ Send a firebase push notification to a list of tokens. @@ -96,6 +97,7 @@ async def _send_firebase_push_notification_by_tokens( await self._send_firebase_push_notification_by_tokens( tokens=tokens[500:], db=db, + message_content=message_content, ) tokens = tokens[:500] @@ -105,18 +107,15 @@ async def _send_firebase_push_notification_by_tokens( # This allow to ensure that the notification will be processed in the background # See https://firebase.google.com/docs/cloud-messaging/concept-options#setting-the-priority-of-a-message # And https://developer.apple.com/documentation/usernotifications/setting_up_a_remote_notification_server/pushing_background_updates_to_your_app - androidconfig = messaging.AndroidConfig(priority="high") - apnsconfig = messaging.APNSConfig( - headers={"apns-priority": "10", "apns-push-type": "background"}, - payload=messaging.APNSPayload( - aps=messaging.Aps(content_available=True), - ), - ) + message = messaging.MulticastMessage( tokens=tokens, - android=androidconfig, - apns=apnsconfig, + notification=messaging.Notification( + title=message_content.title, + body=message_content.content, + ), ) + result = messaging.send_each_for_multicast(message) except Exception: hyperion_error_logger.exception( @@ -133,6 +132,7 @@ async def _send_firebase_trigger_notification_by_tokens( self, db: AsyncSession, tokens: list[str], + message_content: Message, ): """ Send a firebase trigger notification to a list of tokens. @@ -143,7 +143,7 @@ async def _send_firebase_trigger_notification_by_tokens( # Push without any data or notification may not be processed by the app in the background. # We thus need to send a data object with a dummy key to make sure the notification is processed. # See https://stackoverflow.com/questions/59298850/firebase-messaging-background-message-handler-method-not-called-when-the-app - await self._send_firebase_push_notification_by_tokens(tokens=tokens, db=db) + await self._send_firebase_push_notification_by_tokens(tokens=tokens, db=db, message_content=message_content) async def _add_message_for_user_in_database( self, @@ -211,6 +211,7 @@ async def send_notification_to_users( await self._send_firebase_trigger_notification_by_tokens( tokens=firebase_device_tokens, db=db, + message_content=message, ) except Exception as error: hyperion_error_logger.warning( From 0934b947cf14805404fa8b329d92402abfe3a120 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sat, 16 Nov 2024 12:44:26 +0100 Subject: [PATCH 027/113] RAF --- app/core/notification/cruds_notification.py | 30 ----- .../notification/endpoints_notification.py | 66 ++++------ app/core/notification/notification_types.py | 1 + app/utils/communication/notifications.py | 113 +++++++++--------- 4 files changed, 83 insertions(+), 127 deletions(-) diff --git a/app/core/notification/cruds_notification.py b/app/core/notification/cruds_notification.py index 30d9edceb..4c6e82a6d 100644 --- a/app/core/notification/cruds_notification.py +++ b/app/core/notification/cruds_notification.py @@ -8,36 +8,6 @@ from app.core.notification import models_notification from app.core.notification.notification_types import CustomTopic, Topic - - - -async def create_batch_messages( - messages: list[models_notification.Message], - db: AsyncSession, -) -> None: - db.add_all(messages) - try: - await db.commit() - except IntegrityError: - await db.rollback() - raise - - - -async def remove_messages_by_context_and_firebase_device_tokens_list( - context: str, - tokens: list[str], - db: AsyncSession, -): - await db.execute( - delete(models_notification.Message).where( - models_notification.Message.context == context, - models_notification.Message.firebase_device_token.in_(tokens), - ), - ) - await db.commit() - - async def get_firebase_devices_by_user_id( user_id: str, db: AsyncSession, diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index c59f8cb5c..932e3a54b 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -96,47 +96,6 @@ async def unregister_firebase_device( ) -@router.get( - "/notification/messages/{firebase_token}", - response_model=list[schemas_notification.Message], - status_code=200, -) -async def get_messages( - firebase_token: str, - db: AsyncSession = Depends(get_db), - # If we want to enable authentification for /messages/{firebase_token} endpoint, we may to uncomment the following line - # user: models_core.CoreUser = Depends(is_user()), -): - """ - Get all messages for a specific device from the user - - **The user must be authenticated to use this endpoint** - """ - firebase_device = await cruds_notification.get_firebase_devices_by_user_id_and_firebase_token( - # If we want to enable authentification for /messages/{firebase_token} endpoint, we may to uncomment the following line - firebase_token=firebase_token, - db=db, # user_id=user.id, - ) - - if firebase_device is None: - raise HTTPException( - status_code=404, - detail="Device not found for user", # {user.id}" - ) - - messages = await cruds_notification.get_messages_by_firebase_token( - firebase_token=firebase_token, - db=db, - ) - - await cruds_notification.remove_message_by_firebase_device_token( - firebase_device_token=firebase_token, - db=db, - ) - - return messages - - @router.post( "/notification/topics/{topic_str}/subscribe", status_code=204, @@ -280,6 +239,31 @@ async def send_notification( message=message, ) +@router.post( + "/notification/send/topic", + status_code=201, +) +async def send_notification_topic( + user: models_core.CoreUser = Depends(is_user_a_member_of(GroupType.admin)), + notification_tool: NotificationTool = Depends(get_notification_tool), +): + """ + Send ourself a test notification. + + **Only admins can use this endpoint** + """ + message = schemas_notification.Message( + context="notification-test-topic", + is_visible=True, + title="Test notification topic", + content="Ceci est un test de notification topic", + # The notification will expire in 3 days + expire_on=datetime.now(UTC) + timedelta(days=3), + ) + await notification_tool.send_notification_to_topic( + custom_topic=CustomTopic.from_str("test"), + message=message, + ) @router.post( "/notification/send/future", diff --git a/app/core/notification/notification_types.py b/app/core/notification/notification_types.py index 745b005dc..ae081524b 100644 --- a/app/core/notification/notification_types.py +++ b/app/core/notification/notification_types.py @@ -15,6 +15,7 @@ class Topic(str, Enum): raffle = "raffle" vote = "vote" ph = "ph" + test = "test" class CustomTopic: diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 5225a385d..aec9d0de3 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -103,11 +103,6 @@ async def _send_firebase_push_notification_by_tokens( # We may pass a notification object along the data try: - # Set high priority for android, and background notification for iOS - # This allow to ensure that the notification will be processed in the background - # See https://firebase.google.com/docs/cloud-messaging/concept-options#setting-the-priority-of-a-message - # And https://developer.apple.com/documentation/usernotifications/setting_up_a_remote_notification_server/pushing_background_updates_to_your_app - message = messaging.MulticastMessage( tokens=tokens, notification=messaging.Notification( @@ -128,46 +123,62 @@ async def _send_firebase_push_notification_by_tokens( db=db, ) - async def _send_firebase_trigger_notification_by_tokens( + def _send_firebase_push_notification_by_topic( self, - db: AsyncSession, - tokens: list[str], + custom_topic: CustomTopic, message_content: Message, ): """ - Send a firebase trigger notification to a list of tokens. - This approach let the application know that a new notification is available, - without sending the content of the notification. - This is better for privacy and RGPD compliance. + Send a firebase push notification for a given topic. + Prefer using `self._send_firebase_trigger_notification_by_topic` to send a trigger notification. """ - # Push without any data or notification may not be processed by the app in the background. - # We thus need to send a data object with a dummy key to make sure the notification is processed. - # See https://stackoverflow.com/questions/59298850/firebase-messaging-background-message-handler-method-not-called-when-the-app - await self._send_firebase_push_notification_by_tokens(tokens=tokens, db=db, message_content=message_content) - async def _add_message_for_user_in_database( + if not self.use_firebase: + return + message = messaging.Message( + topic=custom_topic.to_str(), + notification=messaging.Notification( + title=message_content.title, + body=message_content.content, + ), + ) + try: + messaging.send(message) + except messaging.FirebaseError as error: + hyperion_error_logger.error( + f"Notification: Unable to send firebase notification for topic {custom_topic}: {error}", + ) + raise + + async def subscribe_tokens_to_topic( self, - message: Message, + custom_topic: CustomTopic, tokens: list[str], - db: AsyncSession, - ) -> None: - message_models = [ - models_notification.Message( - firebase_device_token=token, - **message.model_dump(), + ): + """ + Subscribe a list of tokens to a given topic. + """ + if not self.use_firebase: + return + + response = messaging.subscribe_to_topic(tokens, custom_topic.to_str()) + if response.failure_count > 0: + hyperion_error_logger.info( + f"Notification: Failed to subscribe to topic {custom_topic} due to {[error.reason for error in response.errors]}", ) - for token in tokens - ] - # We need to remove old messages with the same context and token - # as there can only be one message per context and token - await cruds_notification.remove_messages_by_context_and_firebase_device_tokens_list( - context=message.context, - tokens=tokens, - db=db, - ) + async def unsubscribe_tokens_to_topic( + self, + custom_topic: CustomTopic, + tokens: list[str], + ): + """ + Unsubscribe a list of tokens to a given topic. + """ + if not self.use_firebase: + return - await cruds_notification.create_batch_messages(messages=message_models, db=db) + messaging.unsubscribe_from_topic(tokens, custom_topic.to_str()) async def send_notification_to_users( self, @@ -196,19 +207,8 @@ async def send_notification_to_users( ) ) - if len(firebase_device_tokens) == 0: - hyperion_error_logger.warning( - f"Notification: No firebase device token found for the message {message.title} {message.content}", - ) - - await self._add_message_for_user_in_database( - message=message, - tokens=firebase_device_tokens, - db=db, - ) - try: - await self._send_firebase_trigger_notification_by_tokens( + await self._send_firebase_push_notification_by_tokens( tokens=firebase_device_tokens, db=db, message_content=message, @@ -237,15 +237,12 @@ async def send_notification_to_topic( ) return - user_ids = await cruds_notification.get_user_ids_by_topic( - custom_topic=custom_topic, - db=db, - ) - await self.send_notification_to_users( - user_ids=user_ids, - message=message, - db=db, - ) + try: + self._send_firebase_push_notification_by_topic(custom_topic=custom_topic, message_content=message) + except Exception as error: + hyperion_error_logger.warning( + f"Notification: Unable to send firebase notification for topic {custom_topic}: {error}", + ) async def subscribe_user_to_topic( self, @@ -275,6 +272,9 @@ async def subscribe_user_to_topic( topic_membership=topic_membership, db=db, ) + tokens = await cruds_notification.get_firebase_tokens_by_user_ids(user_ids=[user_id], db=db) + await self.subscribe_tokens_to_topic(custom_topic=custom_topic, tokens=tokens) + async def unsubscribe_user_to_topic( self, @@ -290,7 +290,8 @@ async def unsubscribe_user_to_topic( user_id=user_id, db=db, ) - + tokens = await cruds_notification.get_firebase_tokens_by_user_ids(user_ids=[user_id], db=db) + await self.unsubscribe_tokens_to_topic(custom_topic=custom_topic, tokens=tokens) class NotificationTool: """ From 2e01c3e1b511e966bfea16315bd64de95a500e09 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sat, 16 Nov 2024 18:00:55 +0100 Subject: [PATCH 028/113] Edit message content for new notif system --- .../notification/endpoints_notification.py | 17 ++++----------- app/core/notification/schemas_notification.py | 21 +------------------ app/modules/advert/endpoints_advert.py | 5 +---- app/modules/amap/endpoints_amap.py | 10 ++------- app/modules/booking/endpoints_booking.py | 5 +---- app/modules/cinema/endpoints_cinema.py | 7 +------ app/modules/loan/endpoints_loan.py | 18 ++++------------ app/modules/ph/endpoints_ph.py | 9 +------- app/utils/communication/notifications.py | 2 +- 9 files changed, 16 insertions(+), 78 deletions(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index 932e3a54b..d8e04eced 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -227,12 +227,10 @@ async def send_notification( **Only admins can use this endpoint** """ message = schemas_notification.Message( - context="notification-test", - is_visible=True, title="Test notification", content="Ceci est un test de notification", - # The notification will expire in 3 days - expire_on=datetime.now(UTC) + timedelta(days=3), + action_module="test", + ) await notification_tool.send_notification_to_user( user_id=user.id, @@ -253,12 +251,9 @@ async def send_notification_topic( **Only admins can use this endpoint** """ message = schemas_notification.Message( - context="notification-test-topic", - is_visible=True, title="Test notification topic", content="Ceci est un test de notification topic", - # The notification will expire in 3 days - expire_on=datetime.now(UTC) + timedelta(days=3), + action_module="test", ) await notification_tool.send_notification_to_topic( custom_topic=CustomTopic.from_str("test"), @@ -279,13 +274,9 @@ async def send_future_notification( **Only admins can use this endpoint** """ message = schemas_notification.Message( - context="future-notification-test", - is_visible=True, title="Test notification future", content="Ceci est un test de notification future", - # The notification will expire in 3 days - expire_on=datetime.now(UTC) + timedelta(days=3), - delivery_datetime=datetime.now(UTC) + timedelta(minutes=3), + action_module="test", ) await notification_tool.send_notification_to_user( user_id=user.id, diff --git a/app/core/notification/schemas_notification.py b/app/core/notification/schemas_notification.py index 633b3c164..b88833416 100644 --- a/app/core/notification/schemas_notification.py +++ b/app/core/notification/schemas_notification.py @@ -4,17 +4,6 @@ class Message(BaseModel): - # A context represents a topic (ex: a loan), - # there can only by one notification per context (ex: the loan should be returned, the loan is overdue or has been returned) - context: str = Field( - description="A context represents a topic. There can only by one notification per context.", - ) - # `firebase_device_token` is only contained in the database but is not returned by the API - - is_visible: bool = Field( - description="A message can be visible or not, if it is not visible, it should only trigger an action", - ) - title: str | None = None content: str | None = None @@ -22,17 +11,9 @@ class Message(BaseModel): None, description="An identifier for the module that should be triggered when the notification is clicked", ) - action_table: str | None = None - - delivery_datetime: datetime | None = Field( - None, - description="The date the notification should be shown", - ) - expire_on: datetime - model_config = ConfigDict(from_attributes=True) + class FirebaseDevice(BaseModel): user_id: str = Field(description="The Hyperion user id") firebase_device_token: str = Field("Firebase device token") - model_config = ConfigDict(from_attributes=True) diff --git a/app/modules/advert/endpoints_advert.py b/app/modules/advert/endpoints_advert.py index 093859a93..070b0a2e3 100644 --- a/app/modules/advert/endpoints_advert.py +++ b/app/modules/advert/endpoints_advert.py @@ -266,12 +266,9 @@ async def create_advert( except ValueError as error: raise HTTPException(status_code=400, detail=str(error)) message = Message( - context=f"advert-new-{id}", - is_visible=True, title=f"📣 Annonce - {result.title}", content=result.content, - # The notification will expire in 3 days - expire_on=datetime.now(UTC) + timedelta(days=3), + action_module="advert", ) await notification_tool.send_notification_to_topic( diff --git a/app/modules/amap/endpoints_amap.py b/app/modules/amap/endpoints_amap.py index 24eef6478..e1380318c 100644 --- a/app/modules/amap/endpoints_amap.py +++ b/app/modules/amap/endpoints_amap.py @@ -719,12 +719,9 @@ async def open_ordering_of_delivery( await cruds_amap.open_ordering_of_delivery(delivery_id=delivery_id, db=db) message = Message( - context=f"amap-open-ordering-{delivery_id}", - is_visible=True, title="🛒 AMAP - Nouvelle livraison disponible", content="Viens commander !", - # The notification will expire in 3 days - expire_on=datetime.now(UTC) + timedelta(days=3), + action_module="amap", ) await notification_tool.send_notification_to_topic( custom_topic=CustomTopic(Topic.amap), @@ -904,12 +901,9 @@ async def create_cash_of_user( ) message = Message( - context=f"amap-cash-{user_id}", - is_visible=True, title="AMAP - Solde mis à jour", content=f"Votre nouveau solde est de {cash} €.", - # The notification will expire in 3 days - expire_on=datetime.now(UTC) + timedelta(days=3), + action_module="amap", ) await notification_tool.send_notification_to_user( user_id=user_id, diff --git a/app/modules/booking/endpoints_booking.py b/app/modules/booking/endpoints_booking.py index 35eb5cc3c..bb44350bc 100644 --- a/app/modules/booking/endpoints_booking.py +++ b/app/modules/booking/endpoints_booking.py @@ -277,12 +277,9 @@ async def create_booking( if manager_group: message = Message( - context=f"booking-new-{id}", - is_visible=True, title="📅 Réservations - Nouvelle réservation", content=content, - # The notification will expire in 3 days - expire_on=datetime.now(UTC) + timedelta(days=3), + module="booking", ) await notification_tool.send_notification_to_users( diff --git a/app/modules/cinema/endpoints_cinema.py b/app/modules/cinema/endpoints_cinema.py index 8fa931e4d..e7118b90d 100644 --- a/app/modules/cinema/endpoints_cinema.py +++ b/app/modules/cinema/endpoints_cinema.py @@ -141,14 +141,9 @@ async def create_session( for next_session in next_week_sessions: message_content += f"{get_date_day(next_session.start)} {next_session.start.day} {get_date_month(next_session.start)} - {next_session.name}\n" message = Message( - # We use sunday date as context to avoid sending the recap twice - context=f"cinema-recap-{sunday}", - is_visible=True, title="🎬 Cinéma - Programme de la semaine", content=message_content, - delivery_datetime=sunday, - # The notification will expire the next sunday - expire_on=sunday + timedelta(days=7), + action_module="cinema", ) await notification_tool.send_notification_to_topic( diff --git a/app/modules/loan/endpoints_loan.py b/app/modules/loan/endpoints_loan.py index 995ed1056..1c7b1dc95 100644 --- a/app/modules/loan/endpoints_loan.py +++ b/app/modules/loan/endpoints_loan.py @@ -613,11 +613,9 @@ async def create_loan( ] message = Message( - context=f"loan-new-{loan.id}-begin-notif", - is_visible=True, title="📦 Nouveau prêt", content=f"Un prêt a été enregistré pour l'association {loan.loaner.name}", - expire_on=datetime.now(UTC) + timedelta(days=3), + module="loan", ) await notification_tool.send_notification_to_user( user_id=loan.borrower_id, @@ -629,12 +627,9 @@ async def create_loan( expire_on_date = loan.end + timedelta(days=30) expire_on_datetime = datetime.combine(expire_on_date, delivery_time, tzinfo=UTC) message = Message( - context=f"loan-new-{loan.id}-end-notif", - is_visible=True, title="📦 Prêt arrivé à échéance", content=f"N'oublie pas de rendre ton prêt à l'association {loan.loaner.name} !", - delivery_datetime=delivery_datetime, - expire_on=expire_on_datetime, + module="loan", ) await notification_tool.send_notification_to_user( @@ -874,11 +869,9 @@ async def return_loan( ) message = Message( - context=f"loan-new-{loan.id}-end-notif", - is_visible=False, title="", content="", - expire_on=datetime.now(UTC) + timedelta(days=3), + module="", ) await notification_tool.send_notification_to_user( user_id=loan.borrower_id, @@ -937,12 +930,9 @@ async def extend_loan( ) # same context so the first notification will be removed message = Message( - context=f"loan-new-{loan.id}-end-notif", - is_visible=True, title="📦 Prêt arrivé à échéance", content=f"N'oublie pas de rendre ton prêt à l'association {loan.loaner.name} ! ", - delivery_datetime=loan.end, - expire_on=loan.end + timedelta(days=30), + action_module="loan", ) await notification_tool.send_notification_to_user( diff --git a/app/modules/ph/endpoints_ph.py b/app/modules/ph/endpoints_ph.py index e983f1bda..15dbd572a 100644 --- a/app/modules/ph/endpoints_ph.py +++ b/app/modules/ph/endpoints_ph.py @@ -124,16 +124,9 @@ async def create_paper( # We only want to send a notification if the paper was released less than a month ago. if paper_db.release_date >= now.date() - timedelta(days=30): message = Message( - context=f"ph-{paper_db.id}", - is_visible=True, title=f"📗 PH - {paper_db.name}", content="Un nouveau journal est disponible! 🎉", - delivery_datetime=datetime.combine( - paper_db.release_date, - time(hour=8, tzinfo=UTC), - ), - # The notification will expire in 10 days - expire_on=now + timedelta(days=10), + action_module="ph", ) await notification_tool.send_notification_to_topic( custom_topic=CustomTopic(topic=Topic.ph), diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index aec9d0de3..09145e109 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -101,10 +101,10 @@ async def _send_firebase_push_notification_by_tokens( ) tokens = tokens[:500] - # We may pass a notification object along the data try: message = messaging.MulticastMessage( tokens=tokens, + data={"module":message_content.action_module}, notification=messaging.Notification( title=message_content.title, body=message_content.content, From d595a0a334810afe23ef18a2b82c254f8bed79bb Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 03:59:52 +0100 Subject: [PATCH 029/113] Test future notif --- .../notification/endpoints_notification.py | 4 ++- app/utils/communication/notifications.py | 31 ++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index d8e04eced..f844ef02d 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -278,9 +278,11 @@ async def send_future_notification( content="Ceci est un test de notification future", action_module="test", ) - await notification_tool.send_notification_to_user( + await notification_tool.send_future_notification_to_user( user_id=user.id, message=message, + defer_date=datetime.now() + timedelta(minutes=3), + job_id = "test-notification" ) diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 09145e109..b6f2c70ac 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -1,7 +1,8 @@ +from datetime import datetime import logging import firebase_admin -from fastapi import BackgroundTasks +from fastapi import BackgroundTasks, Depends from firebase_admin import credentials, messaging from sqlalchemy.ext.asyncio import AsyncSession @@ -9,6 +10,8 @@ from app.core.notification import cruds_notification, models_notification from app.core.notification.notification_types import CustomTopic from app.core.notification.schemas_notification import Message +from app.dependencies import get_scheduler +from app.types.scheduler import Scheduler hyperion_error_logger = logging.getLogger("hyperion.error") @@ -341,3 +344,29 @@ async def send_notification_to_topic( message=message, db=self.db, ) + + async def send_future_notification_to_user( + self, + user_id: str, + message: Message, + defer_date: datetime, + job_id: str, + scheduler: Scheduler = Depends(get_scheduler), + ) -> None: + + await scheduler.queue_job_defer_to(self.send_notification_to_users, + user_ids=[user_id], + message=message, job_id=job_id, defer_date=defer_date) + + async def send_future_notification_to_topic( + self, + message: Message, + custom_topic: CustomTopic, + defer_date: datetime, + job_id: str, + scheduler: Scheduler = Depends(get_scheduler), + ) -> None: + + await scheduler.queue_job_defer_to(self.send_notification_to_topic, + custom_topic=custom_topic, + message=message, job_id=job_id, defer_date=defer_date) \ No newline at end of file From a847d347066d6844b088b1a101849639805629a1 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 04:12:42 +0100 Subject: [PATCH 030/113] SUper commit --- .../communication/future_notifications.py | 44 +++++++++++++++++++ app/utils/communication/notifications.py | 28 +----------- 2 files changed, 45 insertions(+), 27 deletions(-) create mode 100644 app/utils/communication/future_notifications.py diff --git a/app/utils/communication/future_notifications.py b/app/utils/communication/future_notifications.py new file mode 100644 index 000000000..cfc77a224 --- /dev/null +++ b/app/utils/communication/future_notifications.py @@ -0,0 +1,44 @@ +from datetime import datetime + +from fastapi import BackgroundTasks, Depends +from app.core.notification.notification_types import CustomTopic +from app.core.notification.schemas_notification import Message +from app.dependencies import get_scheduler +from app.types.scheduler import Scheduler +from app.utils.communication.notifications import NotificationManager, NotificationTool +from sqlalchemy.ext.asyncio import AsyncSession + + +class FutureNotificationTool(NotificationTool): + def __init__( + self, + background_tasks: BackgroundTasks, + notification_manager: NotificationManager, + db: AsyncSession, + ): + super().__init__(background_tasks, notification_manager, db) + async def send_future_notification_to_user( + self, + user_id: str, + message: Message, + defer_date: datetime, + job_id: str, + scheduler: Scheduler = Depends(get_scheduler), + ) -> None: + + await scheduler.queue_job_defer_to(self.send_notification_to_users, + user_ids=[user_id], + message=message, job_id=job_id, defer_date=defer_date) + + async def send_future_notification_to_topic( + self, + message: Message, + custom_topic: CustomTopic, + defer_date: datetime, + job_id: str, + scheduler: Scheduler = Depends(get_scheduler), + ) -> None: + + await scheduler.queue_job_defer_to(self.send_notification_to_topic, + custom_topic=custom_topic, + message=message, job_id=job_id, defer_date=defer_date) \ No newline at end of file diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index b6f2c70ac..e14023e1c 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -343,30 +343,4 @@ async def send_notification_to_topic( custom_topic=custom_topic, message=message, db=self.db, - ) - - async def send_future_notification_to_user( - self, - user_id: str, - message: Message, - defer_date: datetime, - job_id: str, - scheduler: Scheduler = Depends(get_scheduler), - ) -> None: - - await scheduler.queue_job_defer_to(self.send_notification_to_users, - user_ids=[user_id], - message=message, job_id=job_id, defer_date=defer_date) - - async def send_future_notification_to_topic( - self, - message: Message, - custom_topic: CustomTopic, - defer_date: datetime, - job_id: str, - scheduler: Scheduler = Depends(get_scheduler), - ) -> None: - - await scheduler.queue_job_defer_to(self.send_notification_to_topic, - custom_topic=custom_topic, - message=message, job_id=job_id, defer_date=defer_date) \ No newline at end of file + ) \ No newline at end of file From 616199c5fc7f60e425dedd7e80fc93cdf4ac41d4 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 04:14:19 +0100 Subject: [PATCH 031/113] Hot fix --- app/utils/communication/notifications.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index e14023e1c..65423507e 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -10,8 +10,6 @@ from app.core.notification import cruds_notification, models_notification from app.core.notification.notification_types import CustomTopic from app.core.notification.schemas_notification import Message -from app.dependencies import get_scheduler -from app.types.scheduler import Scheduler hyperion_error_logger = logging.getLogger("hyperion.error") From c267060955427cba7c295b592eb1b30e0cc2f6f0 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 05:39:59 +0100 Subject: [PATCH 032/113] redis not config --- app/types/scheduler.py | 60 ++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index aa72dba7a..0beee3ecb 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -8,11 +8,23 @@ scheduler_logger = logging.getLogger("scheduler") +async def create_scheduler(settings): + scheduler = Scheduler(settings) + if scheduler.settings.host != "": + await scheduler.async_init() + return scheduler + + class Scheduler: """Disappears sometimes for no reason""" - def __init__(self, redis_pool): - self.redis_pool = redis_pool + def __init__(self, redis_settings): + self.settings = redis_settings + + async def async_init(self): + if self.settings.host != "": + self.redis_pool = await create_pool(self.settings) + scheduler_logger.debug(f"Pool in init {self.redis_pool}") async def queue_job_time_defer( self, @@ -25,15 +37,15 @@ async def queue_job_time_defer( Queue a job to execute job_function in defer_seconds amount of seconds job_id will allow to abort if needed """ - job = await self.redis_pool.enqueue_job( - "run_task", - job_function=job_function, - _job_id=job_id, - _defer_by=timedelta(seconds=defer_seconds), - **kwargs, - ) - scheduler_logger.debug(f"Job {job_id} queued {job}") - return job + if self.settings.host != "": + job = await self.redis_pool.enqueue_job( + "run_task", + job_function=job_function, + _job_id=job_id, + _defer_by=timedelta(seconds=defer_seconds), + ) + scheduler_logger.debug(f"Job {job_id} queued {job}") + return job async def queue_job_defer_to( self, @@ -46,20 +58,22 @@ async def queue_job_defer_to( Queue a job to execute job_function at defer_date job_id will allow to abort if needed """ - job = await self.redis_pool.enqueue_job( - "run_task", - job_function=job_function, - _job_id=job_id, - _defer_until=defer_date, - **kwargs, - ) - scheduler_logger.debug(f"Job {job_id} queued {job}") - return job + if self.settings.host != "": + job = await self.redis_pool.enqueue_job( + "run_task", + job_function=job_function, + _job_id=job_id, + _defer_until=defer_date, + **kwargs, + ) + scheduler_logger.debug(f"Job {job_id} queued {job}") + return job async def cancel_job(self, job_id): """ cancel a queued job based on its job_id """ - job = Job(job_id, redis=self.redis_pool) - scheduler_logger.debug(f"Job aborted {job}") - await job.abort() + if self.settings.host != "": + job = Job(job_id, redis=self.redis_pool) + scheduler_logger.debug(f"Job aborted {job}") + await job.abort() From 399597d93cde652f762ce1a7dfdcb43f1feb2731 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 06:53:46 +0100 Subject: [PATCH 033/113] Circular pb --- app/core/notification/endpoints_notification.py | 6 ++++-- app/dependencies.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index f844ef02d..ec179a073 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -13,11 +13,13 @@ from app.core.notification.notification_types import CustomTopic, Topic from app.dependencies import ( get_db, + get_future_notification_tool, get_notification_manager, get_notification_tool, is_user, is_user_in, ) +from app.utils.communication.future_notifications import FutureNotificationTool from app.utils.communication.notifications import NotificationManager, NotificationTool router = APIRouter(tags=["Notifications"]) @@ -266,7 +268,7 @@ async def send_notification_topic( ) async def send_future_notification( user: models_core.CoreUser = Depends(is_user_in(GroupType.admin)), - notification_tool: NotificationTool = Depends(get_notification_tool), + notification_tool: NotificationTool = Depends(get_future_notification_tool), ): """ Send ourself a test notification. @@ -278,7 +280,7 @@ async def send_future_notification( content="Ceci est un test de notification future", action_module="test", ) - await notification_tool.send_future_notification_to_user( + await future_notification_tool.send_future_notification_to_user( user_id=user.id, message=message, defer_date=datetime.now() + timedelta(minutes=3), diff --git a/app/dependencies.py b/app/dependencies.py index 2ead1cf55..c9008750b 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -33,6 +33,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils +from app.utils.communication.future_notifications import FutureNotificationTool from app.utils.communication.notifications import NotificationManager, NotificationTool from app.utils.redis import connect from app.utils.tools import is_user_external, is_user_member_of_an_allowed_group @@ -227,6 +228,21 @@ def get_notification_tool( db=db, ) +def get_future_notification_tool( + background_tasks: BackgroundTasks, + db: AsyncSession = Depends(get_db), + notification_manager: NotificationManager = Depends(get_notification_manager), + ) -> FutureNotificationTool: + """ + Dependency that returns a notification tool, allowing to send push notification as a background tasks. + """ + + return FutureNotificationTool( + background_tasks=background_tasks, + notification_manager=notification_manager, + scheduler=Depends(get_scheduler), + db=db, + ) def get_drive_file_manager() -> DriveFileManager: """ From 3734b69bc46aeef5735fd4952c350d02e2333e39 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 06:54:35 +0100 Subject: [PATCH 034/113] Still circular issue --- app/dependencies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/dependencies.py b/app/dependencies.py index c9008750b..cd9b41cb1 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -240,7 +240,6 @@ def get_future_notification_tool( return FutureNotificationTool( background_tasks=background_tasks, notification_manager=notification_manager, - scheduler=Depends(get_scheduler), db=db, ) From 082720db5e211938267047ff396755b34a0236be Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 07:01:23 +0100 Subject: [PATCH 035/113] Fix circular import --- app/dependencies.py | 1 + app/utils/communication/future_notifications.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index cd9b41cb1..c9008750b 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -240,6 +240,7 @@ def get_future_notification_tool( return FutureNotificationTool( background_tasks=background_tasks, notification_manager=notification_manager, + scheduler=Depends(get_scheduler), db=db, ) diff --git a/app/utils/communication/future_notifications.py b/app/utils/communication/future_notifications.py index cfc77a224..b5d7cecf3 100644 --- a/app/utils/communication/future_notifications.py +++ b/app/utils/communication/future_notifications.py @@ -3,7 +3,6 @@ from fastapi import BackgroundTasks, Depends from app.core.notification.notification_types import CustomTopic from app.core.notification.schemas_notification import Message -from app.dependencies import get_scheduler from app.types.scheduler import Scheduler from app.utils.communication.notifications import NotificationManager, NotificationTool from sqlalchemy.ext.asyncio import AsyncSession @@ -14,6 +13,7 @@ def __init__( self, background_tasks: BackgroundTasks, notification_manager: NotificationManager, + scheduler: Scheduler, db: AsyncSession, ): super().__init__(background_tasks, notification_manager, db) @@ -23,7 +23,7 @@ async def send_future_notification_to_user( message: Message, defer_date: datetime, job_id: str, - scheduler: Scheduler = Depends(get_scheduler), + scheduler: Scheduler, ) -> None: await scheduler.queue_job_defer_to(self.send_notification_to_users, @@ -36,7 +36,7 @@ async def send_future_notification_to_topic( custom_topic: CustomTopic, defer_date: datetime, job_id: str, - scheduler: Scheduler = Depends(get_scheduler), + scheduler: Scheduler, ) -> None: await scheduler.queue_job_defer_to(self.send_notification_to_topic, From 04f257cb4079741f76690363b1afecbab2fecbf4 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 12:49:26 +0100 Subject: [PATCH 036/113] Fix --- .../notification/endpoints_notification.py | 8 ++-- app/core/notification/schemas_notification.py | 5 +- app/dependencies.py | 2 +- app/modules/advert/endpoints_advert.py | 2 +- app/modules/amap/endpoints_amap.py | 2 +- app/modules/booking/endpoints_booking.py | 2 +- app/modules/ph/endpoints_ph.py | 2 +- app/types/scheduler.py | 1 + .../communication/future_notifications.py | 46 ++++++++++++++----- app/utils/communication/notifications.py | 7 ++- tests/test_notification.py | 7 --- 11 files changed, 50 insertions(+), 34 deletions(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index ec179a073..3f9c65bc7 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -232,7 +232,7 @@ async def send_notification( title="Test notification", content="Ceci est un test de notification", action_module="test", - + ) await notification_tool.send_notification_to_user( user_id=user.id, @@ -280,11 +280,11 @@ async def send_future_notification( content="Ceci est un test de notification future", action_module="test", ) - await future_notification_tool.send_future_notification_to_user( + await future_notification_tool.send_future_notification_to_user_time_defer( user_id=user.id, message=message, - defer_date=datetime.now() + timedelta(minutes=3), - job_id = "test-notification" + defer_seconds=30, + job_id = "test-notification", ) diff --git a/app/core/notification/schemas_notification.py b/app/core/notification/schemas_notification.py index b88833416..971d691f6 100644 --- a/app/core/notification/schemas_notification.py +++ b/app/core/notification/schemas_notification.py @@ -1,6 +1,5 @@ -from datetime import datetime -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, Field class Message(BaseModel): @@ -11,7 +10,7 @@ class Message(BaseModel): None, description="An identifier for the module that should be triggered when the notification is clicked", ) - + class FirebaseDevice(BaseModel): diff --git a/app/dependencies.py b/app/dependencies.py index c9008750b..1e62ad0ca 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -13,7 +13,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from typing import Any, cast import redis -from arq import ArqRedis, create_pool +from arq import ArqRedis from arq.connections import RedisSettings from fastapi import BackgroundTasks, Depends, HTTPException, Request, status from sqlalchemy.ext.asyncio import ( diff --git a/app/modules/advert/endpoints_advert.py b/app/modules/advert/endpoints_advert.py index 070b0a2e3..44a285ee9 100644 --- a/app/modules/advert/endpoints_advert.py +++ b/app/modules/advert/endpoints_advert.py @@ -1,6 +1,6 @@ import logging import uuid -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from fastapi import Depends, File, HTTPException, Query, UploadFile from fastapi.responses import FileResponse diff --git a/app/modules/amap/endpoints_amap.py b/app/modules/amap/endpoints_amap.py index e1380318c..c797eec55 100644 --- a/app/modules/amap/endpoints_amap.py +++ b/app/modules/amap/endpoints_amap.py @@ -1,6 +1,6 @@ import logging import uuid -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from fastapi import Depends, HTTPException, Response from redis import Redis diff --git a/app/modules/booking/endpoints_booking.py b/app/modules/booking/endpoints_booking.py index bb44350bc..9e450d1df 100644 --- a/app/modules/booking/endpoints_booking.py +++ b/app/modules/booking/endpoints_booking.py @@ -1,6 +1,6 @@ import logging import uuid -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from zoneinfo import ZoneInfo from fastapi import Depends, HTTPException diff --git a/app/modules/ph/endpoints_ph.py b/app/modules/ph/endpoints_ph.py index 15dbd572a..b23dba1aa 100644 --- a/app/modules/ph/endpoints_ph.py +++ b/app/modules/ph/endpoints_ph.py @@ -1,5 +1,5 @@ import uuid -from datetime import UTC, datetime, time, timedelta +from datetime import UTC, datetime, timedelta from fastapi import Depends, File, HTTPException, UploadFile from fastapi.responses import FileResponse diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 0beee3ecb..03c8c9a22 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -43,6 +43,7 @@ async def queue_job_time_defer( job_function=job_function, _job_id=job_id, _defer_by=timedelta(seconds=defer_seconds), + **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") return job diff --git a/app/utils/communication/future_notifications.py b/app/utils/communication/future_notifications.py index b5d7cecf3..39712fce1 100644 --- a/app/utils/communication/future_notifications.py +++ b/app/utils/communication/future_notifications.py @@ -1,11 +1,12 @@ from datetime import datetime -from fastapi import BackgroundTasks, Depends +from fastapi import BackgroundTasks +from sqlalchemy.ext.asyncio import AsyncSession + from app.core.notification.notification_types import CustomTopic from app.core.notification.schemas_notification import Message from app.types.scheduler import Scheduler from app.utils.communication.notifications import NotificationManager, NotificationTool -from sqlalchemy.ext.asyncio import AsyncSession class FutureNotificationTool(NotificationTool): @@ -17,28 +18,51 @@ def __init__( db: AsyncSession, ): super().__init__(background_tasks, notification_manager, db) - async def send_future_notification_to_user( + self.scheduler = scheduler + async def send_future_notification_to_user_defer_to( self, user_id: str, message: Message, defer_date: datetime, job_id: str, - scheduler: Scheduler, ) -> None: - - await scheduler.queue_job_defer_to(self.send_notification_to_users, + + await self.scheduler.queue_job_defer_to(self.send_notification_to_users, user_ids=[user_id], message=message, job_id=job_id, defer_date=defer_date) - - async def send_future_notification_to_topic( + + async def send_future_notification_to_topic_defer_to( self, message: Message, custom_topic: CustomTopic, defer_date: datetime, job_id: str, - scheduler: Scheduler, ) -> None: + + await self.scheduler.queue_job_defer_to(self.send_notification_to_topic, + custom_topic=custom_topic, + message=message, job_id=job_id, defer_date=defer_date) - await scheduler.queue_job_defer_to(self.send_notification_to_topic, + async def send_future_notification_to_user_time_defer( + self, + user_id: str, + message: Message, + defer_seconds: float, + job_id: str, + ) -> None: + + await self.scheduler.queue_job_time_defer(self.send_notification_to_users, + user_ids=[user_id], + message=message, job_id=job_id, defer_seconds=defer_seconds) + + async def send_future_notification_to_topic_time_defer( + self, + message: Message, + custom_topic: CustomTopic, + defer_seconds: float, + job_id: str, + ) -> None: + + await self.scheduler.queue_job_time_defer(self.send_notification_to_topic, custom_topic=custom_topic, - message=message, job_id=job_id, defer_date=defer_date) \ No newline at end of file + message=message, job_id=job_id, defer_seconds=defer_seconds) diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 65423507e..89ff993d2 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -1,8 +1,7 @@ -from datetime import datetime import logging import firebase_admin -from fastapi import BackgroundTasks, Depends +from fastapi import BackgroundTasks from firebase_admin import credentials, messaging from sqlalchemy.ext.asyncio import AsyncSession @@ -275,7 +274,7 @@ async def subscribe_user_to_topic( ) tokens = await cruds_notification.get_firebase_tokens_by_user_ids(user_ids=[user_id], db=db) await self.subscribe_tokens_to_topic(custom_topic=custom_topic, tokens=tokens) - + async def unsubscribe_user_to_topic( self, @@ -341,4 +340,4 @@ async def send_notification_to_topic( custom_topic=custom_topic, message=message, db=self.db, - ) \ No newline at end of file + ) diff --git a/tests/test_notification.py b/tests/test_notification.py index 19757940c..760e8f4e4 100644 --- a/tests/test_notification.py +++ b/tests/test_notification.py @@ -90,13 +90,6 @@ def test_get_devices(client: TestClient) -> None: assert json[0]["firebase_device_token"] == FIREBASE_TOKEN_1 -def test_get_messages(client: TestClient) -> None: - response = client.get( - f"/notification/messages/{FIREBASE_TOKEN_1}", - ) - assert response.status_code == 200 - - def test_subscribe_to_topic(client: TestClient) -> None: response = client.post( f"/notification/topics/{TOPIC_1}/subscribe", From b21eedf7c10252f7b60da0177080b805dd4d9622 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 12:53:59 +0100 Subject: [PATCH 037/113] Trying to fix lint and format --- app/core/notification/cruds_notification.py | 1 + app/core/notification/endpoints_notification.py | 2 +- app/utils/communication/future_notifications.py | 2 +- app/utils/communication/notifications.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/core/notification/cruds_notification.py b/app/core/notification/cruds_notification.py index 4c6e82a6d..c0f39d120 100644 --- a/app/core/notification/cruds_notification.py +++ b/app/core/notification/cruds_notification.py @@ -8,6 +8,7 @@ from app.core.notification import models_notification from app.core.notification.notification_types import CustomTopic, Topic + async def get_firebase_devices_by_user_id( user_id: str, db: AsyncSession, diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index 3f9c65bc7..0e1e20323 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -1,4 +1,4 @@ -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from fastapi import APIRouter, Body, Depends, HTTPException, Path from sqlalchemy.ext.asyncio import AsyncSession diff --git a/app/utils/communication/future_notifications.py b/app/utils/communication/future_notifications.py index 39712fce1..533bc2e7c 100644 --- a/app/utils/communication/future_notifications.py +++ b/app/utils/communication/future_notifications.py @@ -42,7 +42,7 @@ async def send_future_notification_to_topic_defer_to( await self.scheduler.queue_job_defer_to(self.send_notification_to_topic, custom_topic=custom_topic, message=message, job_id=job_id, defer_date=defer_date) - + async def send_future_notification_to_user_time_defer( self, user_id: str, diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 89ff993d2..383a56fc3 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -145,7 +145,7 @@ def _send_firebase_push_notification_by_topic( try: messaging.send(message) except messaging.FirebaseError as error: - hyperion_error_logger.error( + hyperion_error_logger.exception( f"Notification: Unable to send firebase notification for topic {custom_topic}: {error}", ) raise From 0eb281b3d687b841b411ab5017502890aacac90f Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 14:22:13 +0100 Subject: [PATCH 038/113] Fix schedular import --- app/dependencies.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index 1e62ad0ca..7a56fee49 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -232,15 +232,16 @@ def get_future_notification_tool( background_tasks: BackgroundTasks, db: AsyncSession = Depends(get_db), notification_manager: NotificationManager = Depends(get_notification_manager), + scheduler : Scheduler =Depends(get_scheduler), ) -> FutureNotificationTool: """ - Dependency that returns a notification tool, allowing to send push notification as a background tasks. + Dependency that returns a notification tool, allowing to send push notification as a schedule tasks. """ return FutureNotificationTool( background_tasks=background_tasks, notification_manager=notification_manager, - scheduler=Depends(get_scheduler), + scheduler=scheduler, db=db, ) From 9823e63f8bcf9e7f49763aefddd3903a766fc14c Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sat, 16 Nov 2024 22:59:07 +0100 Subject: [PATCH 039/113] First test for scheduler --- app/core/endpoints_core.py | 6 +++ app/dependencies.py | 19 +++------ app/types/scheduler.py | 87 ++++++++------------------------------ app/worker.py | 18 ++------ docker-compose.yaml | 2 +- 5 files changed, 32 insertions(+), 100 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index ea1bdd34a..dfb761422 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -38,6 +38,12 @@ async def read_information( """ Return information about Hyperion. This endpoint can be used to check if the API is up. """ + scheduler = await get_scheduler() + + def testing_print(): + print("Super, ça marche") + + await scheduler.queue_job(testing_print, "testingprint10", 10) return schemas_core.CoreInformation( ready=True, diff --git a/app/dependencies.py b/app/dependencies.py index 7a56fee49..8b9e1b9b7 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -13,7 +13,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from typing import Any, cast import redis -from arq import ArqRedis +from arq import ArqRedis, create_pool from arq.connections import RedisSettings from fastapi import BackgroundTasks, Depends, HTTPException, Request, status from sqlalchemy.ext.asyncio import ( @@ -166,21 +166,12 @@ def get_redis_client( return redis_client -scheduler_logger = logging.getLogger("scheduler") - - -async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis | None: +async def get_scheduler() -> ArqRedis: global scheduler if scheduler is None: - if settings.REDIS_HOST != "": - scheduler_logger.debug("Initializing Scheduler") - arq_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) - scheduler = Scheduler(await create_pool(arq_settings)) - scheduler_logger.debug(f"Scheduler {scheduler} initialized") + settings = get_settings() + arq_settings = RedisSettings(host=settings.REDIS_HOST, port=settings.REDIS_PORT) + scheduler = Scheduler(await create_pool(arq_settings)) return scheduler diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 03c8c9a22..9a4dcc5e0 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -1,80 +1,27 @@ import logging -from collections.abc import Callable, Coroutine -from datetime import datetime, timedelta -from typing import Any +from datetime import timedelta +from arq import ArqRedis from arq.jobs import Job scheduler_logger = logging.getLogger("scheduler") -async def create_scheduler(settings): - scheduler = Scheduler(settings) - if scheduler.settings.host != "": - await scheduler.async_init() - return scheduler - - class Scheduler: - """Disappears sometimes for no reason""" - - def __init__(self, redis_settings): - self.settings = redis_settings - - async def async_init(self): - if self.settings.host != "": - self.redis_pool = await create_pool(self.settings) - scheduler_logger.debug(f"Pool in init {self.redis_pool}") - - async def queue_job_time_defer( - self, - job_function: Callable[..., Coroutine[Any, Any, Any]], - job_id: str, - defer_seconds: float, - **kwargs, - ): - """ - Queue a job to execute job_function in defer_seconds amount of seconds - job_id will allow to abort if needed - """ - if self.settings.host != "": - job = await self.redis_pool.enqueue_job( - "run_task", - job_function=job_function, - _job_id=job_id, - _defer_by=timedelta(seconds=defer_seconds), - **kwargs, - ) - scheduler_logger.debug(f"Job {job_id} queued {job}") - return job - - async def queue_job_defer_to( - self, - job_function: Callable[..., Coroutine[Any, Any, Any]], - job_id: str, - defer_date: datetime, - **kwargs, - ): - """ - Queue a job to execute job_function at defer_date - job_id will allow to abort if needed - """ - if self.settings.host != "": - job = await self.redis_pool.enqueue_job( - "run_task", - job_function=job_function, - _job_id=job_id, - _defer_until=defer_date, - **kwargs, - ) - scheduler_logger.debug(f"Job {job_id} queued {job}") - return job + def __init__(self, redis_pool: ArqRedis): + self.redis_pool = redis_pool + + async def queue_job(self, job_function, job_id, defer_time): + job = await self.redis_pool.enqueue_job( + "run_task", + job_function=job_function, + _job_id=job_id, + _defer_by=timedelta(seconds=defer_time), + ) + scheduler_logger.debug(f"Job queued {job}") + return job async def cancel_job(self, job_id): - """ - cancel a queued job based on its job_id - """ - if self.settings.host != "": - job = Job(job_id, redis=self.redis_pool) - scheduler_logger.debug(f"Job aborted {job}") - await job.abort() + job = Job(job_id, redis=self.redis_pool) + scheduler_logger.debug(f"Job aborted {job}") + await job.abort() diff --git a/app/worker.py b/app/worker.py index f9407882a..3573f6472 100644 --- a/app/worker.py +++ b/app/worker.py @@ -1,26 +1,14 @@ import logging -from arq.connections import RedisSettings - -from app.dependencies import get_settings - scheduler_logger = logging.getLogger("scheduler") -async def run_task(ctx, job_function, **kwargs): +async def run_task(ctx, job_function): scheduler_logger.debug(f"Job function consumed {job_function}") - return await job_function(**kwargs) - - -settings = get_settings() # Ce fichier ne doit être lancé que par la prod + return await job_function() class WorkerSettings: functions = [run_task] allow_abort_jobs = True - keep_result_forever = True - redis_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) + keep_result = 0 diff --git a/docker-compose.yaml b/docker-compose.yaml index 44e570e96..3200546f1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -25,7 +25,7 @@ services: - REDIS_PASSWORD=${REDIS_PASSWORD} hyperion-scheduler-worker: - image: hyperion-app + image: hyperion-scheduler-worker container_name: hyperion-scheduler-worker depends_on: - hyperion-db From 42d4ddf408af609ad20a6f01d5443046d1209093 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sat, 16 Nov 2024 23:34:04 +0100 Subject: [PATCH 040/113] Fix docker files --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 3200546f1..44e570e96 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -25,7 +25,7 @@ services: - REDIS_PASSWORD=${REDIS_PASSWORD} hyperion-scheduler-worker: - image: hyperion-scheduler-worker + image: hyperion-app container_name: hyperion-scheduler-worker depends_on: - hyperion-db From c85b491a02cc0a4707b714527d849ed5db4f3c1f Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:04:30 +0100 Subject: [PATCH 041/113] Worker redis settings configuration --- app/worker.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/worker.py b/app/worker.py index 3573f6472..432dd98fd 100644 --- a/app/worker.py +++ b/app/worker.py @@ -1,5 +1,9 @@ import logging +from arq.connections import RedisSettings + +from app.dependencies import get_settings + scheduler_logger = logging.getLogger("scheduler") @@ -8,7 +12,15 @@ async def run_task(ctx, job_function): return await job_function() +settings = get_settings() # Ce fichier ne doit être lancé que par la prod + + class WorkerSettings: functions = [run_task] allow_abort_jobs = True keep_result = 0 + settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) From ce21f49d32c3964d1b7a69472a4be83d1184fd99 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:10:21 +0100 Subject: [PATCH 042/113] Fixed stupid mistake --- app/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/worker.py b/app/worker.py index 432dd98fd..bdc6b4da2 100644 --- a/app/worker.py +++ b/app/worker.py @@ -19,7 +19,7 @@ class WorkerSettings: functions = [run_task] allow_abort_jobs = True keep_result = 0 - settings = RedisSettings( + redis_settings = RedisSettings( host=settings.REDIS_HOST, port=settings.REDIS_PORT, password=settings.REDIS_PASSWORD, From 6c9f3a4ef187e8ec200a03e601ab782d013def1b Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:14:48 +0100 Subject: [PATCH 043/113] Fix depency scheduler --- app/dependencies.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/dependencies.py b/app/dependencies.py index 8b9e1b9b7..9b553e547 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -170,7 +170,11 @@ async def get_scheduler() -> ArqRedis: global scheduler if scheduler is None: settings = get_settings() - arq_settings = RedisSettings(host=settings.REDIS_HOST, port=settings.REDIS_PORT) + arq_settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) scheduler = Scheduler(await create_pool(arq_settings)) return scheduler From 3a196e0a9596ec22d0085f0e45bd0e34e7cedd69 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:18:27 +0100 Subject: [PATCH 044/113] Denesting test function --- app/core/endpoints_core.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index dfb761422..344e50349 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -26,6 +26,10 @@ hyperion_error_logger = logging.getLogger("hyperion.error") +def testing_print(): + print("Super, ça marche") + + @router.get( "/information", response_model=schemas_core.CoreInformation, @@ -40,9 +44,6 @@ async def read_information( """ scheduler = await get_scheduler() - def testing_print(): - print("Super, ça marche") - await scheduler.queue_job(testing_print, "testingprint10", 10) return schemas_core.CoreInformation( From 6d8a69d97d89e9368411369ae5685e36315a8252 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:26:40 +0100 Subject: [PATCH 045/113] Slight testing changes --- app/core/endpoints_core.py | 2 +- app/dependencies.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index 344e50349..8c70509be 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -44,7 +44,7 @@ async def read_information( """ scheduler = await get_scheduler() - await scheduler.queue_job(testing_print, "testingprint10", 10) + await scheduler.queue_job(testing_print, uuid4(), 10) return schemas_core.CoreInformation( ready=True, diff --git a/app/dependencies.py b/app/dependencies.py index 9b553e547..66d0b9dc0 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -170,6 +170,8 @@ async def get_scheduler() -> ArqRedis: global scheduler if scheduler is None: settings = get_settings() + scheduler_logger = logging.getLogger("scheduler") + scheduler_logger.debug("Initializing Scheduler") arq_settings = RedisSettings( host=settings.REDIS_HOST, port=settings.REDIS_PORT, From aa48936214b6d32d095a093d85fc49c15daaeb6d Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:27:10 +0100 Subject: [PATCH 046/113] Slight testing changes --- app/types/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 9a4dcc5e0..6546ea381 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -18,7 +18,7 @@ async def queue_job(self, job_function, job_id, defer_time): _job_id=job_id, _defer_by=timedelta(seconds=defer_time), ) - scheduler_logger.debug(f"Job queued {job}") + scheduler_logger.debug(f"Job {job_id} queued {job}") return job async def cancel_job(self, job_id): From 05f16e94be95ac36232573330a8ae5990609f5d3 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:30:07 +0100 Subject: [PATCH 047/113] Fixed uuid type --- app/core/endpoints_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index 8c70509be..d8a97f6dd 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -44,7 +44,7 @@ async def read_information( """ scheduler = await get_scheduler() - await scheduler.queue_job(testing_print, uuid4(), 10) + await scheduler.queue_job(testing_print, str(uuid4()), 10) return schemas_core.CoreInformation( ready=True, From 01ad4b19d078d56378b03037bc65f81fd92051f5 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:39:53 +0100 Subject: [PATCH 048/113] Used dependency injection --- app/core/endpoints_core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index d8a97f6dd..55bb0caf2 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -42,8 +42,6 @@ async def read_information( """ Return information about Hyperion. This endpoint can be used to check if the API is up. """ - scheduler = await get_scheduler() - await scheduler.queue_job(testing_print, str(uuid4()), 10) return schemas_core.CoreInformation( From c84abe12cd70733880d6cc4e93395c282dc38f15 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 00:52:13 +0100 Subject: [PATCH 049/113] More logging --- app/dependencies.py | 1 + app/types/scheduler.py | 1 + 2 files changed, 2 insertions(+) diff --git a/app/dependencies.py b/app/dependencies.py index 66d0b9dc0..d934b00a1 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -178,6 +178,7 @@ async def get_scheduler() -> ArqRedis: password=settings.REDIS_PASSWORD, ) scheduler = Scheduler(await create_pool(arq_settings)) + scheduler_logger.debug(f"Scheduler {scheduler} initialized") return scheduler diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 6546ea381..0df60c44a 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -10,6 +10,7 @@ class Scheduler: def __init__(self, redis_pool: ArqRedis): self.redis_pool = redis_pool + scheduler_logger.debug(f"Pool in init {self.redis_pool}") async def queue_job(self, job_function, job_id, defer_time): job = await self.redis_pool.enqueue_job( From e77a89f86066801962d6e15a945c856f401442bb Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 01:11:14 +0100 Subject: [PATCH 050/113] test --- app/core/endpoints_core.py | 1 + app/worker.py | 10 +++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index 55bb0caf2..29541053f 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -28,6 +28,7 @@ def testing_print(): print("Super, ça marche") + return 42 @router.get( diff --git a/app/worker.py b/app/worker.py index bdc6b4da2..e7efd431e 100644 --- a/app/worker.py +++ b/app/worker.py @@ -2,7 +2,7 @@ from arq.connections import RedisSettings -from app.dependencies import get_settings +from app.dependencies import get_scheduler, get_settings scheduler_logger = logging.getLogger("scheduler") @@ -18,9 +18,5 @@ async def run_task(ctx, job_function): class WorkerSettings: functions = [run_task] allow_abort_jobs = True - keep_result = 0 - redis_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) + keep_result_forever = True + redis_pool = get_scheduler().redis_pool From ebdc9b7ac9cb85387ef5d1579099a3c9cd48e4cb Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 01:21:45 +0100 Subject: [PATCH 051/113] test 2 --- app/dependencies.py | 4 +++- app/worker.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index d934b00a1..2c92d646d 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -166,11 +166,13 @@ def get_redis_client( return redis_client +scheduler_logger = logging.getLogger("scheduler") + + async def get_scheduler() -> ArqRedis: global scheduler if scheduler is None: settings = get_settings() - scheduler_logger = logging.getLogger("scheduler") scheduler_logger.debug("Initializing Scheduler") arq_settings = RedisSettings( host=settings.REDIS_HOST, diff --git a/app/worker.py b/app/worker.py index e7efd431e..78be2af1c 100644 --- a/app/worker.py +++ b/app/worker.py @@ -2,7 +2,7 @@ from arq.connections import RedisSettings -from app.dependencies import get_scheduler, get_settings +from app.dependencies import get_settings scheduler_logger = logging.getLogger("scheduler") @@ -19,4 +19,8 @@ class WorkerSettings: functions = [run_task] allow_abort_jobs = True keep_result_forever = True - redis_pool = get_scheduler().redis_pool + redis_settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) From c5521e9e1406fc816b53179deecc9595ffc46f8b Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 01:27:47 +0100 Subject: [PATCH 052/113] test 3 --- app/core/endpoints_core.py | 2 +- app/dependencies.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index 29541053f..bde7826f2 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -26,7 +26,7 @@ hyperion_error_logger = logging.getLogger("hyperion.error") -def testing_print(): +async def testing_print(): print("Super, ça marche") return 42 diff --git a/app/dependencies.py b/app/dependencies.py index 2c92d646d..57031b89e 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -169,10 +169,9 @@ def get_redis_client( scheduler_logger = logging.getLogger("scheduler") -async def get_scheduler() -> ArqRedis: +async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: global scheduler if scheduler is None: - settings = get_settings() scheduler_logger.debug("Initializing Scheduler") arq_settings = RedisSettings( host=settings.REDIS_HOST, From be5342ef3148659f33158988376615932b3fdacb Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 01:55:34 +0100 Subject: [PATCH 053/113] test 4 --- app/core/endpoints_core.py | 1 + app/dependencies.py | 3 ++- app/types/scheduler.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index bde7826f2..c6a09a31c 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -44,6 +44,7 @@ async def read_information( Return information about Hyperion. This endpoint can be used to check if the API is up. """ await scheduler.queue_job(testing_print, str(uuid4()), 10) + print(scheduler) return schemas_core.CoreInformation( ready=True, diff --git a/app/dependencies.py b/app/dependencies.py index 57031b89e..e597a4b2d 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -171,6 +171,7 @@ def get_redis_client( async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: global scheduler + scheduler_logger.debug(f"Current scheduler {scheduler.name}") if scheduler is None: scheduler_logger.debug("Initializing Scheduler") arq_settings = RedisSettings( @@ -179,7 +180,7 @@ async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: password=settings.REDIS_PASSWORD, ) scheduler = Scheduler(await create_pool(arq_settings)) - scheduler_logger.debug(f"Scheduler {scheduler} initialized") + scheduler_logger.debug(f"Scheduler {scheduler.name} initialized") return scheduler diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 0df60c44a..f33ff71ae 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -1,5 +1,6 @@ import logging from datetime import timedelta +from uuid import uuid4 from arq import ArqRedis from arq.jobs import Job @@ -11,6 +12,7 @@ class Scheduler: def __init__(self, redis_pool: ArqRedis): self.redis_pool = redis_pool scheduler_logger.debug(f"Pool in init {self.redis_pool}") + name = str(uuid4()) async def queue_job(self, job_function, job_id, defer_time): job = await self.redis_pool.enqueue_job( From db40d3ae161532bba35c71b04c7192daefc3a216 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 01:56:55 +0100 Subject: [PATCH 054/113] test 4 fixed --- app/dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dependencies.py b/app/dependencies.py index e597a4b2d..659b47c7e 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -171,7 +171,7 @@ def get_redis_client( async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: global scheduler - scheduler_logger.debug(f"Current scheduler {scheduler.name}") + scheduler_logger.debug(f"Current scheduler {scheduler}") if scheduler is None: scheduler_logger.debug("Initializing Scheduler") arq_settings = RedisSettings( From 9ab1f2a9a2fe15dc78b66e6f9068dc9403c0e5a3 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 02:10:12 +0100 Subject: [PATCH 055/113] test 4 fixed 2 --- app/dependencies.py | 1 + app/types/scheduler.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/dependencies.py b/app/dependencies.py index 659b47c7e..2cb30b6d5 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -7,6 +7,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): ``` """ +import asyncio import logging from collections.abc import AsyncGenerator, Callable, Coroutine from functools import lru_cache diff --git a/app/types/scheduler.py b/app/types/scheduler.py index f33ff71ae..4bd294f08 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -12,7 +12,7 @@ class Scheduler: def __init__(self, redis_pool: ArqRedis): self.redis_pool = redis_pool scheduler_logger.debug(f"Pool in init {self.redis_pool}") - name = str(uuid4()) + self.name = str(uuid4()) async def queue_job(self, job_function, job_id, defer_time): job = await self.redis_pool.enqueue_job( From c8a3404f1b93bd86b701a36719ca9c11ec2c8dfa Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 02:23:12 +0100 Subject: [PATCH 056/113] test 5 with lock --- app/dependencies.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index 2cb30b6d5..afb40e9b4 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -174,14 +174,16 @@ async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: global scheduler scheduler_logger.debug(f"Current scheduler {scheduler}") if scheduler is None: - scheduler_logger.debug("Initializing Scheduler") - arq_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) - scheduler = Scheduler(await create_pool(arq_settings)) - scheduler_logger.debug(f"Scheduler {scheduler.name} initialized") + lock = asyncio.Lock() + async with lock: + scheduler_logger.debug("Initializing Scheduler") + arq_settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) + scheduler = Scheduler(await create_pool(arq_settings)) + scheduler_logger.debug(f"Scheduler {scheduler.name} initialized") return scheduler From 906d9686466409428b3139bd72fcdb5b625b8bd4 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 02:32:31 +0100 Subject: [PATCH 057/113] test 6 with enter --- app/types/scheduler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 4bd294f08..6f1e35c1f 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -14,6 +14,9 @@ def __init__(self, redis_pool: ArqRedis): scheduler_logger.debug(f"Pool in init {self.redis_pool}") self.name = str(uuid4()) + def __enter__(self): + return self + async def queue_job(self, job_function, job_id, defer_time): job = await self.redis_pool.enqueue_job( "run_task", From b16e2b99c761d99e48522dbdb6afd283cc60e9c4 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 02:48:41 +0100 Subject: [PATCH 058/113] Test 6 --- app/dependencies.py | 20 +++++++++----------- app/types/scheduler.py | 20 ++++++++++++++------ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index afb40e9b4..37eef80e9 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -30,7 +30,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.core.groups.groups_type import AccountType, GroupType, get_ecl_account_types from app.core.payment.payment_tool import PaymentTool from app.modules.raid.utils.drive.drive_file_manager import DriveFileManager -from app.types.scheduler import Scheduler +from app.types.scheduler import Scheduler, create_scheduler from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils @@ -174,16 +174,14 @@ async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: global scheduler scheduler_logger.debug(f"Current scheduler {scheduler}") if scheduler is None: - lock = asyncio.Lock() - async with lock: - scheduler_logger.debug("Initializing Scheduler") - arq_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) - scheduler = Scheduler(await create_pool(arq_settings)) - scheduler_logger.debug(f"Scheduler {scheduler.name} initialized") + scheduler_logger.debug("Initializing Scheduler") + arq_settings = RedisSettings( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + ) + scheduler = await create_scheduler(arq_settings) + scheduler_logger.debug(f"Scheduler {scheduler} initialized") return scheduler diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 6f1e35c1f..cc56ac611 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -2,20 +2,28 @@ from datetime import timedelta from uuid import uuid4 -from arq import ArqRedis +from arq import create_pool from arq.jobs import Job scheduler_logger = logging.getLogger("scheduler") +async def create_scheduler(settings): + scheduler = Scheduler(settings) + await scheduler.async_init() + return scheduler + + class Scheduler: - def __init__(self, redis_pool: ArqRedis): - self.redis_pool = redis_pool + def __init__(self, redis_settings): + self.settings = redis_settings + + async def async_init(self): + self.redis_pool = await create_pool(self.settings) scheduler_logger.debug(f"Pool in init {self.redis_pool}") - self.name = str(uuid4()) - def __enter__(self): - return self + def __del__(self): + scheduler_logger.debug(f"Del here with {self.redis_pool}") async def queue_job(self, job_function, job_id, defer_time): job = await self.redis_pool.enqueue_job( From 959445b5b9c799f42d547c9448ba31872f238c7b Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 03:01:44 +0100 Subject: [PATCH 059/113] Test_6 --- app/types/scheduler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index cc56ac611..c39b52fbb 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -15,8 +15,11 @@ async def create_scheduler(settings): class Scheduler: + current_instance = None # Trying to fool the garbage collector + def __init__(self, redis_settings): self.settings = redis_settings + self.current_instance = self async def async_init(self): self.redis_pool = await create_pool(self.settings) From 377382383f7a54ef2baf9ee722df9198f3472c5e Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 03:28:12 +0100 Subject: [PATCH 060/113] Working scheduler functions --- app/core/endpoints_core.py | 2 +- app/types/scheduler.py | 42 ++++++++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index c6a09a31c..6a5c3eba9 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -43,7 +43,7 @@ async def read_information( """ Return information about Hyperion. This endpoint can be used to check if the API is up. """ - await scheduler.queue_job(testing_print, str(uuid4()), 10) + await scheduler.queue_job_time_defer(testing_print, str(uuid4()), 10) print(scheduler) return schemas_core.CoreInformation( diff --git a/app/types/scheduler.py b/app/types/scheduler.py index c39b52fbb..d50e6cf3f 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -1,5 +1,6 @@ import logging -from datetime import timedelta +from datetime import datetime, timedelta +from typing import Any, Callable, Coroutine from uuid import uuid4 from arq import create_pool @@ -15,30 +16,55 @@ async def create_scheduler(settings): class Scheduler: - current_instance = None # Trying to fool the garbage collector - def __init__(self, redis_settings): self.settings = redis_settings - self.current_instance = self async def async_init(self): self.redis_pool = await create_pool(self.settings) scheduler_logger.debug(f"Pool in init {self.redis_pool}") - def __del__(self): - scheduler_logger.debug(f"Del here with {self.redis_pool}") + async def queue_job_time_defer( + self, + job_function: Callable[..., Coroutine[Any, Any, Any]], + job_id: str, + defer_seconds: float, + ): + """ + Queue a job to execute job_function in defer_seconds amount of seconds + job_id will allow to abort if needed + """ + job = await self.redis_pool.enqueue_job( + "run_task", + job_function=job_function, + _job_id=job_id, + _defer_by=timedelta(seconds=defer_seconds), + ) + scheduler_logger.debug(f"Job {job_id} queued {job}") + return job - async def queue_job(self, job_function, job_id, defer_time): + async def queue_job_defer_to( + self, + job_function: Callable[..., Coroutine[Any, Any, Any]], + job_id: str, + defer_date: datetime, + ): + """ + Queue a job to execute job_function at defer_date + job_id will allow to abort if needed + """ job = await self.redis_pool.enqueue_job( "run_task", job_function=job_function, _job_id=job_id, - _defer_by=timedelta(seconds=defer_time), + _defer_until=defer_date, ) scheduler_logger.debug(f"Job {job_id} queued {job}") return job async def cancel_job(self, job_id): + """ + cancel a queued job based on its job_id + """ job = Job(job_id, redis=self.redis_pool) scheduler_logger.debug(f"Job aborted {job}") await job.abort() From 209a02e761ccfd108785be20814078acdf5529a0 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Sun, 17 Nov 2024 03:28:17 +0100 Subject: [PATCH 061/113] Working scheduler functions --- app/types/scheduler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index d50e6cf3f..4e43a3dd5 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -16,6 +16,8 @@ async def create_scheduler(settings): class Scheduler: + """Disappears sometimes for no reason""" + def __init__(self, redis_settings): self.settings = redis_settings From ab33a7d25d5037456915b52484e6c9d2c7d247d3 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sat, 26 Oct 2024 23:08:20 +0200 Subject: [PATCH 062/113] First clean --- app/core/notification/cruds_notification.py | 29 +++++++++++++++++++++ app/utils/communication/notifications.py | 13 ++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/app/core/notification/cruds_notification.py b/app/core/notification/cruds_notification.py index c0f39d120..30d9edceb 100644 --- a/app/core/notification/cruds_notification.py +++ b/app/core/notification/cruds_notification.py @@ -9,6 +9,35 @@ from app.core.notification.notification_types import CustomTopic, Topic + + +async def create_batch_messages( + messages: list[models_notification.Message], + db: AsyncSession, +) -> None: + db.add_all(messages) + try: + await db.commit() + except IntegrityError: + await db.rollback() + raise + + + +async def remove_messages_by_context_and_firebase_device_tokens_list( + context: str, + tokens: list[str], + db: AsyncSession, +): + await db.execute( + delete(models_notification.Message).where( + models_notification.Message.context == context, + models_notification.Message.firebase_device_token.in_(tokens), + ), + ) + await db.commit() + + async def get_firebase_devices_by_user_id( user_id: str, db: AsyncSession, diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 383a56fc3..f12c8629b 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -102,9 +102,13 @@ async def _send_firebase_push_notification_by_tokens( tokens = tokens[:500] try: + # Set high priority for android, and background notification for iOS + # This allow to ensure that the notification will be processed in the background + # See https://firebase.google.com/docs/cloud-messaging/concept-options#setting-the-priority-of-a-message + # And https://developer.apple.com/documentation/usernotifications/setting_up_a_remote_notification_server/pushing_background_updates_to_your_app + message = messaging.MulticastMessage( tokens=tokens, - data={"module":message_content.action_module}, notification=messaging.Notification( title=message_content.title, body=message_content.content, @@ -154,12 +158,15 @@ async def subscribe_tokens_to_topic( self, custom_topic: CustomTopic, tokens: list[str], + message_content: Message, ): """ Subscribe a list of tokens to a given topic. """ - if not self.use_firebase: - return + # Push without any data or notification may not be processed by the app in the background. + # We thus need to send a data object with a dummy key to make sure the notification is processed. + # See https://stackoverflow.com/questions/59298850/firebase-messaging-background-message-handler-method-not-called-when-the-app + await self._send_firebase_push_notification_by_tokens(tokens=tokens, db=db, message_content=message_content) response = messaging.subscribe_to_topic(tokens, custom_topic.to_str()) if response.failure_count > 0: From 794f2d3cb841d962fd25d85b69c79fe7d9b953fd Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sat, 16 Nov 2024 12:44:26 +0100 Subject: [PATCH 063/113] RAF --- app/core/notification/cruds_notification.py | 30 ------------------- .../notification/endpoints_notification.py | 26 ++++++++++++++++ app/utils/communication/notifications.py | 16 +++------- 3 files changed, 30 insertions(+), 42 deletions(-) diff --git a/app/core/notification/cruds_notification.py b/app/core/notification/cruds_notification.py index 30d9edceb..4c6e82a6d 100644 --- a/app/core/notification/cruds_notification.py +++ b/app/core/notification/cruds_notification.py @@ -8,36 +8,6 @@ from app.core.notification import models_notification from app.core.notification.notification_types import CustomTopic, Topic - - - -async def create_batch_messages( - messages: list[models_notification.Message], - db: AsyncSession, -) -> None: - db.add_all(messages) - try: - await db.commit() - except IntegrityError: - await db.rollback() - raise - - - -async def remove_messages_by_context_and_firebase_device_tokens_list( - context: str, - tokens: list[str], - db: AsyncSession, -): - await db.execute( - delete(models_notification.Message).where( - models_notification.Message.context == context, - models_notification.Message.firebase_device_token.in_(tokens), - ), - ) - await db.commit() - - async def get_firebase_devices_by_user_id( user_id: str, db: AsyncSession, diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index 0e1e20323..3e926f600 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -239,6 +239,32 @@ async def send_notification( message=message, ) +@router.post( + "/notification/send/topic", + status_code=201, +) +async def send_notification_topic( + user: models_core.CoreUser = Depends(is_user_a_member_of(GroupType.admin)), + notification_tool: NotificationTool = Depends(get_notification_tool), +): + """ + Send ourself a test notification. + + **Only admins can use this endpoint** + """ + message = schemas_notification.Message( + context="notification-test-topic", + is_visible=True, + title="Test notification topic", + content="Ceci est un test de notification topic", + # The notification will expire in 3 days + expire_on=datetime.now(UTC) + timedelta(days=3), + ) + await notification_tool.send_notification_to_topic( + custom_topic=CustomTopic.from_str("test"), + message=message, + ) + @router.post( "/notification/send/topic", status_code=201, diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index f12c8629b..17f861734 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -102,11 +102,6 @@ async def _send_firebase_push_notification_by_tokens( tokens = tokens[:500] try: - # Set high priority for android, and background notification for iOS - # This allow to ensure that the notification will be processed in the background - # See https://firebase.google.com/docs/cloud-messaging/concept-options#setting-the-priority-of-a-message - # And https://developer.apple.com/documentation/usernotifications/setting_up_a_remote_notification_server/pushing_background_updates_to_your_app - message = messaging.MulticastMessage( tokens=tokens, notification=messaging.Notification( @@ -149,7 +144,7 @@ def _send_firebase_push_notification_by_topic( try: messaging.send(message) except messaging.FirebaseError as error: - hyperion_error_logger.exception( + hyperion_error_logger.error( f"Notification: Unable to send firebase notification for topic {custom_topic}: {error}", ) raise @@ -158,15 +153,12 @@ async def subscribe_tokens_to_topic( self, custom_topic: CustomTopic, tokens: list[str], - message_content: Message, ): """ Subscribe a list of tokens to a given topic. """ - # Push without any data or notification may not be processed by the app in the background. - # We thus need to send a data object with a dummy key to make sure the notification is processed. - # See https://stackoverflow.com/questions/59298850/firebase-messaging-background-message-handler-method-not-called-when-the-app - await self._send_firebase_push_notification_by_tokens(tokens=tokens, db=db, message_content=message_content) + if not self.use_firebase: + return response = messaging.subscribe_to_topic(tokens, custom_topic.to_str()) if response.failure_count > 0: @@ -281,7 +273,7 @@ async def subscribe_user_to_topic( ) tokens = await cruds_notification.get_firebase_tokens_by_user_ids(user_ids=[user_id], db=db) await self.subscribe_tokens_to_topic(custom_topic=custom_topic, tokens=tokens) - + async def unsubscribe_user_to_topic( self, From ebf8342d7fc6795acd86e7f5fb7fad74cec1183a Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sat, 16 Nov 2024 18:00:55 +0100 Subject: [PATCH 064/113] Edit message content for new notif system --- .../notification/endpoints_notification.py | 32 ++----------------- app/core/notification/schemas_notification.py | 2 +- app/utils/communication/notifications.py | 1 + 3 files changed, 5 insertions(+), 30 deletions(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index 3e926f600..f59878cb8 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -232,7 +232,7 @@ async def send_notification( title="Test notification", content="Ceci est un test de notification", action_module="test", - + ) await notification_tool.send_notification_to_user( user_id=user.id, @@ -253,12 +253,9 @@ async def send_notification_topic( **Only admins can use this endpoint** """ message = schemas_notification.Message( - context="notification-test-topic", - is_visible=True, title="Test notification topic", content="Ceci est un test de notification topic", - # The notification will expire in 3 days - expire_on=datetime.now(UTC) + timedelta(days=3), + action_module="test", ) await notification_tool.send_notification_to_topic( custom_topic=CustomTopic.from_str("test"), @@ -270,31 +267,8 @@ async def send_notification_topic( status_code=201, ) async def send_notification_topic( - user: models_core.CoreUser = Depends(is_user_a_member_of(GroupType.admin)), - notification_tool: NotificationTool = Depends(get_notification_tool), -): - """ - Send ourself a test notification. - - **Only admins can use this endpoint** - """ - message = schemas_notification.Message( - title="Test notification topic", - content="Ceci est un test de notification topic", - action_module="test", - ) - await notification_tool.send_notification_to_topic( - custom_topic=CustomTopic.from_str("test"), - message=message, - ) - -@router.post( - "/notification/send/future", - status_code=201, -) -async def send_future_notification( user: models_core.CoreUser = Depends(is_user_in(GroupType.admin)), - notification_tool: NotificationTool = Depends(get_future_notification_tool), + notification_tool: NotificationTool = Depends(get_notification_tool), ): """ Send ourself a test notification. diff --git a/app/core/notification/schemas_notification.py b/app/core/notification/schemas_notification.py index 971d691f6..03c68f707 100644 --- a/app/core/notification/schemas_notification.py +++ b/app/core/notification/schemas_notification.py @@ -10,7 +10,7 @@ class Message(BaseModel): None, description="An identifier for the module that should be triggered when the notification is clicked", ) - + class FirebaseDevice(BaseModel): diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 17f861734..09145e109 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -104,6 +104,7 @@ async def _send_firebase_push_notification_by_tokens( try: message = messaging.MulticastMessage( tokens=tokens, + data={"module":message_content.action_module}, notification=messaging.Notification( title=message_content.title, body=message_content.content, From 12ac5955591bc21d8ab6b019383f9200968b9f90 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 03:59:52 +0100 Subject: [PATCH 065/113] Test future notif --- .../notification/endpoints_notification.py | 6 ++-- app/types/scheduler.py | 2 ++ app/utils/communication/notifications.py | 31 ++++++++++++++++++- app/worker.py | 4 +-- 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index f59878cb8..e0486e4a2 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -280,11 +280,11 @@ async def send_notification_topic( content="Ceci est un test de notification future", action_module="test", ) - await future_notification_tool.send_future_notification_to_user_time_defer( + await notification_tool.send_future_notification_to_user( user_id=user.id, message=message, - defer_seconds=30, - job_id = "test-notification", + defer_date=datetime.now() + timedelta(minutes=3), + job_id = "test-notification" ) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 4e43a3dd5..1b67b2dd2 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -49,6 +49,7 @@ async def queue_job_defer_to( job_function: Callable[..., Coroutine[Any, Any, Any]], job_id: str, defer_date: datetime, + **kwargs, ): """ Queue a job to execute job_function at defer_date @@ -59,6 +60,7 @@ async def queue_job_defer_to( job_function=job_function, _job_id=job_id, _defer_until=defer_date, + **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") return job diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 09145e109..b6f2c70ac 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -1,7 +1,8 @@ +from datetime import datetime import logging import firebase_admin -from fastapi import BackgroundTasks +from fastapi import BackgroundTasks, Depends from firebase_admin import credentials, messaging from sqlalchemy.ext.asyncio import AsyncSession @@ -9,6 +10,8 @@ from app.core.notification import cruds_notification, models_notification from app.core.notification.notification_types import CustomTopic from app.core.notification.schemas_notification import Message +from app.dependencies import get_scheduler +from app.types.scheduler import Scheduler hyperion_error_logger = logging.getLogger("hyperion.error") @@ -341,3 +344,29 @@ async def send_notification_to_topic( message=message, db=self.db, ) + + async def send_future_notification_to_user( + self, + user_id: str, + message: Message, + defer_date: datetime, + job_id: str, + scheduler: Scheduler = Depends(get_scheduler), + ) -> None: + + await scheduler.queue_job_defer_to(self.send_notification_to_users, + user_ids=[user_id], + message=message, job_id=job_id, defer_date=defer_date) + + async def send_future_notification_to_topic( + self, + message: Message, + custom_topic: CustomTopic, + defer_date: datetime, + job_id: str, + scheduler: Scheduler = Depends(get_scheduler), + ) -> None: + + await scheduler.queue_job_defer_to(self.send_notification_to_topic, + custom_topic=custom_topic, + message=message, job_id=job_id, defer_date=defer_date) \ No newline at end of file diff --git a/app/worker.py b/app/worker.py index 78be2af1c..f9407882a 100644 --- a/app/worker.py +++ b/app/worker.py @@ -7,9 +7,9 @@ scheduler_logger = logging.getLogger("scheduler") -async def run_task(ctx, job_function): +async def run_task(ctx, job_function, **kwargs): scheduler_logger.debug(f"Job function consumed {job_function}") - return await job_function() + return await job_function(**kwargs) settings = get_settings() # Ce fichier ne doit être lancé que par la prod From 53ee3ff5667e93047ec622f40b1d2611715ba571 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 04:12:42 +0100 Subject: [PATCH 066/113] SUper commit --- .../communication/future_notifications.py | 50 +++++-------------- app/utils/communication/notifications.py | 28 +---------- 2 files changed, 14 insertions(+), 64 deletions(-) diff --git a/app/utils/communication/future_notifications.py b/app/utils/communication/future_notifications.py index 533bc2e7c..cfc77a224 100644 --- a/app/utils/communication/future_notifications.py +++ b/app/utils/communication/future_notifications.py @@ -1,12 +1,12 @@ from datetime import datetime -from fastapi import BackgroundTasks -from sqlalchemy.ext.asyncio import AsyncSession - +from fastapi import BackgroundTasks, Depends from app.core.notification.notification_types import CustomTopic from app.core.notification.schemas_notification import Message +from app.dependencies import get_scheduler from app.types.scheduler import Scheduler from app.utils.communication.notifications import NotificationManager, NotificationTool +from sqlalchemy.ext.asyncio import AsyncSession class FutureNotificationTool(NotificationTool): @@ -14,55 +14,31 @@ def __init__( self, background_tasks: BackgroundTasks, notification_manager: NotificationManager, - scheduler: Scheduler, db: AsyncSession, ): super().__init__(background_tasks, notification_manager, db) - self.scheduler = scheduler - async def send_future_notification_to_user_defer_to( + async def send_future_notification_to_user( self, user_id: str, message: Message, defer_date: datetime, job_id: str, + scheduler: Scheduler = Depends(get_scheduler), ) -> None: - - await self.scheduler.queue_job_defer_to(self.send_notification_to_users, + + await scheduler.queue_job_defer_to(self.send_notification_to_users, user_ids=[user_id], message=message, job_id=job_id, defer_date=defer_date) - - async def send_future_notification_to_topic_defer_to( + + async def send_future_notification_to_topic( self, message: Message, custom_topic: CustomTopic, defer_date: datetime, job_id: str, + scheduler: Scheduler = Depends(get_scheduler), ) -> None: - - await self.scheduler.queue_job_defer_to(self.send_notification_to_topic, - custom_topic=custom_topic, - message=message, job_id=job_id, defer_date=defer_date) - - async def send_future_notification_to_user_time_defer( - self, - user_id: str, - message: Message, - defer_seconds: float, - job_id: str, - ) -> None: - - await self.scheduler.queue_job_time_defer(self.send_notification_to_users, - user_ids=[user_id], - message=message, job_id=job_id, defer_seconds=defer_seconds) - - async def send_future_notification_to_topic_time_defer( - self, - message: Message, - custom_topic: CustomTopic, - defer_seconds: float, - job_id: str, - ) -> None: - - await self.scheduler.queue_job_time_defer(self.send_notification_to_topic, + + await scheduler.queue_job_defer_to(self.send_notification_to_topic, custom_topic=custom_topic, - message=message, job_id=job_id, defer_seconds=defer_seconds) + message=message, job_id=job_id, defer_date=defer_date) \ No newline at end of file diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index b6f2c70ac..e14023e1c 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -343,30 +343,4 @@ async def send_notification_to_topic( custom_topic=custom_topic, message=message, db=self.db, - ) - - async def send_future_notification_to_user( - self, - user_id: str, - message: Message, - defer_date: datetime, - job_id: str, - scheduler: Scheduler = Depends(get_scheduler), - ) -> None: - - await scheduler.queue_job_defer_to(self.send_notification_to_users, - user_ids=[user_id], - message=message, job_id=job_id, defer_date=defer_date) - - async def send_future_notification_to_topic( - self, - message: Message, - custom_topic: CustomTopic, - defer_date: datetime, - job_id: str, - scheduler: Scheduler = Depends(get_scheduler), - ) -> None: - - await scheduler.queue_job_defer_to(self.send_notification_to_topic, - custom_topic=custom_topic, - message=message, job_id=job_id, defer_date=defer_date) \ No newline at end of file + ) \ No newline at end of file From 5234e43b10f7fa257c9bc15c6e7a13c5b79d24f4 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 04:14:19 +0100 Subject: [PATCH 067/113] Hot fix --- app/utils/communication/notifications.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index e14023e1c..65423507e 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -10,8 +10,6 @@ from app.core.notification import cruds_notification, models_notification from app.core.notification.notification_types import CustomTopic from app.core.notification.schemas_notification import Message -from app.dependencies import get_scheduler -from app.types.scheduler import Scheduler hyperion_error_logger = logging.getLogger("hyperion.error") From 4f4612c1cebe0a61db7dd2b2e2c0f540ed70b5d6 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 05:39:59 +0100 Subject: [PATCH 068/113] redis not config --- app/types/scheduler.py | 51 +++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 1b67b2dd2..3b5cb703f 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -11,7 +11,8 @@ async def create_scheduler(settings): scheduler = Scheduler(settings) - await scheduler.async_init() + if scheduler.settings.host != "": + await scheduler.async_init() return scheduler @@ -22,8 +23,9 @@ def __init__(self, redis_settings): self.settings = redis_settings async def async_init(self): - self.redis_pool = await create_pool(self.settings) - scheduler_logger.debug(f"Pool in init {self.redis_pool}") + if self.settings.host != "": + self.redis_pool = await create_pool(self.settings) + scheduler_logger.debug(f"Pool in init {self.redis_pool}") async def queue_job_time_defer( self, @@ -35,14 +37,15 @@ async def queue_job_time_defer( Queue a job to execute job_function in defer_seconds amount of seconds job_id will allow to abort if needed """ - job = await self.redis_pool.enqueue_job( - "run_task", - job_function=job_function, - _job_id=job_id, - _defer_by=timedelta(seconds=defer_seconds), - ) - scheduler_logger.debug(f"Job {job_id} queued {job}") - return job + if self.settings.host != "": + job = await self.redis_pool.enqueue_job( + "run_task", + job_function=job_function, + _job_id=job_id, + _defer_by=timedelta(seconds=defer_seconds), + ) + scheduler_logger.debug(f"Job {job_id} queued {job}") + return job async def queue_job_defer_to( self, @@ -55,20 +58,22 @@ async def queue_job_defer_to( Queue a job to execute job_function at defer_date job_id will allow to abort if needed """ - job = await self.redis_pool.enqueue_job( - "run_task", - job_function=job_function, - _job_id=job_id, - _defer_until=defer_date, - **kwargs, - ) - scheduler_logger.debug(f"Job {job_id} queued {job}") - return job + if self.settings.host != "": + job = await self.redis_pool.enqueue_job( + "run_task", + job_function=job_function, + _job_id=job_id, + _defer_until=defer_date, + **kwargs, + ) + scheduler_logger.debug(f"Job {job_id} queued {job}") + return job async def cancel_job(self, job_id): """ cancel a queued job based on its job_id """ - job = Job(job_id, redis=self.redis_pool) - scheduler_logger.debug(f"Job aborted {job}") - await job.abort() + if self.settings.host != "": + job = Job(job_id, redis=self.redis_pool) + scheduler_logger.debug(f"Job aborted {job}") + await job.abort() From a7ca58a13376500336b2b42d2c80a6e318e65ae7 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 06:53:46 +0100 Subject: [PATCH 069/113] Circular pb --- app/core/notification/endpoints_notification.py | 2 +- app/dependencies.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index e0486e4a2..d97b09e5b 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -280,7 +280,7 @@ async def send_notification_topic( content="Ceci est un test de notification future", action_module="test", ) - await notification_tool.send_future_notification_to_user( + await future_notification_tool.send_future_notification_to_user( user_id=user.id, message=message, defer_date=datetime.now() + timedelta(minutes=3), diff --git a/app/dependencies.py b/app/dependencies.py index 37eef80e9..a3ea4fcc5 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -233,16 +233,15 @@ def get_future_notification_tool( background_tasks: BackgroundTasks, db: AsyncSession = Depends(get_db), notification_manager: NotificationManager = Depends(get_notification_manager), - scheduler : Scheduler =Depends(get_scheduler), ) -> FutureNotificationTool: """ - Dependency that returns a notification tool, allowing to send push notification as a schedule tasks. + Dependency that returns a notification tool, allowing to send push notification as a background tasks. """ return FutureNotificationTool( background_tasks=background_tasks, notification_manager=notification_manager, - scheduler=scheduler, + scheduler=Depends(get_scheduler), db=db, ) From 88c6978a0bd7202feefe7c4aa423db47312d37d7 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 06:54:35 +0100 Subject: [PATCH 070/113] Still circular issue --- app/dependencies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/dependencies.py b/app/dependencies.py index a3ea4fcc5..a8523a330 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -241,7 +241,6 @@ def get_future_notification_tool( return FutureNotificationTool( background_tasks=background_tasks, notification_manager=notification_manager, - scheduler=Depends(get_scheduler), db=db, ) From 11c9fab84e2d67b1d2b105dd65e5b8a69cd8cee9 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 07:01:23 +0100 Subject: [PATCH 071/113] Fix circular import --- app/dependencies.py | 1 + app/utils/communication/future_notifications.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index a8523a330..a3ea4fcc5 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -241,6 +241,7 @@ def get_future_notification_tool( return FutureNotificationTool( background_tasks=background_tasks, notification_manager=notification_manager, + scheduler=Depends(get_scheduler), db=db, ) diff --git a/app/utils/communication/future_notifications.py b/app/utils/communication/future_notifications.py index cfc77a224..b5d7cecf3 100644 --- a/app/utils/communication/future_notifications.py +++ b/app/utils/communication/future_notifications.py @@ -3,7 +3,6 @@ from fastapi import BackgroundTasks, Depends from app.core.notification.notification_types import CustomTopic from app.core.notification.schemas_notification import Message -from app.dependencies import get_scheduler from app.types.scheduler import Scheduler from app.utils.communication.notifications import NotificationManager, NotificationTool from sqlalchemy.ext.asyncio import AsyncSession @@ -14,6 +13,7 @@ def __init__( self, background_tasks: BackgroundTasks, notification_manager: NotificationManager, + scheduler: Scheduler, db: AsyncSession, ): super().__init__(background_tasks, notification_manager, db) @@ -23,7 +23,7 @@ async def send_future_notification_to_user( message: Message, defer_date: datetime, job_id: str, - scheduler: Scheduler = Depends(get_scheduler), + scheduler: Scheduler, ) -> None: await scheduler.queue_job_defer_to(self.send_notification_to_users, @@ -36,7 +36,7 @@ async def send_future_notification_to_topic( custom_topic: CustomTopic, defer_date: datetime, job_id: str, - scheduler: Scheduler = Depends(get_scheduler), + scheduler: Scheduler, ) -> None: await scheduler.queue_job_defer_to(self.send_notification_to_topic, From 241381c7499a350df9da74d5bb027b101af12b7b Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 12:49:26 +0100 Subject: [PATCH 072/113] Fix --- .../notification/endpoints_notification.py | 8 ++-- app/core/notification/schemas_notification.py | 2 +- app/dependencies.py | 5 +- app/types/scheduler.py | 6 ++- .../communication/future_notifications.py | 46 ++++++++++++++----- app/utils/communication/notifications.py | 7 ++- 6 files changed, 49 insertions(+), 25 deletions(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index d97b09e5b..f3504537c 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -232,7 +232,7 @@ async def send_notification( title="Test notification", content="Ceci est un test de notification", action_module="test", - + ) await notification_tool.send_notification_to_user( user_id=user.id, @@ -280,11 +280,11 @@ async def send_notification_topic( content="Ceci est un test de notification future", action_module="test", ) - await future_notification_tool.send_future_notification_to_user( + await future_notification_tool.send_future_notification_to_user_time_defer( user_id=user.id, message=message, - defer_date=datetime.now() + timedelta(minutes=3), - job_id = "test-notification" + defer_seconds=30, + job_id = "test-notification", ) diff --git a/app/core/notification/schemas_notification.py b/app/core/notification/schemas_notification.py index 03c68f707..971d691f6 100644 --- a/app/core/notification/schemas_notification.py +++ b/app/core/notification/schemas_notification.py @@ -10,7 +10,7 @@ class Message(BaseModel): None, description="An identifier for the module that should be triggered when the notification is clicked", ) - + class FirebaseDevice(BaseModel): diff --git a/app/dependencies.py b/app/dependencies.py index a3ea4fcc5..0abe7f331 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -7,14 +7,13 @@ async def get_users(db: AsyncSession = Depends(get_db)): ``` """ -import asyncio import logging from collections.abc import AsyncGenerator, Callable, Coroutine from functools import lru_cache from typing import Any, cast import redis -from arq import ArqRedis, create_pool +from arq import ArqRedis from arq.connections import RedisSettings from fastapi import BackgroundTasks, Depends, HTTPException, Request, status from sqlalchemy.ext.asyncio import ( @@ -30,7 +29,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.core.groups.groups_type import AccountType, GroupType, get_ecl_account_types from app.core.payment.payment_tool import PaymentTool from app.modules.raid.utils.drive.drive_file_manager import DriveFileManager -from app.types.scheduler import Scheduler, create_scheduler +from app.types.scheduler import create_scheduler from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 3b5cb703f..e14c18f1f 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -1,7 +1,7 @@ import logging +from collections.abc import Callable, Coroutine from datetime import datetime, timedelta -from typing import Any, Callable, Coroutine -from uuid import uuid4 +from typing import Any from arq import create_pool from arq.jobs import Job @@ -32,6 +32,7 @@ async def queue_job_time_defer( job_function: Callable[..., Coroutine[Any, Any, Any]], job_id: str, defer_seconds: float, + **kwargs, ): """ Queue a job to execute job_function in defer_seconds amount of seconds @@ -43,6 +44,7 @@ async def queue_job_time_defer( job_function=job_function, _job_id=job_id, _defer_by=timedelta(seconds=defer_seconds), + **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") return job diff --git a/app/utils/communication/future_notifications.py b/app/utils/communication/future_notifications.py index b5d7cecf3..39712fce1 100644 --- a/app/utils/communication/future_notifications.py +++ b/app/utils/communication/future_notifications.py @@ -1,11 +1,12 @@ from datetime import datetime -from fastapi import BackgroundTasks, Depends +from fastapi import BackgroundTasks +from sqlalchemy.ext.asyncio import AsyncSession + from app.core.notification.notification_types import CustomTopic from app.core.notification.schemas_notification import Message from app.types.scheduler import Scheduler from app.utils.communication.notifications import NotificationManager, NotificationTool -from sqlalchemy.ext.asyncio import AsyncSession class FutureNotificationTool(NotificationTool): @@ -17,28 +18,51 @@ def __init__( db: AsyncSession, ): super().__init__(background_tasks, notification_manager, db) - async def send_future_notification_to_user( + self.scheduler = scheduler + async def send_future_notification_to_user_defer_to( self, user_id: str, message: Message, defer_date: datetime, job_id: str, - scheduler: Scheduler, ) -> None: - - await scheduler.queue_job_defer_to(self.send_notification_to_users, + + await self.scheduler.queue_job_defer_to(self.send_notification_to_users, user_ids=[user_id], message=message, job_id=job_id, defer_date=defer_date) - - async def send_future_notification_to_topic( + + async def send_future_notification_to_topic_defer_to( self, message: Message, custom_topic: CustomTopic, defer_date: datetime, job_id: str, - scheduler: Scheduler, ) -> None: + + await self.scheduler.queue_job_defer_to(self.send_notification_to_topic, + custom_topic=custom_topic, + message=message, job_id=job_id, defer_date=defer_date) - await scheduler.queue_job_defer_to(self.send_notification_to_topic, + async def send_future_notification_to_user_time_defer( + self, + user_id: str, + message: Message, + defer_seconds: float, + job_id: str, + ) -> None: + + await self.scheduler.queue_job_time_defer(self.send_notification_to_users, + user_ids=[user_id], + message=message, job_id=job_id, defer_seconds=defer_seconds) + + async def send_future_notification_to_topic_time_defer( + self, + message: Message, + custom_topic: CustomTopic, + defer_seconds: float, + job_id: str, + ) -> None: + + await self.scheduler.queue_job_time_defer(self.send_notification_to_topic, custom_topic=custom_topic, - message=message, job_id=job_id, defer_date=defer_date) \ No newline at end of file + message=message, job_id=job_id, defer_seconds=defer_seconds) diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 65423507e..89ff993d2 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -1,8 +1,7 @@ -from datetime import datetime import logging import firebase_admin -from fastapi import BackgroundTasks, Depends +from fastapi import BackgroundTasks from firebase_admin import credentials, messaging from sqlalchemy.ext.asyncio import AsyncSession @@ -275,7 +274,7 @@ async def subscribe_user_to_topic( ) tokens = await cruds_notification.get_firebase_tokens_by_user_ids(user_ids=[user_id], db=db) await self.subscribe_tokens_to_topic(custom_topic=custom_topic, tokens=tokens) - + async def unsubscribe_user_to_topic( self, @@ -341,4 +340,4 @@ async def send_notification_to_topic( custom_topic=custom_topic, message=message, db=self.db, - ) \ No newline at end of file + ) From aec0b97dfb48ff2bd0268b8b0c965bc9b03e1197 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 12:53:59 +0100 Subject: [PATCH 073/113] Trying to fix lint and format --- app/core/notification/cruds_notification.py | 1 + app/utils/communication/future_notifications.py | 2 +- app/utils/communication/notifications.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/core/notification/cruds_notification.py b/app/core/notification/cruds_notification.py index 4c6e82a6d..c0f39d120 100644 --- a/app/core/notification/cruds_notification.py +++ b/app/core/notification/cruds_notification.py @@ -8,6 +8,7 @@ from app.core.notification import models_notification from app.core.notification.notification_types import CustomTopic, Topic + async def get_firebase_devices_by_user_id( user_id: str, db: AsyncSession, diff --git a/app/utils/communication/future_notifications.py b/app/utils/communication/future_notifications.py index 39712fce1..533bc2e7c 100644 --- a/app/utils/communication/future_notifications.py +++ b/app/utils/communication/future_notifications.py @@ -42,7 +42,7 @@ async def send_future_notification_to_topic_defer_to( await self.scheduler.queue_job_defer_to(self.send_notification_to_topic, custom_topic=custom_topic, message=message, job_id=job_id, defer_date=defer_date) - + async def send_future_notification_to_user_time_defer( self, user_id: str, diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 89ff993d2..383a56fc3 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -145,7 +145,7 @@ def _send_firebase_push_notification_by_topic( try: messaging.send(message) except messaging.FirebaseError as error: - hyperion_error_logger.error( + hyperion_error_logger.exception( f"Notification: Unable to send firebase notification for topic {custom_topic}: {error}", ) raise From 39c01dc55bde2648506bc3ca77cb03de5c80e903 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 14:22:13 +0100 Subject: [PATCH 074/113] Fix schedular import --- app/dependencies.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index 0abe7f331..366e2533f 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -29,7 +29,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.core.groups.groups_type import AccountType, GroupType, get_ecl_account_types from app.core.payment.payment_tool import PaymentTool from app.modules.raid.utils.drive.drive_file_manager import DriveFileManager -from app.types.scheduler import create_scheduler +from app.types.scheduler import Scheduler, create_scheduler from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils @@ -232,15 +232,16 @@ def get_future_notification_tool( background_tasks: BackgroundTasks, db: AsyncSession = Depends(get_db), notification_manager: NotificationManager = Depends(get_notification_manager), + scheduler : Scheduler =Depends(get_scheduler), ) -> FutureNotificationTool: """ - Dependency that returns a notification tool, allowing to send push notification as a background tasks. + Dependency that returns a notification tool, allowing to send push notification as a schedule tasks. """ return FutureNotificationTool( background_tasks=background_tasks, notification_manager=notification_manager, - scheduler=Depends(get_scheduler), + scheduler=scheduler, db=db, ) From 77e69f914b228b06859d1cf77dd7551467395001 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Sun, 17 Nov 2024 16:28:00 +0100 Subject: [PATCH 075/113] class name --- app/utils/communication/future_notifications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/utils/communication/future_notifications.py b/app/utils/communication/future_notifications.py index 533bc2e7c..aca9705a8 100644 --- a/app/utils/communication/future_notifications.py +++ b/app/utils/communication/future_notifications.py @@ -51,7 +51,7 @@ async def send_future_notification_to_user_time_defer( job_id: str, ) -> None: - await self.scheduler.queue_job_time_defer(self.send_notification_to_users, + await self.scheduler.queue_job_time_defer(job_function=NotificationTool.send_notification_to_users, user_ids=[user_id], message=message, job_id=job_id, defer_seconds=defer_seconds) From bb9cf142b85dcda9b1a4d9d261bd5d7368933694 Mon Sep 17 00:00:00 2001 From: armanddidierjean <95971503+armanddidierjean@users.noreply.github.com> Date: Sun, 17 Nov 2024 19:11:04 +0100 Subject: [PATCH 076/113] Run arq scheduler in asyncio task --- app/app.py | 18 +++++- app/core/endpoints_core.py | 3 +- app/dependencies.py | 33 ++++------ app/types/scheduler.py | 127 +++++++++++++++++++++++++------------ app/worker.py | 26 -------- 5 files changed, 117 insertions(+), 90 deletions(-) delete mode 100644 app/worker.py diff --git a/app/app.py b/app/app.py index dce3941b1..7c185ae9b 100644 --- a/app/app.py +++ b/app/app.py @@ -30,15 +30,19 @@ from app.dependencies import ( get_db, get_redis_client, + get_scheduler, get_websocket_connection_manager, init_and_get_db_engine, ) from app.modules.module_list import module_list +from app.types.exceptions import ContentHTTPException, GoogleAPIInvalidCredentialsError +from app.types.scheduler import Scheduler from app.types.exceptions import ( ContentHTTPException, GoogleAPIInvalidCredentialsError, ) from app.types.sqlalchemy import Base +from app.types.websocket import WebsocketConnectionManager from app.utils import initialization from app.utils.redis import limiter @@ -352,14 +356,26 @@ async def lifespan(app: FastAPI) -> AsyncGenerator: # We expect this error to be raised if the credentials were never set before pass - ws_manager = app.dependency_overrides.get( + ws_manager: WebsocketConnectionManager = app.dependency_overrides.get( get_websocket_connection_manager, get_websocket_connection_manager, )(settings=settings) + arq_scheduler: Scheduler = app.dependency_overrides.get( + get_scheduler, + get_scheduler, + )() + await ws_manager.connect_broadcaster() + await arq_scheduler.start( + redis_host=settings.REDIS_HOST, + redis_port=settings.REDIS_PORT, + redis_password=settings.REDIS_PASSWORD, + ) + yield hyperion_error_logger.info("Shutting down") + await arq_scheduler.close() await ws_manager.disconnect_broadcaster() # Initialize app diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index 6a5c3eba9..ac2388f4f 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -43,8 +43,7 @@ async def read_information( """ Return information about Hyperion. This endpoint can be used to check if the API is up. """ - await scheduler.queue_job_time_defer(testing_print, str(uuid4()), 10) - print(scheduler) + await scheduler.queue_job_time_defer(testing_print, str(uuid4()), 5) return schemas_core.CoreInformation( ready=True, diff --git a/app/dependencies.py b/app/dependencies.py index 366e2533f..cce6aa531 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -29,7 +29,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.core.groups.groups_type import AccountType, GroupType, get_ecl_account_types from app.core.payment.payment_tool import PaymentTool from app.modules.raid.utils.drive.drive_file_manager import DriveFileManager -from app.types.scheduler import Scheduler, create_scheduler +from app.types.scheduler import Scheduler from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils @@ -45,9 +45,10 @@ async def get_users(db: AsyncSession = Depends(get_db)): redis_client: redis.Redis | bool | None = ( None # Create a global variable for the redis client, so that it can be instancied in the startup event ) -scheduler: ArqRedis | None = None # Is None if the redis client is not instantiated, is False if the redis client is instancied but not connected, is a redis.Redis object if the redis client is connected +scheduler: Scheduler | None = None + websocket_connection_manager: WebsocketConnectionManager | None = None engine: AsyncEngine | None = ( @@ -166,22 +167,10 @@ def get_redis_client( return redis_client -scheduler_logger = logging.getLogger("scheduler") - - -async def get_scheduler(settings: Settings = Depends(get_settings)) -> ArqRedis: +def get_scheduler() -> Scheduler: global scheduler - scheduler_logger.debug(f"Current scheduler {scheduler}") if scheduler is None: - scheduler_logger.debug("Initializing Scheduler") - arq_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) - scheduler = await create_scheduler(arq_settings) - scheduler_logger.debug(f"Scheduler {scheduler} initialized") - + scheduler = Scheduler() return scheduler @@ -228,12 +217,13 @@ def get_notification_tool( db=db, ) + def get_future_notification_tool( - background_tasks: BackgroundTasks, - db: AsyncSession = Depends(get_db), - notification_manager: NotificationManager = Depends(get_notification_manager), - scheduler : Scheduler =Depends(get_scheduler), - ) -> FutureNotificationTool: + background_tasks: BackgroundTasks, + db: AsyncSession = Depends(get_db), + notification_manager: NotificationManager = Depends(get_notification_manager), + scheduler: Scheduler = Depends(get_scheduler), +) -> FutureNotificationTool: """ Dependency that returns a notification tool, allowing to send push notification as a schedule tasks. """ @@ -245,6 +235,7 @@ def get_future_notification_tool( db=db, ) + def get_drive_file_manager() -> DriveFileManager: """ Dependency that returns the drive file manager. diff --git a/app/types/scheduler.py b/app/types/scheduler.py index e14c18f1f..a9581964f 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -1,31 +1,72 @@ +import asyncio import logging from collections.abc import Callable, Coroutine from datetime import datetime, timedelta from typing import Any -from arq import create_pool +from arq import Worker, create_pool, cron +from arq.connections import RedisSettings from arq.jobs import Job +from arq.typing import WorkerSettingsBase +from arq.worker import create_worker scheduler_logger = logging.getLogger("scheduler") -async def create_scheduler(settings): - scheduler = Scheduler(settings) - if scheduler.settings.host != "": - await scheduler.async_init() - return scheduler +async def run_task(ctx, job_function, **kwargs): + scheduler_logger.debug(f"Job function consumed {job_function}") + return await job_function(**kwargs) class Scheduler: - """Disappears sometimes for no reason""" + """ + An [arq](https://arq-docs.helpmanual.io/) scheduler. + The wrapper is intended to be used inside a FastAPI worker. - def __init__(self, redis_settings): - self.settings = redis_settings + The scheduler use a Redis database to store planned jobs. + """ - async def async_init(self): - if self.settings.host != "": - self.redis_pool = await create_pool(self.settings) - scheduler_logger.debug(f"Pool in init {self.redis_pool}") + # See https://github.com/fastapi/fastapi/discussions/9143#discussioncomment-5157572 + + def __init__(self): + # ArqWorker, in charge of scheduling and executing tasks + self.worker: Worker | None = None + # Task will contain the asyncio task that runs the worker + self.task: asyncio.Task | None = None + + async def start( + self, + redis_host: str, + redis_port: int, + redis_password: str, + **kwargs, + ): + class ArqWorkerSettings(WorkerSettingsBase): + functions = [run_task] + allow_abort_jobs = True + keep_result_forever = True + redis_settings = RedisSettings( + host=redis_host, + port=redis_port, + password=redis_password, + ) + + # We pass handle_signals=False to avoid arq from handling signals + # See https://github.com/python-arq/arq/issues/182 + self.worker = create_worker( + ArqWorkerSettings, + handle_signals=False, + **kwargs, + ) + # We run the worker in an asyncio task + self.task = asyncio.create_task(self.worker.async_run()) + + scheduler_logger.info("Scheduler started") + + async def close(self): + # If the worker was started, we close it + if self.worker is not None: + await self.worker.close() async def queue_job_time_defer( self, @@ -33,21 +74,23 @@ async def queue_job_time_defer( job_id: str, defer_seconds: float, **kwargs, - ): + ) -> Job | None: """ Queue a job to execute job_function in defer_seconds amount of seconds job_id will allow to abort if needed """ - if self.settings.host != "": - job = await self.redis_pool.enqueue_job( - "run_task", - job_function=job_function, - _job_id=job_id, - _defer_by=timedelta(seconds=defer_seconds), - **kwargs, - ) - scheduler_logger.debug(f"Job {job_id} queued {job}") - return job + if self.worker is None: + scheduler_logger.error("Scheduler not started") + return None + job = await self.worker.pool.enqueue_job( + "run_task", + job_function=job_function, + _job_id=job_id, + _defer_by=timedelta(seconds=defer_seconds), + **kwargs, + ) + scheduler_logger.debug(f"Job {job_id} queued {job}") + return job async def queue_job_defer_to( self, @@ -55,27 +98,31 @@ async def queue_job_defer_to( job_id: str, defer_date: datetime, **kwargs, - ): + ) -> Job | None: """ Queue a job to execute job_function at defer_date job_id will allow to abort if needed """ - if self.settings.host != "": - job = await self.redis_pool.enqueue_job( - "run_task", - job_function=job_function, - _job_id=job_id, - _defer_until=defer_date, - **kwargs, - ) - scheduler_logger.debug(f"Job {job_id} queued {job}") - return job + if self.worker is None: + scheduler_logger.error("Scheduler not started") + return None + job = await self.worker.pool.enqueue_job( + "run_task", + job_function=job_function, + _job_id=job_id, + _defer_until=defer_date, + **kwargs, + ) + scheduler_logger.debug(f"Job {job_id} queued {job}") + return job - async def cancel_job(self, job_id): + async def cancel_job(self, job_id: str): """ cancel a queued job based on its job_id """ - if self.settings.host != "": - job = Job(job_id, redis=self.redis_pool) - scheduler_logger.debug(f"Job aborted {job}") - await job.abort() + if self.worker is None: + scheduler_logger.error("Scheduler not started") + return None + job = Job(job_id, redis=self.worker.pool) + scheduler_logger.debug(f"Job aborted {job}") + await job.abort() diff --git a/app/worker.py b/app/worker.py deleted file mode 100644 index f9407882a..000000000 --- a/app/worker.py +++ /dev/null @@ -1,26 +0,0 @@ -import logging - -from arq.connections import RedisSettings - -from app.dependencies import get_settings - -scheduler_logger = logging.getLogger("scheduler") - - -async def run_task(ctx, job_function, **kwargs): - scheduler_logger.debug(f"Job function consumed {job_function}") - return await job_function(**kwargs) - - -settings = get_settings() # Ce fichier ne doit être lancé que par la prod - - -class WorkerSettings: - functions = [run_task] - allow_abort_jobs = True - keep_result_forever = True - redis_settings = RedisSettings( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, - ) From 7c4068efc6265508b106c1d12bd5043a77b117ca Mon Sep 17 00:00:00 2001 From: armanddidierjean <95971503+armanddidierjean@users.noreply.github.com> Date: Sun, 17 Nov 2024 19:11:14 +0100 Subject: [PATCH 077/113] Don't start an arq worker --- Dockerfile | 3 --- docker-compose.yaml | 9 --------- worker.sh | 5 ----- 3 files changed, 17 deletions(-) delete mode 100644 worker.sh diff --git a/Dockerfile b/Dockerfile index ee09330c0..5c419e628 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,4 @@ COPY app app/ COPY start.sh . RUN chmod +x start.sh -COPY worker.sh . -RUN chmod +x worker.sh - ENTRYPOINT ["./start.sh"] diff --git a/docker-compose.yaml b/docker-compose.yaml index 44e570e96..f78b1a8d8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -23,15 +23,6 @@ services: command: redis-server --requirepass ${REDIS_PASSWORD} environment: - REDIS_PASSWORD=${REDIS_PASSWORD} - - hyperion-scheduler-worker: - image: hyperion-app - container_name: hyperion-scheduler-worker - depends_on: - - hyperion-db - - hyperion-redis - entrypoint: "./worker.sh" - env_file: .env hyperion-app: image: hyperion-app diff --git a/worker.sh b/worker.sh deleted file mode 100644 index e3c6bb331..000000000 --- a/worker.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -e - -echo "Starting Scheduler Worker..." -exec arq app.worker.WorkerSettings From 2e7381deab206c6f6879802dbc7855232b158a1f Mon Sep 17 00:00:00 2001 From: armanddidierjean <95971503+armanddidierjean@users.noreply.github.com> Date: Mon, 18 Nov 2024 20:09:20 +0100 Subject: [PATCH 078/113] Pass get_db dependency to get a db session in planned tasks --- app/app.py | 9 +++++++++ app/types/scheduler.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/app/app.py b/app/app.py index 7c185ae9b..b6a7441b7 100644 --- a/app/app.py +++ b/app/app.py @@ -19,6 +19,7 @@ from fastapi.routing import APIRoute from sqlalchemy.engine import Connection, Engine from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session from app import api @@ -366,11 +367,19 @@ async def lifespan(app: FastAPI) -> AsyncGenerator: get_scheduler, )() + _get_db: Callable[[], AsyncGenerator[AsyncSession, None]] = ( + app.dependency_overrides.get( + get_db, + get_db, + ) + ) + await ws_manager.connect_broadcaster() await arq_scheduler.start( redis_host=settings.REDIS_HOST, redis_port=settings.REDIS_PORT, redis_password=settings.REDIS_PASSWORD, + _get_db=_get_db, ) yield diff --git a/app/types/scheduler.py b/app/types/scheduler.py index a9581964f..778aec0e9 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -1,6 +1,6 @@ import asyncio import logging -from collections.abc import Callable, Coroutine +from collections.abc import AsyncGenerator, Callable, Coroutine from datetime import datetime, timedelta from typing import Any @@ -9,12 +9,23 @@ from arq.jobs import Job from arq.typing import WorkerSettingsBase from arq.worker import create_worker +from sqlalchemy.ext.asyncio import AsyncSession scheduler_logger = logging.getLogger("scheduler") -async def run_task(ctx, job_function, **kwargs): +async def run_task( + ctx, + job_function: Callable[..., Coroutine[Any, Any, Any]], + _get_db: Callable[[], AsyncGenerator[AsyncSession, None]], + **kwargs, +): scheduler_logger.debug(f"Job function consumed {job_function}") + + if "db" in kwargs: + async for db in _get_db(): + kwargs["db"] = db + return await job_function(**kwargs) @@ -33,14 +44,25 @@ def __init__(self): self.worker: Worker | None = None # Task will contain the asyncio task that runs the worker self.task: asyncio.Task | None = None + # Pointer to the get_db dependency + # self._get_db: Callable[[], AsyncGenerator[AsyncSession, None]] async def start( self, redis_host: str, redis_port: int, redis_password: str, + _get_db: Callable[[], AsyncGenerator[AsyncSession, None]], **kwargs, ): + """ + Parameters: + - redis_host: str + - redis_port: int + - redis_password: str + - get_db: Callable[[], AsyncGenerator[AsyncSession, None]] a pointer to `get_db` dependency + """ + class ArqWorkerSettings(WorkerSettingsBase): functions = [run_task] allow_abort_jobs = True @@ -61,6 +83,8 @@ class ArqWorkerSettings(WorkerSettingsBase): # We run the worker in an asyncio task self.task = asyncio.create_task(self.worker.async_run()) + self._get_db = _get_db + scheduler_logger.info("Scheduler started") async def close(self): @@ -87,6 +111,7 @@ async def queue_job_time_defer( job_function=job_function, _job_id=job_id, _defer_by=timedelta(seconds=defer_seconds), + _get_db=self._get_db, **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") @@ -111,6 +136,7 @@ async def queue_job_defer_to( job_function=job_function, _job_id=job_id, _defer_until=defer_date, + _get_db=self._get_db, **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") From 565f3b74da5e3ee26a306531abb41a78d4fab4ab Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Mon, 18 Nov 2024 21:50:53 +0100 Subject: [PATCH 079/113] Feature : working future notif --- .../notification/endpoints_notification.py | 40 +++++++++-- app/dependencies.py | 23 +------ .../communication/future_notifications.py | 68 ------------------- app/utils/communication/notifications.py | 59 +++++++++++++++- 4 files changed, 94 insertions(+), 96 deletions(-) delete mode 100644 app/utils/communication/future_notifications.py diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index f3504537c..7205756ed 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -13,13 +13,13 @@ from app.core.notification.notification_types import CustomTopic, Topic from app.dependencies import ( get_db, - get_future_notification_tool, get_notification_manager, get_notification_tool, + get_scheduler, is_user, is_user_in, ) -from app.utils.communication.future_notifications import FutureNotificationTool +from app.types.scheduler import Scheduler from app.utils.communication.notifications import NotificationManager, NotificationTool router = APIRouter(tags=["Notifications"]) @@ -268,6 +268,7 @@ async def send_notification_topic( ) async def send_notification_topic( user: models_core.CoreUser = Depends(is_user_in(GroupType.admin)), + scheduler: Scheduler = Depends(get_scheduler), notification_tool: NotificationTool = Depends(get_notification_tool), ): """ @@ -280,13 +281,40 @@ async def send_notification_topic( content="Ceci est un test de notification future", action_module="test", ) - await future_notification_tool.send_future_notification_to_user_time_defer( - user_id=user.id, + await notification_tool.send_future_notification_to_users_time_defer( + user_ids=[user.id], message=message, - defer_seconds=30, - job_id = "test-notification", + defer_seconds=10, + job_id="test25", + scheduler=scheduler, ) +@router.post( + "/notification/send/topic/future", + status_code=201, +) +async def send_future_notification_topic( + user: models_core.CoreUser = Depends(is_user_a_member_of(GroupType.admin)), + notification_tool: NotificationTool = Depends(get_notification_tool), + scheduler: Scheduler = Depends(get_scheduler), +): + """ + Send ourself a test notification. + + **Only admins can use this endpoint** + """ + message = schemas_notification.Message( + title="Test notification future topic", + content="Ceci est un test de notification future topic", + action_module="test", + ) + await notification_tool.send_future_notification_to_topic_time_defer( + custom_topic=CustomTopic.from_str("test"), + message=message, + defer_seconds=10, + job_id="test26", + scheduler=scheduler, + ) @router.get( "/notification/devices", diff --git a/app/dependencies.py b/app/dependencies.py index cce6aa531..7b7a35d70 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -33,7 +33,6 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils -from app.utils.communication.future_notifications import FutureNotificationTool from app.utils.communication.notifications import NotificationManager, NotificationTool from app.utils.redis import connect from app.utils.tools import is_user_external, is_user_member_of_an_allowed_group @@ -206,6 +205,7 @@ def get_notification_tool( background_tasks: BackgroundTasks, db: AsyncSession = Depends(get_db), notification_manager: NotificationManager = Depends(get_notification_manager), + scheduler: Scheduler = Depends(get_scheduler), ) -> NotificationTool: """ Dependency that returns a notification tool, allowing to send push notification as a background tasks. @@ -214,26 +214,7 @@ def get_notification_tool( return NotificationTool( background_tasks=background_tasks, notification_manager=notification_manager, - db=db, - ) - - -def get_future_notification_tool( - background_tasks: BackgroundTasks, - db: AsyncSession = Depends(get_db), - notification_manager: NotificationManager = Depends(get_notification_manager), - scheduler: Scheduler = Depends(get_scheduler), -) -> FutureNotificationTool: - """ - Dependency that returns a notification tool, allowing to send push notification as a schedule tasks. - """ - - return FutureNotificationTool( - background_tasks=background_tasks, - notification_manager=notification_manager, - scheduler=scheduler, - db=db, - ) + db=db,) def get_drive_file_manager() -> DriveFileManager: diff --git a/app/utils/communication/future_notifications.py b/app/utils/communication/future_notifications.py deleted file mode 100644 index aca9705a8..000000000 --- a/app/utils/communication/future_notifications.py +++ /dev/null @@ -1,68 +0,0 @@ -from datetime import datetime - -from fastapi import BackgroundTasks -from sqlalchemy.ext.asyncio import AsyncSession - -from app.core.notification.notification_types import CustomTopic -from app.core.notification.schemas_notification import Message -from app.types.scheduler import Scheduler -from app.utils.communication.notifications import NotificationManager, NotificationTool - - -class FutureNotificationTool(NotificationTool): - def __init__( - self, - background_tasks: BackgroundTasks, - notification_manager: NotificationManager, - scheduler: Scheduler, - db: AsyncSession, - ): - super().__init__(background_tasks, notification_manager, db) - self.scheduler = scheduler - async def send_future_notification_to_user_defer_to( - self, - user_id: str, - message: Message, - defer_date: datetime, - job_id: str, - ) -> None: - - await self.scheduler.queue_job_defer_to(self.send_notification_to_users, - user_ids=[user_id], - message=message, job_id=job_id, defer_date=defer_date) - - async def send_future_notification_to_topic_defer_to( - self, - message: Message, - custom_topic: CustomTopic, - defer_date: datetime, - job_id: str, - ) -> None: - - await self.scheduler.queue_job_defer_to(self.send_notification_to_topic, - custom_topic=custom_topic, - message=message, job_id=job_id, defer_date=defer_date) - - async def send_future_notification_to_user_time_defer( - self, - user_id: str, - message: Message, - defer_seconds: float, - job_id: str, - ) -> None: - - await self.scheduler.queue_job_time_defer(job_function=NotificationTool.send_notification_to_users, - user_ids=[user_id], - message=message, job_id=job_id, defer_seconds=defer_seconds) - - async def send_future_notification_to_topic_time_defer( - self, - message: Message, - custom_topic: CustomTopic, - defer_seconds: float, - job_id: str, - ) -> None: - - await self.scheduler.queue_job_time_defer(self.send_notification_to_topic, - custom_topic=custom_topic, - message=message, job_id=job_id, defer_seconds=defer_seconds) diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 383a56fc3..fe75c4fab 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -1,3 +1,4 @@ +from datetime import datetime import logging import firebase_admin @@ -9,6 +10,7 @@ from app.core.notification import cruds_notification, models_notification from app.core.notification.notification_types import CustomTopic from app.core.notification.schemas_notification import Message +from app.types.scheduler import Scheduler hyperion_error_logger = logging.getLogger("hyperion.error") @@ -307,10 +309,12 @@ def __init__( background_tasks: BackgroundTasks, notification_manager: NotificationManager, db: AsyncSession, + #scheduler: Scheduler, ): self.background_tasks = background_tasks self.notification_manager = notification_manager self.db = db + #self.scheduler = scheduler async def send_notification_to_users(self, user_ids: list[str], message: Message): self.background_tasks.add_task( @@ -319,7 +323,26 @@ async def send_notification_to_users(self, user_ids: list[str], message: Message message=message, db=self.db, ) - + + async def send_future_notification_to_users_time_defer( + self, + user_ids: list[str], + message: Message, + scheduler: Scheduler, + defer_seconds:float, + job_id:str + ): + + await scheduler.queue_job_time_defer(self.notification_manager.send_notification_to_users, + user_ids=user_ids, + message=message, + db=None, + job_id=job_id, + defer_seconds=defer_seconds, + ) + + + async def send_notification_to_user( self, user_id: str, @@ -341,3 +364,37 @@ async def send_notification_to_topic( message=message, db=self.db, ) + + async def send_future_notification_to_topic_defer_to( + self, + custom_topic: CustomTopic, + message: Message, + scheduler: Scheduler, + defer_date:datetime, + job_id:str + ): + + await scheduler.queue_job_defer_to(self.notification_manager.send_notification_to_topic, + custom_topic=custom_topic, + message=message, + db=None, + job_id=job_id, + defer_date=defer_date, + ) + + async def send_future_notification_to_topic_time_defer( + self, + custom_topic: CustomTopic, + message: Message, + scheduler: Scheduler, + defer_seconds:float, + job_id:str + ): + + await scheduler.queue_job_time_defer(self.notification_manager.send_notification_to_topic, + custom_topic=custom_topic, + message=message, + db=None, + job_id=job_id, + defer_seconds=defer_seconds, + ) \ No newline at end of file From 591f1fdcacfd91a436164512442ad220d8a307b8 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Mon, 18 Nov 2024 22:15:50 +0100 Subject: [PATCH 080/113] Implements endpoints --- app/modules/cinema/endpoints_cinema.py | 8 +++- app/modules/loan/endpoints_loan.py | 60 ++++++++++++++---------- app/modules/ph/endpoints_ph.py | 24 ++++++++-- app/utils/communication/notifications.py | 17 +++++++ 4 files changed, 78 insertions(+), 31 deletions(-) diff --git a/app/modules/cinema/endpoints_cinema.py b/app/modules/cinema/endpoints_cinema.py index e7118b90d..b1fbd896e 100644 --- a/app/modules/cinema/endpoints_cinema.py +++ b/app/modules/cinema/endpoints_cinema.py @@ -16,6 +16,7 @@ get_db, get_notification_tool, get_request_id, + get_scheduler, get_settings, is_user_a_member, is_user_in, @@ -23,6 +24,7 @@ from app.modules.cinema import cruds_cinema, schemas_cinema from app.types.content_type import ContentType from app.types.module import Module +from app.types.scheduler import Scheduler from app.utils.communication.date_manager import ( get_date_day, get_date_month, @@ -120,6 +122,7 @@ async def create_session( db: AsyncSession = Depends(get_db), user: models_core.CoreUser = Depends(is_user_in(GroupType.cinema)), notification_tool: NotificationTool = Depends(get_notification_tool), + scheduler:Scheduler = Depends(get_scheduler), ): db_session = schemas_cinema.CineSessionComplete( id=str(uuid.uuid4()), @@ -146,9 +149,12 @@ async def create_session( action_module="cinema", ) - await notification_tool.send_notification_to_topic( + await notification_tool.send_future_notification_to_topic_defer_to( custom_topic=CustomTopic(topic=Topic.cinema), message=message, + scheduler=scheduler, + defer_date=sunday, + job_id=f"cinema_weekly_{sunday}", ) return result diff --git a/app/modules/loan/endpoints_loan.py b/app/modules/loan/endpoints_loan.py index 1c7b1dc95..675a5e06b 100644 --- a/app/modules/loan/endpoints_loan.py +++ b/app/modules/loan/endpoints_loan.py @@ -12,11 +12,13 @@ from app.dependencies import ( get_db, get_notification_tool, + get_scheduler, is_user_a_member, is_user_in, ) from app.modules.loan import cruds_loan, models_loan, schemas_loan from app.types.module import Module +from app.types.scheduler import Scheduler from app.utils.communication.notifications import NotificationTool from app.utils.tools import ( is_group_id_valid, @@ -499,6 +501,7 @@ async def create_loan( db: AsyncSession = Depends(get_db), user: models_core.CoreUser = Depends(is_user_a_member), notification_tool: NotificationTool = Depends(get_notification_tool), + scheduler: Scheduler = Depends(get_scheduler), ): """ Create a new loan in database and add the requested items @@ -624,17 +627,19 @@ async def create_loan( delivery_time = time(11, 00, 00, tzinfo=UTC) delivery_datetime = datetime.combine(loan.end, delivery_time, tzinfo=UTC) - expire_on_date = loan.end + timedelta(days=30) - expire_on_datetime = datetime.combine(expire_on_date, delivery_time, tzinfo=UTC) + message = Message( title="📦 Prêt arrivé à échéance", content=f"N'oublie pas de rendre ton prêt à l'association {loan.loaner.name} !", module="loan", ) - await notification_tool.send_notification_to_user( - user_id=loan.borrower_id, + await notification_tool.send_future_notification_to_users_defer_to( + user_ids=[loan.borrower_id], message=message, + scheduler=scheduler, + defer_date=delivery_datetime, + job_id=f"loan_start_{loan.id}", ) return schemas_loan.Loan(items_qty=items_qty_ret, **loan.__dict__) @@ -867,16 +872,16 @@ async def return_loan( returned=True, returned_date=datetime.now(UTC), ) - - message = Message( - title="", - content="", - module="", - ) - await notification_tool.send_notification_to_user( - user_id=loan.borrower_id, - message=message, - ) + #TODO + # message = Message( + # title="", + # content="", + # module="", + # ) + # await notification_tool.send_notification_to_user( + # user_id=loan.borrower_id, + # message=message, + # ) @module.router.post( @@ -889,6 +894,7 @@ async def extend_loan( db: AsyncSession = Depends(get_db), user: models_core.CoreUser = Depends(is_user_a_member), notification_tool: NotificationTool = Depends(get_notification_tool), + scheduler: Scheduler = Depends(get_scheduler), ): """ A new `end` date or an extended `duration` can be provided. If the two are provided, only `end` will be used. @@ -928,14 +934,18 @@ async def extend_loan( loan_update=loan_update, db=db, ) - # same context so the first notification will be removed - message = Message( - title="📦 Prêt arrivé à échéance", - content=f"N'oublie pas de rendre ton prêt à l'association {loan.loaner.name} ! ", - action_module="loan", - ) - - await notification_tool.send_notification_to_user( - user_id=loan.borrower_id, - message=message, - ) + #TODO + # # same context so the first notification will be removed + # message = Message( + # title="📦 Prêt arrivé à échéance", + # content=f"N'oublie pas de rendre ton prêt à l'association {loan.loaner.name} ! ", + # action_module="loan", + # ) + + # await notification_tool.send_future_notification_to_users_defer_to( + # user_ids=[loan.borrower_id], + # message=message, + # scheduler=scheduler, + # defer_date=loan.end, + # job_id=f"loan_end_{loan.id}", + # ) diff --git a/app/modules/ph/endpoints_ph.py b/app/modules/ph/endpoints_ph.py index b23dba1aa..632f76a44 100644 --- a/app/modules/ph/endpoints_ph.py +++ b/app/modules/ph/endpoints_ph.py @@ -1,5 +1,5 @@ import uuid -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime, time, timedelta from fastapi import Depends, File, HTTPException, UploadFile from fastapi.responses import FileResponse @@ -13,12 +13,14 @@ get_db, get_notification_tool, get_request_id, + get_scheduler, is_user_a_member, is_user_in, ) from app.modules.ph import cruds_ph, models_ph, schemas_ph from app.types.content_type import ContentType from app.types.module import Module +from app.types.scheduler import Scheduler from app.utils.communication.notifications import NotificationTool from app.utils.tools import ( delete_file_from_data, @@ -105,6 +107,7 @@ async def create_paper( db: AsyncSession = Depends(get_db), user: models_core.CoreUser = Depends(is_user_in(GroupType.ph)), notification_tool: NotificationTool = Depends(get_notification_tool), + scheduler:Scheduler = Depends(get_scheduler), ): """Create a new paper.""" @@ -128,10 +131,21 @@ async def create_paper( content="Un nouveau journal est disponible! 🎉", action_module="ph", ) - await notification_tool.send_notification_to_topic( - custom_topic=CustomTopic(topic=Topic.ph), - message=message, - ) + if paper_db.release_date == now.date(): + await notification_tool.send_notification_to_topic( + custom_topic=CustomTopic(topic=Topic.ph), + message=message, + ) + else : + delivery_time = time(11, 00, 00, tzinfo=UTC) + release_date = datetime.combine(paper_db.release_date, delivery_time) + await notification_tool.send_future_notification_to_topic_defer_to( + custom_topic=CustomTopic(topic=Topic.ph), + message=message, + scheduler=scheduler, + defer_date=release_date, + job_id=f"ph_{paper_db.id}", + ) return await cruds_ph.create_paper(paper=paper_db, db=db) except ValueError as error: diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index fe75c4fab..d385b902f 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -341,6 +341,23 @@ async def send_future_notification_to_users_time_defer( defer_seconds=defer_seconds, ) + async def send_future_notification_to_users_defer_to( + self, + user_ids: list[str], + message: Message, + scheduler: Scheduler, + defer_date:datetime, + job_id:str + ): + + await scheduler.queue_job_defer_to(self.notification_manager.send_notification_to_users, + user_ids=user_ids, + message=message, + db=None, + job_id=job_id, + defer_date=defer_date, + ) + async def send_notification_to_user( From edc7fd4ae56f59a1b5ece7bfce5b3b238549faa8 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:14:03 +0100 Subject: [PATCH 081/113] Feat : add future and cancel notif for loan --- app/modules/loan/endpoints_loan.py | 64 +++++++++++++++++------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/app/modules/loan/endpoints_loan.py b/app/modules/loan/endpoints_loan.py index 675a5e06b..8c1f9ad1f 100644 --- a/app/modules/loan/endpoints_loan.py +++ b/app/modules/loan/endpoints_loan.py @@ -827,7 +827,7 @@ async def return_loan( loan_id: str, db: AsyncSession = Depends(get_db), user: models_core.CoreUser = Depends(is_user_a_member), - notification_tool: NotificationTool = Depends(get_notification_tool), + scheduler: Scheduler = Depends(get_scheduler), ): """ Mark a loan as returned. This will update items availability. @@ -872,16 +872,7 @@ async def return_loan( returned=True, returned_date=datetime.now(UTC), ) - #TODO - # message = Message( - # title="", - # content="", - # module="", - # ) - # await notification_tool.send_notification_to_user( - # user_id=loan.borrower_id, - # message=message, - # ) + await scheduler.cancel_job(f"loan_end_{loan.id}") @module.router.post( @@ -912,7 +903,7 @@ async def extend_loan( status_code=400, detail="Invalid loan_id", ) - + end = loan.end # The user should be a member of the loaner's manager group if not is_user_member_of_an_allowed_group(user, [loan.loaner.group_manager_id]): raise HTTPException( @@ -921,12 +912,14 @@ async def extend_loan( ) if loan_extend.end is not None: + end = loan_extend.end loan_update = schemas_loan.LoanUpdate( - end=loan_extend.end, + end=end, ) elif loan_extend.duration is not None: + end = loan.end + timedelta(seconds=loan_extend.duration) loan_update = schemas_loan.LoanUpdate( - end=loan.end + timedelta(seconds=loan_extend.duration), + end=end, ) await cruds_loan.update_loan( @@ -935,17 +928,32 @@ async def extend_loan( db=db, ) #TODO - # # same context so the first notification will be removed - # message = Message( - # title="📦 Prêt arrivé à échéance", - # content=f"N'oublie pas de rendre ton prêt à l'association {loan.loaner.name} ! ", - # action_module="loan", - # ) - - # await notification_tool.send_future_notification_to_users_defer_to( - # user_ids=[loan.borrower_id], - # message=message, - # scheduler=scheduler, - # defer_date=loan.end, - # job_id=f"loan_end_{loan.id}", - # ) + + await scheduler.cancel_job(f"loan_end_{loan.id}") + + message = Message( + title="📦 Prêt prolongé", + content=f"Ton prêt à l'association {loan.loaner.name} à bien été renouvellé !", + module="loan", + ) + await notification_tool.send_notification_to_user( + user_id=loan.borrower_id, + message=message, + ) + + delivery_time = time(11, 00, 00, tzinfo=UTC) + delivery_datetime = datetime.combine(end, delivery_time, tzinfo=UTC) + + message = Message( + title="📦 Prêt arrivé à échéance", + content=f"N'oublie pas de rendre ton prêt à l'association {loan.loaner.name} !", + module="loan", + ) + + await notification_tool.send_future_notification_to_users_defer_to( + user_ids=[loan.borrower_id], + message=message, + scheduler=scheduler, + defer_date=delivery_datetime, + job_id=f"loan_end_{loan.id}", + ) From 7981fe38725b819bf86382e1db133437159a657d Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:28:33 +0100 Subject: [PATCH 082/113] ruff fix --- app/dependencies.py | 4 +- app/types/scheduler.py | 2 +- app/utils/communication/notifications.py | 70 ++++++++++++------------ 3 files changed, 37 insertions(+), 39 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index 7b7a35d70..a98a43bbc 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -13,8 +13,6 @@ async def get_users(db: AsyncSession = Depends(get_db)): from typing import Any, cast import redis -from arq import ArqRedis -from arq.connections import RedisSettings from fastapi import BackgroundTasks, Depends, HTTPException, Request, status from sqlalchemy.ext.asyncio import ( AsyncEngine, @@ -214,7 +212,7 @@ def get_notification_tool( return NotificationTool( background_tasks=background_tasks, notification_manager=notification_manager, - db=db,) + db=db) def get_drive_file_manager() -> DriveFileManager: diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 778aec0e9..d434c7351 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta from typing import Any -from arq import Worker, create_pool, cron +from arq import Worker from arq.connections import RedisSettings from arq.jobs import Job from arq.typing import WorkerSettingsBase diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index d385b902f..dd219da45 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -1,5 +1,5 @@ -from datetime import datetime import logging +from datetime import datetime import firebase_admin from fastapi import BackgroundTasks @@ -323,16 +323,16 @@ async def send_notification_to_users(self, user_ids: list[str], message: Message message=message, db=self.db, ) - + async def send_future_notification_to_users_time_defer( - self, - user_ids: list[str], - message: Message, - scheduler: Scheduler, - defer_seconds:float, - job_id:str + self, + user_ids: list[str], + message: Message, + scheduler: Scheduler, + defer_seconds:float, + job_id:str, ): - + await scheduler.queue_job_time_defer(self.notification_manager.send_notification_to_users, user_ids=user_ids, message=message, @@ -340,16 +340,16 @@ async def send_future_notification_to_users_time_defer( job_id=job_id, defer_seconds=defer_seconds, ) - + async def send_future_notification_to_users_defer_to( - self, - user_ids: list[str], - message: Message, - scheduler: Scheduler, - defer_date:datetime, - job_id:str + self, + user_ids: list[str], + message: Message, + scheduler: Scheduler, + defer_date:datetime, + job_id:str, ): - + await scheduler.queue_job_defer_to(self.notification_manager.send_notification_to_users, user_ids=user_ids, message=message, @@ -357,9 +357,9 @@ async def send_future_notification_to_users_defer_to( job_id=job_id, defer_date=defer_date, ) - - - + + + async def send_notification_to_user( self, user_id: str, @@ -383,14 +383,14 @@ async def send_notification_to_topic( ) async def send_future_notification_to_topic_defer_to( - self, - custom_topic: CustomTopic, - message: Message, - scheduler: Scheduler, - defer_date:datetime, - job_id:str + self, + custom_topic: CustomTopic, + message: Message, + scheduler: Scheduler, + defer_date:datetime, + job_id:str, ): - + await scheduler.queue_job_defer_to(self.notification_manager.send_notification_to_topic, custom_topic=custom_topic, message=message, @@ -400,18 +400,18 @@ async def send_future_notification_to_topic_defer_to( ) async def send_future_notification_to_topic_time_defer( - self, - custom_topic: CustomTopic, - message: Message, - scheduler: Scheduler, - defer_seconds:float, - job_id:str + self, + custom_topic: CustomTopic, + message: Message, + scheduler: Scheduler, + defer_seconds:float, + job_id:str, ): - + await scheduler.queue_job_time_defer(self.notification_manager.send_notification_to_topic, custom_topic=custom_topic, message=message, db=None, job_id=job_id, defer_seconds=defer_seconds, - ) \ No newline at end of file + ) From 86c38b0679a205755d9d7a6e020f95da2b35d2d7 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Wed, 20 Nov 2024 16:52:40 +0100 Subject: [PATCH 083/113] Allowed for running without REDIS --- app/app.py | 2 +- app/dependencies.py | 9 ++--- app/types/scheduler.py | 75 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 5 deletions(-) diff --git a/app/app.py b/app/app.py index b6a7441b7..df2abc56f 100644 --- a/app/app.py +++ b/app/app.py @@ -365,7 +365,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator: arq_scheduler: Scheduler = app.dependency_overrides.get( get_scheduler, get_scheduler, - )() + )(settings=settings) _get_db: Callable[[], AsyncGenerator[AsyncSession, None]] = ( app.dependency_overrides.get( diff --git a/app/dependencies.py b/app/dependencies.py index a98a43bbc..2e63e97d5 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -27,7 +27,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.core.groups.groups_type import AccountType, GroupType, get_ecl_account_types from app.core.payment.payment_tool import PaymentTool from app.modules.raid.utils.drive.drive_file_manager import DriveFileManager -from app.types.scheduler import Scheduler +from app.types.scheduler import DummyScheduler, Scheduler from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils @@ -164,10 +164,10 @@ def get_redis_client( return redis_client -def get_scheduler() -> Scheduler: +def get_scheduler(settings: Settings = Depends(get_settings)) -> Scheduler: global scheduler if scheduler is None: - scheduler = Scheduler() + scheduler = Scheduler() if settings.REDIS_HOST != "" else DummyScheduler() return scheduler @@ -212,7 +212,8 @@ def get_notification_tool( return NotificationTool( background_tasks=background_tasks, notification_manager=notification_manager, - db=db) + db=db, + ) def get_drive_file_manager() -> DriveFileManager: diff --git a/app/types/scheduler.py b/app/types/scheduler.py index d434c7351..735435363 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -152,3 +152,78 @@ async def cancel_job(self, job_id: str): job = Job(job_id, redis=self.worker.pool) scheduler_logger.debug(f"Job aborted {job}") await job.abort() + + +class DummyScheduler(Scheduler): + """ + A Dummy implementation of the Scheduler to allow to run the server without a REDIS config + """ + + # See https://github.com/fastapi/fastapi/discussions/9143#discussioncomment-5157572 + + def __init__(self): + # ArqWorker, in charge of scheduling and executing tasks + self.worker: Worker | None = None + # Task will contain the asyncio task that runs the worker + self.task: asyncio.Task | None = None + # Pointer to the get_db dependency + # self._get_db: Callable[[], AsyncGenerator[AsyncSession, None]] + + async def start( + self, + redis_host: str, + redis_port: int, + redis_password: str, + _get_db: Callable[[], AsyncGenerator[AsyncSession, None]], + **kwargs, + ): + """ + Parameters: + - redis_host: str + - redis_port: int + - redis_password: str + - get_db: Callable[[], AsyncGenerator[AsyncSession, None]] a pointer to `get_db` dependency + """ + self._get_db = _get_db + + scheduler_logger.info("DummyScheduler started") + + async def close(self): + # If the worker was started, we close it + pass + + async def queue_job_time_defer( + self, + job_function: Callable[..., Coroutine[Any, Any, Any]], + job_id: str, + defer_seconds: float, + **kwargs, + ) -> Job | None: + """ + Queue a job to execute job_function in defer_seconds amount of seconds + job_id will allow to abort if needed + """ + scheduler_logger.debug( + f"Job {job_id} queued in DummyScheduler with defer {defer_seconds} seconds", + ) + + async def queue_job_defer_to( + self, + job_function: Callable[..., Coroutine[Any, Any, Any]], + job_id: str, + defer_date: datetime, + **kwargs, + ) -> Job | None: + """ + Queue a job to execute job_function at defer_date + job_id will allow to abort if needed + """ + scheduler_logger.debug( + f"Job {job_id} queued in DummyScheduler with defer to {defer_date}" + ) + + async def cancel_job(self, job_id: str): + """ + cancel a queued job based on its job_id + """ + scheduler_logger.debug(f"Job {job_id} aborted in DummyScheduler") From 80048db8ccd2a77ab4b3a4626dfe8e1913db6cd1 Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Wed, 20 Nov 2024 17:07:43 +0100 Subject: [PATCH 084/113] Fixed type inconsistencies --- app/types/scheduler.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 735435363..63e13ee6c 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -51,7 +51,7 @@ async def start( self, redis_host: str, redis_port: int, - redis_password: str, + redis_password: str | None, _get_db: Callable[[], AsyncGenerator[AsyncSession, None]], **kwargs, ): @@ -70,7 +70,7 @@ class ArqWorkerSettings(WorkerSettingsBase): redis_settings = RedisSettings( host=redis_host, port=redis_port, - password=redis_password, + password=redis_password if redis_password is not None else "", ) # We pass handle_signals=False to avoid arq from handling signals @@ -98,7 +98,7 @@ async def queue_job_time_defer( job_id: str, defer_seconds: float, **kwargs, - ) -> Job | None: + ): """ Queue a job to execute job_function in defer_seconds amount of seconds job_id will allow to abort if needed @@ -115,7 +115,6 @@ async def queue_job_time_defer( **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") - return job async def queue_job_defer_to( self, @@ -123,7 +122,7 @@ async def queue_job_defer_to( job_id: str, defer_date: datetime, **kwargs, - ) -> Job | None: + ): """ Queue a job to execute job_function at defer_date job_id will allow to abort if needed @@ -140,7 +139,6 @@ async def queue_job_defer_to( **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") - return job async def cancel_job(self, job_id: str): """ @@ -173,7 +171,7 @@ async def start( self, redis_host: str, redis_port: int, - redis_password: str, + redis_password: str | None, _get_db: Callable[[], AsyncGenerator[AsyncSession, None]], **kwargs, ): @@ -198,7 +196,7 @@ async def queue_job_time_defer( job_id: str, defer_seconds: float, **kwargs, - ) -> Job | None: + ): """ Queue a job to execute job_function in defer_seconds amount of seconds job_id will allow to abort if needed @@ -213,7 +211,7 @@ async def queue_job_defer_to( job_id: str, defer_date: datetime, **kwargs, - ) -> Job | None: + ): """ Queue a job to execute job_function at defer_date job_id will allow to abort if needed From 4be23a965a5cbb00dc4e93aa2652d2d53cd07ada Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:41:39 +0100 Subject: [PATCH 085/113] Ruff check --- app/app.py | 11 ++- app/core/endpoints_core.py | 6 -- app/types/scheduler.py | 8 ++- app/utils/communication/notifications.py | 88 +++++++++++++----------- 4 files changed, 56 insertions(+), 57 deletions(-) diff --git a/app/app.py b/app/app.py index df2abc56f..03a61b249 100644 --- a/app/app.py +++ b/app/app.py @@ -19,7 +19,6 @@ from fastapi.routing import APIRoute from sqlalchemy.engine import Connection, Engine from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session from app import api @@ -37,18 +36,16 @@ ) from app.modules.module_list import module_list from app.types.exceptions import ContentHTTPException, GoogleAPIInvalidCredentialsError -from app.types.scheduler import Scheduler -from app.types.exceptions import ( - ContentHTTPException, - GoogleAPIInvalidCredentialsError, -) from app.types.sqlalchemy import Base -from app.types.websocket import WebsocketConnectionManager from app.utils import initialization from app.utils.redis import limiter if TYPE_CHECKING: import redis + from sqlalchemy.ext.asyncio import AsyncSession + + from app.types.scheduler import Scheduler + from app.types.websocket import WebsocketConnectionManager # NOTE: We can not get loggers at the top of this file like we do in other files # as the loggers are not yet initialized diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index ac2388f4f..493b9aa3f 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -1,7 +1,6 @@ import logging from os import path from pathlib import Path -from uuid import uuid4 from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import FileResponse @@ -26,10 +25,6 @@ hyperion_error_logger = logging.getLogger("hyperion.error") -async def testing_print(): - print("Super, ça marche") - return 42 - @router.get( "/information", @@ -43,7 +38,6 @@ async def read_information( """ Return information about Hyperion. This endpoint can be used to check if the API is up. """ - await scheduler.queue_job_time_defer(testing_print, str(uuid4()), 5) return schemas_core.CoreInformation( ready=True, diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 63e13ee6c..1e4cdcbfb 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -2,15 +2,17 @@ import logging from collections.abc import AsyncGenerator, Callable, Coroutine from datetime import datetime, timedelta -from typing import Any +from typing import TYPE_CHECKING, Any -from arq import Worker from arq.connections import RedisSettings from arq.jobs import Job from arq.typing import WorkerSettingsBase from arq.worker import create_worker from sqlalchemy.ext.asyncio import AsyncSession +if TYPE_CHECKING: + from arq import Worker + scheduler_logger = logging.getLogger("scheduler") @@ -217,7 +219,7 @@ async def queue_job_defer_to( job_id will allow to abort if needed """ scheduler_logger.debug( - f"Job {job_id} queued in DummyScheduler with defer to {defer_date}" + f"Job {job_id} queued in DummyScheduler with defer to {defer_date}", ) async def cancel_job(self, job_id: str): diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index dd219da45..840356dfc 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -106,7 +106,7 @@ async def _send_firebase_push_notification_by_tokens( try: message = messaging.MulticastMessage( tokens=tokens, - data={"module":message_content.action_module}, + data={"module": message_content.action_module}, notification=messaging.Notification( title=message_content.title, body=message_content.content, @@ -140,15 +140,15 @@ def _send_firebase_push_notification_by_topic( message = messaging.Message( topic=custom_topic.to_str(), notification=messaging.Notification( - title=message_content.title, - body=message_content.content, - ), + title=message_content.title, + body=message_content.content, + ), ) try: messaging.send(message) - except messaging.FirebaseError as error: + except messaging.FirebaseError: hyperion_error_logger.exception( - f"Notification: Unable to send firebase notification for topic {custom_topic}: {error}", + f"Notification: Unable to send firebase notification for topic {custom_topic}", ) raise @@ -240,7 +240,9 @@ async def send_notification_to_topic( return try: - self._send_firebase_push_notification_by_topic(custom_topic=custom_topic, message_content=message) + self._send_firebase_push_notification_by_topic( + custom_topic=custom_topic, message_content=message, + ) except Exception as error: hyperion_error_logger.warning( f"Notification: Unable to send firebase notification for topic {custom_topic}: {error}", @@ -274,9 +276,12 @@ async def subscribe_user_to_topic( topic_membership=topic_membership, db=db, ) - tokens = await cruds_notification.get_firebase_tokens_by_user_ids(user_ids=[user_id], db=db) - await self.subscribe_tokens_to_topic(custom_topic=custom_topic, tokens=tokens) - + tokens = await cruds_notification.get_firebase_tokens_by_user_ids( + user_ids=[user_id], db=db, + ) + await self.subscribe_tokens_to_topic( + custom_topic=custom_topic, tokens=tokens, + ) async def unsubscribe_user_to_topic( self, @@ -292,9 +297,12 @@ async def unsubscribe_user_to_topic( user_id=user_id, db=db, ) - tokens = await cruds_notification.get_firebase_tokens_by_user_ids(user_ids=[user_id], db=db) + tokens = await cruds_notification.get_firebase_tokens_by_user_ids( + user_ids=[user_id], db=db, + ) await self.unsubscribe_tokens_to_topic(custom_topic=custom_topic, tokens=tokens) + class NotificationTool: """ Utility class to send notifications in the background. @@ -309,12 +317,12 @@ def __init__( background_tasks: BackgroundTasks, notification_manager: NotificationManager, db: AsyncSession, - #scheduler: Scheduler, + # scheduler: Scheduler, ): self.background_tasks = background_tasks self.notification_manager = notification_manager self.db = db - #self.scheduler = scheduler + # self.scheduler = scheduler async def send_notification_to_users(self, user_ids: list[str], message: Message): self.background_tasks.add_task( @@ -325,15 +333,15 @@ async def send_notification_to_users(self, user_ids: list[str], message: Message ) async def send_future_notification_to_users_time_defer( - self, - user_ids: list[str], - message: Message, - scheduler: Scheduler, - defer_seconds:float, - job_id:str, - ): - - await scheduler.queue_job_time_defer(self.notification_manager.send_notification_to_users, + self, + user_ids: list[str], + message: Message, + scheduler: Scheduler, + defer_seconds: float, + job_id: str, + ): + await scheduler.queue_job_time_defer( + self.notification_manager.send_notification_to_users, user_ids=user_ids, message=message, db=None, @@ -342,15 +350,15 @@ async def send_future_notification_to_users_time_defer( ) async def send_future_notification_to_users_defer_to( - self, - user_ids: list[str], - message: Message, - scheduler: Scheduler, - defer_date:datetime, - job_id:str, - ): - - await scheduler.queue_job_defer_to(self.notification_manager.send_notification_to_users, + self, + user_ids: list[str], + message: Message, + scheduler: Scheduler, + defer_date: datetime, + job_id: str, + ): + await scheduler.queue_job_defer_to( + self.notification_manager.send_notification_to_users, user_ids=user_ids, message=message, db=None, @@ -358,8 +366,6 @@ async def send_future_notification_to_users_defer_to( defer_date=defer_date, ) - - async def send_notification_to_user( self, user_id: str, @@ -387,11 +393,11 @@ async def send_future_notification_to_topic_defer_to( custom_topic: CustomTopic, message: Message, scheduler: Scheduler, - defer_date:datetime, - job_id:str, + defer_date: datetime, + job_id: str, ): - - await scheduler.queue_job_defer_to(self.notification_manager.send_notification_to_topic, + await scheduler.queue_job_defer_to( + self.notification_manager.send_notification_to_topic, custom_topic=custom_topic, message=message, db=None, @@ -404,11 +410,11 @@ async def send_future_notification_to_topic_time_defer( custom_topic: CustomTopic, message: Message, scheduler: Scheduler, - defer_seconds:float, - job_id:str, + defer_seconds: float, + job_id: str, ): - - await scheduler.queue_job_time_defer(self.notification_manager.send_notification_to_topic, + await scheduler.queue_job_time_defer( + self.notification_manager.send_notification_to_topic, custom_topic=custom_topic, message=message, db=None, From 5fef462d1c29885f343d2dc2b641554e3dc84b30 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Mon, 25 Nov 2024 13:10:43 +0100 Subject: [PATCH 086/113] ruff format --- app/core/endpoints_core.py | 1 - app/core/notification/endpoints_notification.py | 9 ++++++--- app/core/notification/schemas_notification.py | 2 -- app/modules/cinema/endpoints_cinema.py | 2 +- app/modules/loan/endpoints_loan.py | 2 +- app/modules/ph/endpoints_ph.py | 4 ++-- app/utils/communication/notifications.py | 12 ++++++++---- 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index 493b9aa3f..e97b8c263 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -25,7 +25,6 @@ hyperion_error_logger = logging.getLogger("hyperion.error") - @router.get( "/information", response_model=schemas_core.CoreInformation, diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index 7205756ed..ddd5941e6 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -232,13 +232,13 @@ async def send_notification( title="Test notification", content="Ceci est un test de notification", action_module="test", - ) await notification_tool.send_notification_to_user( user_id=user.id, message=message, ) + @router.post( "/notification/send/topic", status_code=201, @@ -258,10 +258,11 @@ async def send_notification_topic( action_module="test", ) await notification_tool.send_notification_to_topic( - custom_topic=CustomTopic.from_str("test"), + custom_topic=CustomTopic.from_str("test"), message=message, ) + @router.post( "/notification/send/topic", status_code=201, @@ -289,6 +290,7 @@ async def send_notification_topic( scheduler=scheduler, ) + @router.post( "/notification/send/topic/future", status_code=201, @@ -309,13 +311,14 @@ async def send_future_notification_topic( action_module="test", ) await notification_tool.send_future_notification_to_topic_time_defer( - custom_topic=CustomTopic.from_str("test"), + custom_topic=CustomTopic.from_str("test"), message=message, defer_seconds=10, job_id="test26", scheduler=scheduler, ) + @router.get( "/notification/devices", status_code=200, diff --git a/app/core/notification/schemas_notification.py b/app/core/notification/schemas_notification.py index 971d691f6..a7ff0f7b2 100644 --- a/app/core/notification/schemas_notification.py +++ b/app/core/notification/schemas_notification.py @@ -1,4 +1,3 @@ - from pydantic import BaseModel, Field @@ -12,7 +11,6 @@ class Message(BaseModel): ) - class FirebaseDevice(BaseModel): user_id: str = Field(description="The Hyperion user id") firebase_device_token: str = Field("Firebase device token") diff --git a/app/modules/cinema/endpoints_cinema.py b/app/modules/cinema/endpoints_cinema.py index b1fbd896e..fe8a61e9a 100644 --- a/app/modules/cinema/endpoints_cinema.py +++ b/app/modules/cinema/endpoints_cinema.py @@ -122,7 +122,7 @@ async def create_session( db: AsyncSession = Depends(get_db), user: models_core.CoreUser = Depends(is_user_in(GroupType.cinema)), notification_tool: NotificationTool = Depends(get_notification_tool), - scheduler:Scheduler = Depends(get_scheduler), + scheduler: Scheduler = Depends(get_scheduler), ): db_session = schemas_cinema.CineSessionComplete( id=str(uuid.uuid4()), diff --git a/app/modules/loan/endpoints_loan.py b/app/modules/loan/endpoints_loan.py index 8c1f9ad1f..4b7032602 100644 --- a/app/modules/loan/endpoints_loan.py +++ b/app/modules/loan/endpoints_loan.py @@ -927,7 +927,7 @@ async def extend_loan( loan_update=loan_update, db=db, ) - #TODO + # TODO await scheduler.cancel_job(f"loan_end_{loan.id}") diff --git a/app/modules/ph/endpoints_ph.py b/app/modules/ph/endpoints_ph.py index 632f76a44..444aa570b 100644 --- a/app/modules/ph/endpoints_ph.py +++ b/app/modules/ph/endpoints_ph.py @@ -107,7 +107,7 @@ async def create_paper( db: AsyncSession = Depends(get_db), user: models_core.CoreUser = Depends(is_user_in(GroupType.ph)), notification_tool: NotificationTool = Depends(get_notification_tool), - scheduler:Scheduler = Depends(get_scheduler), + scheduler: Scheduler = Depends(get_scheduler), ): """Create a new paper.""" @@ -136,7 +136,7 @@ async def create_paper( custom_topic=CustomTopic(topic=Topic.ph), message=message, ) - else : + else: delivery_time = time(11, 00, 00, tzinfo=UTC) release_date = datetime.combine(paper_db.release_date, delivery_time) await notification_tool.send_future_notification_to_topic_defer_to( diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 840356dfc..0ecea1e25 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -241,7 +241,8 @@ async def send_notification_to_topic( try: self._send_firebase_push_notification_by_topic( - custom_topic=custom_topic, message_content=message, + custom_topic=custom_topic, + message_content=message, ) except Exception as error: hyperion_error_logger.warning( @@ -277,10 +278,12 @@ async def subscribe_user_to_topic( db=db, ) tokens = await cruds_notification.get_firebase_tokens_by_user_ids( - user_ids=[user_id], db=db, + user_ids=[user_id], + db=db, ) await self.subscribe_tokens_to_topic( - custom_topic=custom_topic, tokens=tokens, + custom_topic=custom_topic, + tokens=tokens, ) async def unsubscribe_user_to_topic( @@ -298,7 +301,8 @@ async def unsubscribe_user_to_topic( db=db, ) tokens = await cruds_notification.get_firebase_tokens_by_user_ids( - user_ids=[user_id], db=db, + user_ids=[user_id], + db=db, ) await self.unsubscribe_tokens_to_topic(custom_topic=custom_topic, tokens=tokens) From f7bcaec538fffd9a383b5562c55c89625265689a Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+foucblg@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:00:54 +0100 Subject: [PATCH 087/113] Update app/types/scheduler.py Co-authored-by: Armand Didierjean <95971503+armanddidierjean@users.noreply.github.com> --- app/types/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 1e4cdcbfb..cb51e6527 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -98,7 +98,7 @@ async def queue_job_time_defer( self, job_function: Callable[..., Coroutine[Any, Any, Any]], job_id: str, - defer_seconds: float, + defer_seconds: int, **kwargs, ): """ From 40cef5cb4d0ca363e01b43d4552df0b3ce502968 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+foucblg@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:06:59 +0100 Subject: [PATCH 088/113] Update app/modules/advert/endpoints_advert.py Co-authored-by: Armand Didierjean <95971503+armanddidierjean@users.noreply.github.com> --- app/modules/advert/endpoints_advert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/modules/advert/endpoints_advert.py b/app/modules/advert/endpoints_advert.py index 44a285ee9..a52958cc4 100644 --- a/app/modules/advert/endpoints_advert.py +++ b/app/modules/advert/endpoints_advert.py @@ -268,7 +268,7 @@ async def create_advert( message = Message( title=f"📣 Annonce - {result.title}", content=result.content, - action_module="advert", + action_module=module.root, ) await notification_tool.send_notification_to_topic( From c7d5f1c7ef1a3449f96dcc6e8cdfa2898dd37e52 Mon Sep 17 00:00:00 2001 From: Paul Saubusse <146203570+Daihecyy@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:17:37 +0100 Subject: [PATCH 089/113] Update app/types/scheduler.py Simplifying expression Co-authored-by: Armand Didierjean <95971503+armanddidierjean@users.noreply.github.com> --- app/types/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index cb51e6527..d2a0d5a25 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -72,7 +72,7 @@ class ArqWorkerSettings(WorkerSettingsBase): redis_settings = RedisSettings( host=redis_host, port=redis_port, - password=redis_password if redis_password is not None else "", + password=redis_password or "", ) # We pass handle_signals=False to avoid arq from handling signals From 1dbc14d2dde1ca31be055da9f50fb4738f2306e3 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:04:01 +0100 Subject: [PATCH 090/113] Defer seconds -> int --- app/utils/communication/notifications.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 0ecea1e25..7dc306218 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -341,7 +341,7 @@ async def send_future_notification_to_users_time_defer( user_ids: list[str], message: Message, scheduler: Scheduler, - defer_seconds: float, + defer_seconds: int, job_id: str, ): await scheduler.queue_job_time_defer( @@ -414,7 +414,7 @@ async def send_future_notification_to_topic_time_defer( custom_topic: CustomTopic, message: Message, scheduler: Scheduler, - defer_seconds: float, + defer_seconds: int, job_id: str, ): await scheduler.queue_job_time_defer( From 949573a421358aa5685a2fceeb0dc7503d57dbd4 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:32:29 +0100 Subject: [PATCH 091/113] Feat : unique function to send notif --- app/modules/cinema/endpoints_cinema.py | 2 +- app/modules/loan/endpoints_loan.py | 4 +- app/modules/ph/endpoints_ph.py | 2 +- app/utils/communication/notifications.py | 72 +++++++++++++++++++----- 4 files changed, 63 insertions(+), 17 deletions(-) diff --git a/app/modules/cinema/endpoints_cinema.py b/app/modules/cinema/endpoints_cinema.py index fe8a61e9a..f59eeb991 100644 --- a/app/modules/cinema/endpoints_cinema.py +++ b/app/modules/cinema/endpoints_cinema.py @@ -149,7 +149,7 @@ async def create_session( action_module="cinema", ) - await notification_tool.send_future_notification_to_topic_defer_to( + await notification_tool.send_notification_to_topic( custom_topic=CustomTopic(topic=Topic.cinema), message=message, scheduler=scheduler, diff --git a/app/modules/loan/endpoints_loan.py b/app/modules/loan/endpoints_loan.py index 4b7032602..a5e605754 100644 --- a/app/modules/loan/endpoints_loan.py +++ b/app/modules/loan/endpoints_loan.py @@ -634,7 +634,7 @@ async def create_loan( module="loan", ) - await notification_tool.send_future_notification_to_users_defer_to( + await notification_tool.send_notification_to_users( user_ids=[loan.borrower_id], message=message, scheduler=scheduler, @@ -950,7 +950,7 @@ async def extend_loan( module="loan", ) - await notification_tool.send_future_notification_to_users_defer_to( + await notification_tool.send_notification_to_users( user_ids=[loan.borrower_id], message=message, scheduler=scheduler, diff --git a/app/modules/ph/endpoints_ph.py b/app/modules/ph/endpoints_ph.py index 444aa570b..d3f1da40a 100644 --- a/app/modules/ph/endpoints_ph.py +++ b/app/modules/ph/endpoints_ph.py @@ -139,7 +139,7 @@ async def create_paper( else: delivery_time = time(11, 00, 00, tzinfo=UTC) release_date = datetime.combine(paper_db.release_date, delivery_time) - await notification_tool.send_future_notification_to_topic_defer_to( + await notification_tool.send_notification_to_topic( custom_topic=CustomTopic(topic=Topic.ph), message=message, scheduler=scheduler, diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 7dc306218..a6950eb83 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -328,13 +328,38 @@ def __init__( self.db = db # self.scheduler = scheduler - async def send_notification_to_users(self, user_ids: list[str], message: Message): - self.background_tasks.add_task( - self.notification_manager.send_notification_to_users, - user_ids=user_ids, - message=message, - db=self.db, - ) + async def send_notification_to_users( + self, + user_ids: list[str], + message: Message, + scheduler: Scheduler | None = None, + defer_seconds: int | None = None, + defer_date: datetime | None = None, + job_id: str | None = None, + ): + if defer_seconds is not None and scheduler is not None and job_id is not None: + await self.send_future_notification_to_users_time_defer( + user_ids=user_ids, + message=message, + scheduler=scheduler, + defer_seconds=defer_seconds, + job_id=job_id, + ) + elif defer_date is not None and scheduler is not None and job_id is not None: + await self.send_future_notification_to_users_defer_to( + user_ids=user_ids, + message=message, + scheduler=scheduler, + defer_date=defer_date, + job_id=job_id, + ) + else: + self.background_tasks.add_task( + self.notification_manager.send_notification_to_users, + user_ids=user_ids, + message=message, + db=self.db, + ) async def send_future_notification_to_users_time_defer( self, @@ -384,13 +409,34 @@ async def send_notification_to_topic( self, custom_topic: CustomTopic, message: Message, + scheduler: Scheduler | None = None, + defer_seconds: int | None = None, + defer_date: datetime | None = None, + job_id: str | None = None, ): - self.background_tasks.add_task( - self.notification_manager.send_notification_to_topic, - custom_topic=custom_topic, - message=message, - db=self.db, - ) + if defer_seconds is not None and scheduler is not None and job_id is not None: + await self.send_future_notification_to_topic_time_defer( + custom_topic=custom_topic, + message=message, + scheduler=scheduler, + defer_seconds=defer_seconds, + job_id=job_id, + ) + elif defer_date is not None and scheduler is not None and job_id is not None: + await self.send_future_notification_to_topic_defer_to( + custom_topic=custom_topic, + message=message, + scheduler=scheduler, + defer_date=defer_date, + job_id=job_id, + ) + else: + self.background_tasks.add_task( + self.notification_manager.send_notification_to_topic, + custom_topic=custom_topic, + message=message, + db=self.db, + ) async def send_future_notification_to_topic_defer_to( self, From 3aebcc4a488cd36a46f7390b9f0a26ac08b4cc91 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:34:52 +0100 Subject: [PATCH 092/113] Fix avoiding n notification for cine --- app/modules/cinema/endpoints_cinema.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/modules/cinema/endpoints_cinema.py b/app/modules/cinema/endpoints_cinema.py index f59eeb991..8592793e6 100644 --- a/app/modules/cinema/endpoints_cinema.py +++ b/app/modules/cinema/endpoints_cinema.py @@ -149,6 +149,8 @@ async def create_session( action_module="cinema", ) + await scheduler.cancel_job(job_id=f"cinema_weekly_{sunday}") + await notification_tool.send_notification_to_topic( custom_topic=CustomTopic(topic=Topic.cinema), message=message, From de853f898ef6ff2915dcd16fdf3a9b5d8077a769 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:53:44 +0100 Subject: [PATCH 093/113] Fix : update topic membership for new tokens --- .../notification/endpoints_notification.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index ddd5941e6..02914fabb 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -63,6 +63,15 @@ async def register_firebase_device( db=db, ) + user_topics = await cruds_notification.get_topic_memberships_by_user_id( + user_id=user.id, db=db, + ) + for topic in user_topics: + await notification_manager.subscribe_tokens_to_topic( + custom_topic=CustomTopic(topic.topic), + tokens=[firebase_token], + ) + firebase_device = models_notification.FirebaseDevice( user_id=user.id, firebase_device_token=firebase_token, @@ -86,7 +95,7 @@ async def unregister_firebase_device( notification_manager: NotificationManager = Depends(get_notification_manager), ): """ - Unregister a new firebase device for the user + Unregister a firebase device for the user **The user must be authenticated to use this endpoint** """ @@ -97,6 +106,16 @@ async def unregister_firebase_device( db=db, ) + user_topics = await cruds_notification.get_topic_memberships_by_user_id( + user_id=user.id, + db=db, + ) + for topic in user_topics: + await notification_manager.unsubscribe_tokens_to_topic( + custom_topic=CustomTopic(topic.topic), + tokens=[firebase_token], + ) + @router.post( "/notification/topics/{topic_str}/subscribe", From 11c9482b06479843cbfe0b33231795cfba4da31a Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:56:09 +0100 Subject: [PATCH 094/113] lint and format grmf --- app/core/notification/endpoints_notification.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index 02914fabb..17ec6a8d4 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -64,7 +64,8 @@ async def register_firebase_device( ) user_topics = await cruds_notification.get_topic_memberships_by_user_id( - user_id=user.id, db=db, + user_id=user.id, + db=db, ) for topic in user_topics: await notification_manager.subscribe_tokens_to_topic( From 03ce388f75a6b6cdc5e59d903de4f3f5aa2ab47c Mon Sep 17 00:00:00 2001 From: Daihecyy Date: Tue, 3 Dec 2024 17:30:58 +0100 Subject: [PATCH 095/113] Revert test line --- app/core/endpoints_core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index e97b8c263..ad96d9af6 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -17,7 +17,6 @@ is_user_in, ) from app.modules.module_list import module_list -from app.types.scheduler import Scheduler from app.utils.tools import is_group_id_valid router = APIRouter(tags=["Core"]) @@ -32,7 +31,6 @@ ) async def read_information( settings: Settings = Depends(get_settings), - scheduler: Scheduler = Depends(get_scheduler), ): """ Return information about Hyperion. This endpoint can be used to check if the API is up. From 205e0d703132cad99a453a04cdc3325dd0e39e12 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:35:33 +0100 Subject: [PATCH 096/113] Fix : Cancel notification with notif manager --- app/modules/cinema/endpoints_cinema.py | 5 ++++- app/modules/loan/endpoints_loan.py | 13 +++++++++---- app/utils/communication/notifications.py | 7 +++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/app/modules/cinema/endpoints_cinema.py b/app/modules/cinema/endpoints_cinema.py index 8592793e6..99b4ff5ce 100644 --- a/app/modules/cinema/endpoints_cinema.py +++ b/app/modules/cinema/endpoints_cinema.py @@ -149,7 +149,10 @@ async def create_session( action_module="cinema", ) - await scheduler.cancel_job(job_id=f"cinema_weekly_{sunday}") + await notification_tool.cancel_notification( + scheduler=scheduler, + job_id=f"cinema_weekly_{sunday}", + ) await notification_tool.send_notification_to_topic( custom_topic=CustomTopic(topic=Topic.cinema), diff --git a/app/modules/loan/endpoints_loan.py b/app/modules/loan/endpoints_loan.py index a5e605754..4369a68b0 100644 --- a/app/modules/loan/endpoints_loan.py +++ b/app/modules/loan/endpoints_loan.py @@ -828,6 +828,7 @@ async def return_loan( db: AsyncSession = Depends(get_db), user: models_core.CoreUser = Depends(is_user_a_member), scheduler: Scheduler = Depends(get_scheduler), + notification_tool: NotificationTool = Depends(get_notification_tool), ): """ Mark a loan as returned. This will update items availability. @@ -872,7 +873,10 @@ async def return_loan( returned=True, returned_date=datetime.now(UTC), ) - await scheduler.cancel_job(f"loan_end_{loan.id}") + await notification_tool.cancel_notification( + scheduler=scheduler, + job_id=f"loan_end_{loan.id}", + ) @module.router.post( @@ -927,9 +931,10 @@ async def extend_loan( loan_update=loan_update, db=db, ) - # TODO - - await scheduler.cancel_job(f"loan_end_{loan.id}") + await notification_tool.cancel_notification( + scheduler=scheduler, + job_id=f"loan_end_{loan.id}", + ) message = Message( title="📦 Prêt prolongé", diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index a6950eb83..8b5c5b72f 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -471,3 +471,10 @@ async def send_future_notification_to_topic_time_defer( job_id=job_id, defer_seconds=defer_seconds, ) + + async def cancel_notification( + self, + scheduler: Scheduler, + job_id: str, + ): + await scheduler.cancel_job(job_id=job_id) From c6b36fa7587b2a55f74556dcfafa63a056756706 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:39:00 +0100 Subject: [PATCH 097/113] Dummy Scheduler to Offlinescheduler --- app/dependencies.py | 4 ++-- app/types/scheduler.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/dependencies.py b/app/dependencies.py index 2e63e97d5..1531408f9 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -27,7 +27,7 @@ async def get_users(db: AsyncSession = Depends(get_db)): from app.core.groups.groups_type import AccountType, GroupType, get_ecl_account_types from app.core.payment.payment_tool import PaymentTool from app.modules.raid.utils.drive.drive_file_manager import DriveFileManager -from app.types.scheduler import DummyScheduler, Scheduler +from app.types.scheduler import OfflineScheduler, Scheduler from app.types.scopes_type import ScopeType from app.types.websocket import WebsocketConnectionManager from app.utils.auth import auth_utils @@ -167,7 +167,7 @@ def get_redis_client( def get_scheduler(settings: Settings = Depends(get_settings)) -> Scheduler: global scheduler if scheduler is None: - scheduler = Scheduler() if settings.REDIS_HOST != "" else DummyScheduler() + scheduler = Scheduler() if settings.REDIS_HOST != "" else OfflineScheduler() return scheduler diff --git a/app/types/scheduler.py b/app/types/scheduler.py index d2a0d5a25..573667511 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -154,7 +154,7 @@ async def cancel_job(self, job_id: str): await job.abort() -class DummyScheduler(Scheduler): +class OfflineScheduler(Scheduler): """ A Dummy implementation of the Scheduler to allow to run the server without a REDIS config """ @@ -186,7 +186,7 @@ async def start( """ self._get_db = _get_db - scheduler_logger.info("DummyScheduler started") + scheduler_logger.info("OfflineScheduler started") async def close(self): # If the worker was started, we close it @@ -204,7 +204,7 @@ async def queue_job_time_defer( job_id will allow to abort if needed """ scheduler_logger.debug( - f"Job {job_id} queued in DummyScheduler with defer {defer_seconds} seconds", + f"Job {job_id} queued in OfflineScheduler with defer {defer_seconds} seconds", ) async def queue_job_defer_to( @@ -219,11 +219,11 @@ async def queue_job_defer_to( job_id will allow to abort if needed """ scheduler_logger.debug( - f"Job {job_id} queued in DummyScheduler with defer to {defer_date}", + f"Job {job_id} queued in OfflineScheduler with defer to {defer_date}", ) async def cancel_job(self, job_id: str): """ cancel a queued job based on its job_id """ - scheduler_logger.debug(f"Job {job_id} aborted in DummyScheduler") + scheduler_logger.debug(f"Job {job_id} aborted in OfflineScheduler") From 69dd104f25062cc667f7d98bc37e3f924ca26320 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:42:12 +0100 Subject: [PATCH 098/113] Feat : removing scheduler time defer --- .../notification/endpoints_notification.py | 10 ++-- app/types/scheduler.py | 15 ----- app/utils/communication/notifications.py | 55 +------------------ 3 files changed, 7 insertions(+), 73 deletions(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index 17ec6a8d4..36a5e6623 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -1,4 +1,4 @@ -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from fastapi import APIRouter, Body, Depends, HTTPException, Path from sqlalchemy.ext.asyncio import AsyncSession @@ -302,10 +302,10 @@ async def send_notification_topic( content="Ceci est un test de notification future", action_module="test", ) - await notification_tool.send_future_notification_to_users_time_defer( + await notification_tool.send_notification_to_users( user_ids=[user.id], message=message, - defer_seconds=10, + defer_date=datetime.now(UTC) + timedelta(seconds=10), job_id="test25", scheduler=scheduler, ) @@ -330,10 +330,10 @@ async def send_future_notification_topic( content="Ceci est un test de notification future topic", action_module="test", ) - await notification_tool.send_future_notification_to_topic_time_defer( + await notification_tool.send_notification_to_topic( custom_topic=CustomTopic.from_str("test"), message=message, - defer_seconds=10, + defer_date=datetime.now(UTC) + timedelta(seconds=10), job_id="test26", scheduler=scheduler, ) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 573667511..d1cd47d99 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -192,21 +192,6 @@ async def close(self): # If the worker was started, we close it pass - async def queue_job_time_defer( - self, - job_function: Callable[..., Coroutine[Any, Any, Any]], - job_id: str, - defer_seconds: float, - **kwargs, - ): - """ - Queue a job to execute job_function in defer_seconds amount of seconds - job_id will allow to abort if needed - """ - scheduler_logger.debug( - f"Job {job_id} queued in OfflineScheduler with defer {defer_seconds} seconds", - ) - async def queue_job_defer_to( self, job_function: Callable[..., Coroutine[Any, Any, Any]], diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 8b5c5b72f..4b1322944 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -337,15 +337,7 @@ async def send_notification_to_users( defer_date: datetime | None = None, job_id: str | None = None, ): - if defer_seconds is not None and scheduler is not None and job_id is not None: - await self.send_future_notification_to_users_time_defer( - user_ids=user_ids, - message=message, - scheduler=scheduler, - defer_seconds=defer_seconds, - job_id=job_id, - ) - elif defer_date is not None and scheduler is not None and job_id is not None: + if defer_date is not None and scheduler is not None and job_id is not None: await self.send_future_notification_to_users_defer_to( user_ids=user_ids, message=message, @@ -361,23 +353,6 @@ async def send_notification_to_users( db=self.db, ) - async def send_future_notification_to_users_time_defer( - self, - user_ids: list[str], - message: Message, - scheduler: Scheduler, - defer_seconds: int, - job_id: str, - ): - await scheduler.queue_job_time_defer( - self.notification_manager.send_notification_to_users, - user_ids=user_ids, - message=message, - db=None, - job_id=job_id, - defer_seconds=defer_seconds, - ) - async def send_future_notification_to_users_defer_to( self, user_ids: list[str], @@ -410,19 +385,10 @@ async def send_notification_to_topic( custom_topic: CustomTopic, message: Message, scheduler: Scheduler | None = None, - defer_seconds: int | None = None, defer_date: datetime | None = None, job_id: str | None = None, ): - if defer_seconds is not None and scheduler is not None and job_id is not None: - await self.send_future_notification_to_topic_time_defer( - custom_topic=custom_topic, - message=message, - scheduler=scheduler, - defer_seconds=defer_seconds, - job_id=job_id, - ) - elif defer_date is not None and scheduler is not None and job_id is not None: + if defer_date is not None and scheduler is not None and job_id is not None: await self.send_future_notification_to_topic_defer_to( custom_topic=custom_topic, message=message, @@ -455,23 +421,6 @@ async def send_future_notification_to_topic_defer_to( defer_date=defer_date, ) - async def send_future_notification_to_topic_time_defer( - self, - custom_topic: CustomTopic, - message: Message, - scheduler: Scheduler, - defer_seconds: int, - job_id: str, - ): - await scheduler.queue_job_time_defer( - self.notification_manager.send_notification_to_topic, - custom_topic=custom_topic, - message=message, - db=None, - job_id=job_id, - defer_seconds=defer_seconds, - ) - async def cancel_notification( self, scheduler: Scheduler, From 716def37a3a2ad1c5a8e01705b84715755eae512 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:44:54 +0100 Subject: [PATCH 099/113] Feat : arq logging --- app/core/log.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/core/log.py b/app/core/log.py index 2b62d3d4f..a270c7c0a 100644 --- a/app/core/log.py +++ b/app/core/log.py @@ -246,6 +246,14 @@ def get_config_dict(self, settings: Settings): "level": MINIMUM_LOG_LEVEL, "propagate": False, }, + "arq.worker": { + "handlers": [ + "console", + "files_errors", + "matrix_errors", + ], + "level": MINIMUM_LOG_LEVEL, + }, }, } From 0c61eec62e2d506343d2d41b15013cfbdd003d2d Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:49:52 +0100 Subject: [PATCH 100/113] Fix : arq.warker log --- app/core/log.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/core/log.py b/app/core/log.py index a270c7c0a..6e1b62d4b 100644 --- a/app/core/log.py +++ b/app/core/log.py @@ -249,10 +249,11 @@ def get_config_dict(self, settings: Settings): "arq.worker": { "handlers": [ "console", - "files_errors", + "file_errors", "matrix_errors", ], "level": MINIMUM_LOG_LEVEL, + "propagate": False, }, }, } From 0c641a60ba9d7a89b960448465f873db7a1df94a Mon Sep 17 00:00:00 2001 From: armanddidierjean <95971503+armanddidierjean@users.noreply.github.com> Date: Tue, 3 Dec 2024 20:48:38 +0100 Subject: [PATCH 101/113] Dependency injection in the scheduler --- app/app.py | 9 +----- app/types/scheduler.py | 71 +++++++++++++++++++++++++++++++----------- 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/app/app.py b/app/app.py index 03a61b249..1979caefa 100644 --- a/app/app.py +++ b/app/app.py @@ -364,19 +364,12 @@ async def lifespan(app: FastAPI) -> AsyncGenerator: get_scheduler, )(settings=settings) - _get_db: Callable[[], AsyncGenerator[AsyncSession, None]] = ( - app.dependency_overrides.get( - get_db, - get_db, - ) - ) - await ws_manager.connect_broadcaster() await arq_scheduler.start( redis_host=settings.REDIS_HOST, redis_port=settings.REDIS_PORT, redis_password=settings.REDIS_PASSWORD, - _get_db=_get_db, + _dependency_overrides=app.dependency_overrides, ) yield diff --git a/app/types/scheduler.py b/app/types/scheduler.py index d1cd47d99..c48cfbd6c 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -2,33 +2,69 @@ import logging from collections.abc import AsyncGenerator, Callable, Coroutine from datetime import datetime, timedelta +from inspect import iscoroutinefunction, signature from typing import TYPE_CHECKING, Any from arq.connections import RedisSettings from arq.jobs import Job from arq.typing import WorkerSettingsBase from arq.worker import create_worker -from sqlalchemy.ext.asyncio import AsyncSession +from fastapi import params + +from app import dependencies if TYPE_CHECKING: from arq import Worker + from sqlalchemy.ext.asyncio import AsyncSession scheduler_logger = logging.getLogger("scheduler") async def run_task( ctx, - job_function: Callable[..., Coroutine[Any, Any, Any]], - _get_db: Callable[[], AsyncGenerator[AsyncSession, None]], + job_function: Callable[..., Any], + _dependency_overrides: dict[Callable[..., Any], Callable[..., Any]], **kwargs, ): - scheduler_logger.debug(f"Job function consumed {job_function}") + """ + Execute the job_function with the provided kwargs - if "db" in kwargs: - async for db in _get_db(): - kwargs["db"] = db + `job_function` may be a coroutine function or a regular function + + The method will inject the following known dependencies into the job_function if needed: + - `get_db` + """ + scheduler_logger.debug(f"Job function consumed {job_function}") - return await job_function(**kwargs) + sign = signature(job_function) + for param in sign.parameters.values(): + if isinstance(param.default, params.Depends): + # We iterate over the parameters of the job_function + # If we find a Depends object, we may want to inject the dependency + + # It is not easy to support any kind of Depends object + # as the corresponding method may be async or not, or be a special FastAPI object + # like BackgroundTasks or Request + + # For now we filter accepted Depends objects manually + # and only accept `get_db` + if param.default.dependency == dependencies.get_db: + # `get_db` is the real dependency, defined in dependency.py + # `_get_db` may be the real dependency or an override + _get_db: Callable[[], AsyncGenerator[AsyncSession, None]] = ( + _dependency_overrides.get( + dependencies.get_db, + dependencies.get_db, + ) + ) + + async for db in _get_db(): + kwargs["db"] = db + + if iscoroutinefunction(job_function): + return await job_function(**kwargs) + else: + return job_function(**kwargs) class Scheduler: @@ -47,14 +83,14 @@ def __init__(self): # Task will contain the asyncio task that runs the worker self.task: asyncio.Task | None = None # Pointer to the get_db dependency - # self._get_db: Callable[[], AsyncGenerator[AsyncSession, None]] + # self._dependency_overrides: Callable[[], AsyncGenerator[AsyncSession, None]] async def start( self, redis_host: str, redis_port: int, redis_password: str | None, - _get_db: Callable[[], AsyncGenerator[AsyncSession, None]], + _dependency_overrides: dict[Callable[..., Any], Callable[..., Any]], **kwargs, ): """ @@ -62,7 +98,7 @@ async def start( - redis_host: str - redis_port: int - redis_password: str - - get_db: Callable[[], AsyncGenerator[AsyncSession, None]] a pointer to `get_db` dependency + - _dependency_overrides: dict[Callable[..., Any], Callable[..., Any]] a pointer to the app dependency overrides dict """ class ArqWorkerSettings(WorkerSettingsBase): @@ -85,7 +121,7 @@ class ArqWorkerSettings(WorkerSettingsBase): # We run the worker in an asyncio task self.task = asyncio.create_task(self.worker.async_run()) - self._get_db = _get_db + self._dependency_overrides = _dependency_overrides scheduler_logger.info("Scheduler started") @@ -113,7 +149,7 @@ async def queue_job_time_defer( job_function=job_function, _job_id=job_id, _defer_by=timedelta(seconds=defer_seconds), - _get_db=self._get_db, + _dependency_overrides=self._dependency_overrides, **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") @@ -137,7 +173,7 @@ async def queue_job_defer_to( job_function=job_function, _job_id=job_id, _defer_until=defer_date, - _get_db=self._get_db, + _dependency_overrides=self._dependency_overrides, **kwargs, ) scheduler_logger.debug(f"Job {job_id} queued {job}") @@ -167,14 +203,13 @@ def __init__(self): # Task will contain the asyncio task that runs the worker self.task: asyncio.Task | None = None # Pointer to the get_db dependency - # self._get_db: Callable[[], AsyncGenerator[AsyncSession, None]] async def start( self, redis_host: str, redis_port: int, redis_password: str | None, - _get_db: Callable[[], AsyncGenerator[AsyncSession, None]], + _dependency_overrides: dict[Callable[..., Any], Callable[..., Any]], **kwargs, ): """ @@ -182,9 +217,9 @@ async def start( - redis_host: str - redis_port: int - redis_password: str - - get_db: Callable[[], AsyncGenerator[AsyncSession, None]] a pointer to `get_db` dependency + - _dependency_overrides: dict[Callable[..., Any], Callable[..., Any]] a pointer to the app dependency overrides dict """ - self._get_db = _get_db + self._dependency_overrides = _dependency_overrides scheduler_logger.info("OfflineScheduler started") From ba99ad5cbf4a572ee7f31e7e9368abc248d26e79 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Wed, 4 Dec 2024 08:36:41 +0100 Subject: [PATCH 102/113] Lint and format --- app/app.py | 1 - app/core/notification/endpoints_notification.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/app.py b/app/app.py index 1979caefa..fdafb3461 100644 --- a/app/app.py +++ b/app/app.py @@ -42,7 +42,6 @@ if TYPE_CHECKING: import redis - from sqlalchemy.ext.asyncio import AsyncSession from app.types.scheduler import Scheduler from app.types.websocket import WebsocketConnectionManager diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index 36a5e6623..76e989548 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -306,7 +306,7 @@ async def send_notification_topic( user_ids=[user.id], message=message, defer_date=datetime.now(UTC) + timedelta(seconds=10), - job_id="test25", + job_id="test", scheduler=scheduler, ) From 7eea06810b5c482db2431bb71a75ce47e51c7d52 Mon Sep 17 00:00:00 2001 From: Thonyk Date: Wed, 4 Dec 2024 13:02:55 +0100 Subject: [PATCH 103/113] Fix: rebase --- .../notification/endpoints_notification.py | 32 ++----------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index 76e989548..82c918b20 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -264,7 +264,7 @@ async def send_notification( status_code=201, ) async def send_notification_topic( - user: models_core.CoreUser = Depends(is_user_a_member_of(GroupType.admin)), + user: models_core.CoreUser = Depends(is_user_in(GroupType.admin)), notification_tool: NotificationTool = Depends(get_notification_tool), ): """ @@ -283,40 +283,12 @@ async def send_notification_topic( ) -@router.post( - "/notification/send/topic", - status_code=201, -) -async def send_notification_topic( - user: models_core.CoreUser = Depends(is_user_in(GroupType.admin)), - scheduler: Scheduler = Depends(get_scheduler), - notification_tool: NotificationTool = Depends(get_notification_tool), -): - """ - Send ourself a test notification. - - **Only admins can use this endpoint** - """ - message = schemas_notification.Message( - title="Test notification future", - content="Ceci est un test de notification future", - action_module="test", - ) - await notification_tool.send_notification_to_users( - user_ids=[user.id], - message=message, - defer_date=datetime.now(UTC) + timedelta(seconds=10), - job_id="test", - scheduler=scheduler, - ) - - @router.post( "/notification/send/topic/future", status_code=201, ) async def send_future_notification_topic( - user: models_core.CoreUser = Depends(is_user_a_member_of(GroupType.admin)), + user: models_core.CoreUser = Depends(is_user_in(GroupType.admin)), notification_tool: NotificationTool = Depends(get_notification_tool), scheduler: Scheduler = Depends(get_scheduler), ): From 89eea29cafa3acd0b0427de7b62c7ec7d1c9a316 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:20:20 +0100 Subject: [PATCH 104/113] Ruff --- app/core/endpoints_core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/core/endpoints_core.py b/app/core/endpoints_core.py index ad96d9af6..a23c7371b 100644 --- a/app/core/endpoints_core.py +++ b/app/core/endpoints_core.py @@ -11,7 +11,6 @@ from app.core.groups.groups_type import AccountType, GroupType from app.dependencies import ( get_db, - get_scheduler, get_settings, is_user, is_user_in, From 4ceea58835f5c6a9acac942781eee23c4f6607da Mon Sep 17 00:00:00 2001 From: armanddidierjean <95971503+armanddidierjean@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:54:27 +0100 Subject: [PATCH 105/113] Inject db in run_task --- app/types/exceptions.py | 5 ++ app/types/scheduler.py | 102 ++++++++++++++++------------------------ app/utils/tools.py | 18 ++++++- 3 files changed, 62 insertions(+), 63 deletions(-) diff --git a/app/types/exceptions.py b/app/types/exceptions.py index fa1619753..29687babd 100644 --- a/app/types/exceptions.py +++ b/app/types/exceptions.py @@ -126,3 +126,8 @@ def __init__(self, email: str): super().__init__( f"An account with the email {email} already exist", ) + + +class SchedulerNotStartedError(Exception): + def __init__(self): + super().__init__("Scheduler not started") diff --git a/app/types/scheduler.py b/app/types/scheduler.py index c48cfbd6c..7f831c907 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -1,17 +1,18 @@ import asyncio import logging from collections.abc import AsyncGenerator, Callable, Coroutine -from datetime import datetime, timedelta -from inspect import iscoroutinefunction, signature +from datetime import datetime +from inspect import signature from typing import TYPE_CHECKING, Any from arq.connections import RedisSettings from arq.jobs import Job from arq.typing import WorkerSettingsBase from arq.worker import create_worker -from fastapi import params from app import dependencies +from app.types.exceptions import SchedulerNotStartedError +from app.utils.tools import execute_async_or_sync_method if TYPE_CHECKING: from arq import Worker @@ -21,7 +22,7 @@ async def run_task( - ctx, + ctx: dict[Any, Any] | None, job_function: Callable[..., Any], _dependency_overrides: dict[Callable[..., Any], Callable[..., Any]], **kwargs, @@ -31,40 +32,43 @@ async def run_task( `job_function` may be a coroutine function or a regular function - The method will inject the following known dependencies into the job_function if needed: - - `get_db` + The method will inject an `AsyncSession` object, using `get_db`, in the kwargs if the job_function requires it + + NOTE: As a consequence, it is not possible to plan a job using a custom AsyncSession. + Passing a custom AsyncSession would not be advisable as it would require the + db connection to remain open for the duration of the job. """ scheduler_logger.debug(f"Job function consumed {job_function}") + require_db_for_kwargs: list[str] = [] sign = signature(job_function) for param in sign.parameters.values(): - if isinstance(param.default, params.Depends): + if isinstance(param.default, AsyncSession): # We iterate over the parameters of the job_function - # If we find a Depends object, we may want to inject the dependency - - # It is not easy to support any kind of Depends object - # as the corresponding method may be async or not, or be a special FastAPI object - # like BackgroundTasks or Request - - # For now we filter accepted Depends objects manually - # and only accept `get_db` - if param.default.dependency == dependencies.get_db: - # `get_db` is the real dependency, defined in dependency.py - # `_get_db` may be the real dependency or an override - _get_db: Callable[[], AsyncGenerator[AsyncSession, None]] = ( - _dependency_overrides.get( - dependencies.get_db, - dependencies.get_db, - ) - ) - - async for db in _get_db(): - kwargs["db"] = db - - if iscoroutinefunction(job_function): - return await job_function(**kwargs) + # If we find a AsyncSession object, we want to inject the dependency + require_db_for_kwargs.append(param.name) + else: + # We could support other types of dependencies + pass + + # We distinguish between methods requiring a db and those that don't + # to only open the db connection when needed + if require_db_for_kwargs: + # `get_db` is the real dependency, defined in dependency.py + # `_get_db` may be the real dependency or an override + _get_db: Callable[[], AsyncGenerator[AsyncSession, None]] = ( + _dependency_overrides.get( + dependencies.get_db, + dependencies.get_db, + ) + ) + + async for db in _get_db(): + for name in require_db_for_kwargs: + kwargs[name] = db + await execute_async_or_sync_method(job_function, **kwargs) else: - return job_function(**kwargs) + await execute_async_or_sync_method(job_function, **kwargs) class Scheduler: @@ -77,13 +81,14 @@ class Scheduler: # See https://github.com/fastapi/fastapi/discussions/9143#discussioncomment-5157572 + # Pointer to the app dependency overrides dict + _dependency_overrides: dict[Callable[..., Any], Callable[..., Any]] + def __init__(self): # ArqWorker, in charge of scheduling and executing tasks self.worker: Worker | None = None # Task will contain the asyncio task that runs the worker self.task: asyncio.Task | None = None - # Pointer to the get_db dependency - # self._dependency_overrides: Callable[[], AsyncGenerator[AsyncSession, None]] async def start( self, @@ -130,30 +135,6 @@ async def close(self): if self.worker is not None: await self.worker.close() - async def queue_job_time_defer( - self, - job_function: Callable[..., Coroutine[Any, Any, Any]], - job_id: str, - defer_seconds: int, - **kwargs, - ): - """ - Queue a job to execute job_function in defer_seconds amount of seconds - job_id will allow to abort if needed - """ - if self.worker is None: - scheduler_logger.error("Scheduler not started") - return None - job = await self.worker.pool.enqueue_job( - "run_task", - job_function=job_function, - _job_id=job_id, - _defer_by=timedelta(seconds=defer_seconds), - _dependency_overrides=self._dependency_overrides, - **kwargs, - ) - scheduler_logger.debug(f"Job {job_id} queued {job}") - async def queue_job_defer_to( self, job_function: Callable[..., Coroutine[Any, Any, Any]], @@ -166,8 +147,8 @@ async def queue_job_defer_to( job_id will allow to abort if needed """ if self.worker is None: - scheduler_logger.error("Scheduler not started") - return None + raise SchedulerNotStartedError + job = await self.worker.pool.enqueue_job( "run_task", job_function=job_function, @@ -183,8 +164,7 @@ async def cancel_job(self, job_id: str): cancel a queued job based on its job_id """ if self.worker is None: - scheduler_logger.error("Scheduler not started") - return None + raise SchedulerNotStartedError job = Job(job_id, redis=self.worker.pool) scheduler_logger.debug(f"Job aborted {job}") await job.abort() diff --git a/app/utils/tools.py b/app/utils/tools.py index ec5ab5844..7b0077ad4 100644 --- a/app/utils/tools.py +++ b/app/utils/tools.py @@ -4,9 +4,10 @@ import re import secrets import unicodedata -from collections.abc import Sequence +from collections.abc import Callable, Sequence +from inspect import iscoroutinefunction from pathlib import Path -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar import aiofiles import fitz @@ -493,3 +494,16 @@ async def create_and_send_email_migration( hyperion_security_logger.info( f"You can confirm your new email address by clicking the following link: {settings.CLIENT_URL}users/migrate-mail-confirm?token={confirmation_token}", ) + + +async def execute_async_or_sync_method( + job_function: Callable[..., Any], + **kwargs, +): + """ + Execute the job_function with the provided kwargs, either as a coroutine or a regular function. + """ + if iscoroutinefunction(job_function): + return await job_function(**kwargs) + else: + return job_function(**kwargs) From 797932dfd15092c1701fac9dd3fce2eb788e6713 Mon Sep 17 00:00:00 2001 From: armanddidierjean <95971503+armanddidierjean@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:58:11 +0100 Subject: [PATCH 106/113] Don't pass db=None as the new structure does not require it --- app/utils/communication/notifications.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index 4b1322944..d6ec611a3 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -365,7 +365,6 @@ async def send_future_notification_to_users_defer_to( self.notification_manager.send_notification_to_users, user_ids=user_ids, message=message, - db=None, job_id=job_id, defer_date=defer_date, ) @@ -416,7 +415,6 @@ async def send_future_notification_to_topic_defer_to( self.notification_manager.send_notification_to_topic, custom_topic=custom_topic, message=message, - db=None, job_id=job_id, defer_date=defer_date, ) From 7f4f548c0ccb1d2face0523ea40fc4dd67a2c809 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:22:16 +0100 Subject: [PATCH 107/113] Add test endpoint --- .../notification/endpoints_notification.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/app/core/notification/endpoints_notification.py b/app/core/notification/endpoints_notification.py index 82c918b20..3452a4a25 100644 --- a/app/core/notification/endpoints_notification.py +++ b/app/core/notification/endpoints_notification.py @@ -259,6 +259,34 @@ async def send_notification( ) +@router.post( + "/notification/send/future", + status_code=201, +) +async def send_future_notification( + user: models_core.CoreUser = Depends(is_user_in(GroupType.admin)), + notification_tool: NotificationTool = Depends(get_notification_tool), + scheduler: Scheduler = Depends(get_scheduler), +): + """ + Send ourself a test notification. + + **Only admins can use this endpoint** + """ + message = schemas_notification.Message( + title="Test notification future", + content="Ceci est un test de notification future", + action_module="test", + ) + await notification_tool.send_notification_to_users( + user_ids=[user.id], + message=message, + defer_date=datetime.now(UTC) + timedelta(seconds=10), + scheduler=scheduler, + job_id="testtt", + ) + + @router.post( "/notification/send/topic", status_code=201, From 75fc340dc527d91a14add44afadb948800a940c6 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:23:37 +0100 Subject: [PATCH 108/113] Fix action module --- app/core/notification/schemas_notification.py | 5 +---- app/modules/loan/endpoints_loan.py | 8 ++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/app/core/notification/schemas_notification.py b/app/core/notification/schemas_notification.py index a7ff0f7b2..871464355 100644 --- a/app/core/notification/schemas_notification.py +++ b/app/core/notification/schemas_notification.py @@ -5,10 +5,7 @@ class Message(BaseModel): title: str | None = None content: str | None = None - action_module: str | None = Field( - None, - description="An identifier for the module that should be triggered when the notification is clicked", - ) + action_module: str class FirebaseDevice(BaseModel): diff --git a/app/modules/loan/endpoints_loan.py b/app/modules/loan/endpoints_loan.py index 4369a68b0..9d327590e 100644 --- a/app/modules/loan/endpoints_loan.py +++ b/app/modules/loan/endpoints_loan.py @@ -618,7 +618,7 @@ async def create_loan( message = Message( title="📦 Nouveau prêt", content=f"Un prêt a été enregistré pour l'association {loan.loaner.name}", - module="loan", + action_module="loan", ) await notification_tool.send_notification_to_user( user_id=loan.borrower_id, @@ -631,7 +631,7 @@ async def create_loan( message = Message( title="📦 Prêt arrivé à échéance", content=f"N'oublie pas de rendre ton prêt à l'association {loan.loaner.name} !", - module="loan", + action_module="loan", ) await notification_tool.send_notification_to_users( @@ -939,7 +939,7 @@ async def extend_loan( message = Message( title="📦 Prêt prolongé", content=f"Ton prêt à l'association {loan.loaner.name} à bien été renouvellé !", - module="loan", + action_module="loan", ) await notification_tool.send_notification_to_user( user_id=loan.borrower_id, @@ -952,7 +952,7 @@ async def extend_loan( message = Message( title="📦 Prêt arrivé à échéance", content=f"N'oublie pas de rendre ton prêt à l'association {loan.loaner.name} !", - module="loan", + action_module="loan", ) await notification_tool.send_notification_to_users( From f99230373304c1e15c43c0a676220ef1fe89a868 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:23:47 +0100 Subject: [PATCH 109/113] Fix action module --- app/utils/communication/notifications.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/utils/communication/notifications.py b/app/utils/communication/notifications.py index d6ec611a3..07dfcf37e 100644 --- a/app/utils/communication/notifications.py +++ b/app/utils/communication/notifications.py @@ -106,7 +106,7 @@ async def _send_firebase_push_notification_by_tokens( try: message = messaging.MulticastMessage( tokens=tokens, - data={"module": message_content.action_module}, + data={"action_module": message_content.action_module}, notification=messaging.Notification( title=message_content.title, body=message_content.content, @@ -333,7 +333,6 @@ async def send_notification_to_users( user_ids: list[str], message: Message, scheduler: Scheduler | None = None, - defer_seconds: int | None = None, defer_date: datetime | None = None, job_id: str | None = None, ): From d9611c2514ed3a218b97db26b882b6cd3d03d564 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:23:56 +0100 Subject: [PATCH 110/113] Fix import --- app/types/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index 7f831c907..d41398e8f 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -9,6 +9,7 @@ from arq.jobs import Job from arq.typing import WorkerSettingsBase from arq.worker import create_worker +from sqlalchemy.ext.asyncio import AsyncSession from app import dependencies from app.types.exceptions import SchedulerNotStartedError @@ -16,7 +17,6 @@ if TYPE_CHECKING: from arq import Worker - from sqlalchemy.ext.asyncio import AsyncSession scheduler_logger = logging.getLogger("scheduler") From 556f244f584332ab55c497f24dff2a06ca036e9e Mon Sep 17 00:00:00 2001 From: armanddidierjean <95971503+armanddidierjean@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:33:02 +0100 Subject: [PATCH 111/113] Check if annotation is AsyncSession instead of default value --- app/types/scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/types/scheduler.py b/app/types/scheduler.py index d41398e8f..4b8e7bcc8 100644 --- a/app/types/scheduler.py +++ b/app/types/scheduler.py @@ -43,7 +43,8 @@ async def run_task( require_db_for_kwargs: list[str] = [] sign = signature(job_function) for param in sign.parameters.values(): - if isinstance(param.default, AsyncSession): + # See https://docs.python.org/3/library/inspect.html#inspect.Parameter.annotation + if param.annotation is AsyncSession: # We iterate over the parameters of the job_function # If we find a AsyncSession object, we want to inject the dependency require_db_for_kwargs.append(param.name) From cabe34239fc323f27d663ebf21e2439f56e686f4 Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:04:09 +0100 Subject: [PATCH 112/113] Fix missing action_module --- app/modules/booking/endpoints_booking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/modules/booking/endpoints_booking.py b/app/modules/booking/endpoints_booking.py index 9e450d1df..f5d06bd19 100644 --- a/app/modules/booking/endpoints_booking.py +++ b/app/modules/booking/endpoints_booking.py @@ -279,7 +279,7 @@ async def create_booking( message = Message( title="📅 Réservations - Nouvelle réservation", content=content, - module="booking", + action_module="booking", ) await notification_tool.send_notification_to_users( From 6b4065773cd1a5c2d1afc311afe8d9680259bc8e Mon Sep 17 00:00:00 2001 From: Foucauld Bellanger <63885990+Foukki@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:39:12 +0100 Subject: [PATCH 113/113] Fixing test --- tests/commons.py | 8 ++++++++ tests/conftest.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/tests/commons.py b/tests/commons.py index 88ba81d40..fb5dee6db 100644 --- a/tests/commons.py +++ b/tests/commons.py @@ -21,6 +21,7 @@ from app.dependencies import get_settings from app.types.exceptions import RedisConnectionError from app.types.floors_type import FloorsType +from app.types.scheduler import OfflineScheduler, Scheduler from app.types.sqlalchemy import Base from app.utils.redis import connect, disconnect from app.utils.tools import ( @@ -108,6 +109,13 @@ def change_redis_client_status(activated: bool) -> None: redis_client = False +def override_get_scheduler( + settings: Settings = Depends(get_settings), +) -> Scheduler: # As we don't want the limiter to be activated, except during the designed test, we add an "activate"/"deactivate" option + """Override the get_redis_client function to use the testing session""" + return OfflineScheduler() + + async def create_user_with_groups( groups: list[GroupType], account_type: AccountType = AccountType.student, diff --git a/tests/conftest.py b/tests/conftest.py index d8d346cb5..dc9b40f8c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ get_db, get_payment_tool, get_redis_client, + get_scheduler, get_settings, get_unsafe_db, ) @@ -13,6 +14,7 @@ override_get_db, override_get_payment_tool, override_get_redis_client, + override_get_scheduler, override_get_settings, override_get_unsafe_db, settings, @@ -28,5 +30,6 @@ def client() -> TestClient: test_app.dependency_overrides[get_settings] = override_get_settings test_app.dependency_overrides[get_redis_client] = override_get_redis_client test_app.dependency_overrides[get_payment_tool] = override_get_payment_tool + test_app.dependency_overrides[get_scheduler] = override_get_scheduler return TestClient(test_app) # Create a client to execute tests