feat: backup db content locally and copy sql file to s3 bucket

README.md

@@ -112,11 +112,14 @@ This file will have to hold the following information:
 - `S3_REGION`: your S3 bucket is geographically identified by its location's region
 - `S3_ENDPOINT_URL`: the URL providing a S3 endpoint by your cloud provider
 - `BUCKET_NAME`: the name of the storage bucket
+- `DATABASE_URL`: database URL, including username and password 
 
 Optionally, the following information can be added:
 - `SENTRY_DSN`: the URL of the [Sentry](https://sentry.io/) project, which monitors back-end errors and report them back.
 - `SENTRY_SERVER_NAME`: the server tag to apply to events.
 - `BUCKET_MEDIA_FOLDER`: the optional subfolder to put the media files in
+- `BUCKET_DB_BACKUP_FOLDER`: subfolder where to store db backups
+- `POSTGRES_USER`: db username ; required for backups
 
 So your `.env` file should look like something similar to:
 ```
@@ -125,6 +128,8 @@ S3_SECRET_KEY=YOUR_SECRET_KEY
 S3_REGION=bucket-region
 S3_ENDPOINT_URL='https://s3.mydomain.com/'
 BUCKET_NAME=my_storage_bucket_name
+POSTGRES_USER=my_postgres_user
+DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/${POSTGRES_DB}
 SENTRY_DSN='https://replace.with.you.sentry.dsn/'
 SENTRY_SERVER_NAME=my_storage_bucket_name
 ```
@@ -141,6 +146,9 @@ The full package documentation is available [here](http://pyronear-api.herokuapp
 
 This project is a REST-API, and you can interact with the service through HTTP requests. However, if you want to ease the integration into a Python project, take a look at our [Python client](client).
 
+### DB backup
+
+From the project root folder, you can run `sh scripts/dump_db.sh` to dump the db content to `db_backup` and upload it to s3
 
 ## Contributing
 

docker-compose.yml

@@ -7,6 +7,7 @@ services:
     command: uvicorn app.main:app --reload --workers 1 --host 0.0.0.0 --port 8080
     volumes:
       - ./src/:/app/
+      - ./db_backup:/app/db_backup/
     ports:
       - 8080:8080
     environment:

scripts/dump_db.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# Copyright (C) 2020-2023, Pyronear.
+
+# This program is licensed under the Apache License 2.0.
+# See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
+
+# Dump db content to sql file and copy it to s3: should be run from package root directory
+
+# Set environment variables from .env
+export $(grep -v '^#' .env | xargs)
+
+mkdir -p db_backup
+fname=db_backup/dump_$(date +%Y-%m-%d_%H_%M_%S).sql
+echo "Dumping db content to ${fname}"
+docker compose exec -T db pg_dumpall -c -U "${POSTGRES_USER}" > "${fname}"
+
+if [ ! -s "$fname" ]; then
+  echo "Dump failed"
+  exit 1
+fi
+
+if [ -z "$BUCKET_DB_BACKUP_FOLDER" ]; then
+  echo "BUCKET_DB_BACKUP_FOLDER not defined"
+  exit 1
+fi
+
+dest=$BUCKET_DB_BACKUP_FOLDER/$(basename "${fname}")
+echo "Copying ${fname} to s3: ${BUCKET_NAME}/${dest}"
+docker compose exec -T backend python -c "import asyncio; from app.services import bucket_service; print(asyncio.run(bucket_service.upload_file(\"${dest}\", \"${fname}\")))"

src/app/api/endpoints/media.py

@@ -149,7 +149,7 @@ async def upload_media_from_device(
         return await crud.get_entry(media, media_id)
     else:
         # Failed upload
-        if not (await bucket_service.upload_file(bucket_key, file.file)):  # type: ignore[arg-type]
+        if not (await bucket_service.upload_file(bucket_key, file.file)):
             raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed upload")
         # Data integrity check
         file_meta = await bucket_service.get_file_metadata(bucket_key)

src/app/config.py

@@ -9,7 +9,6 @@
 
 from dotenv import load_dotenv
 
-# load_dotenv(".env_tests" if "pytest" in sys.modules else ".env")
 load_dotenv("../.env")
 
 PROJECT_NAME: str = "Pyronear - Alert API"

src/app/services/bucket/s3.py

@@ -4,7 +4,7 @@
 # See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
 
 import logging
-from typing import Any, Dict
+from typing import Any, BinaryIO, Dict, Union
 
 import boto3
 from fastapi import HTTPException
@@ -55,11 +55,15 @@ async def get_public_url(self, bucket_key: str, url_expiration: int = 3600) -> s
         # Generate a public URL for it using boto3 presign URL generation
         return self._s3.generate_presigned_url("get_object", Params=file_params, ExpiresIn=url_expiration)
 
-    async def upload_file(self, bucket_key: str, file_binary: bytes) -> bool:
+    async def upload_file(self, bucket_key: str, file_or_binary: Union[str, BinaryIO]) -> bool:
         """Upload a file to bucket and return whether the upload succeeded"""
         # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Bucket.upload_fileobj
+        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/upload_file.html
         try:
-            self._s3.upload_fileobj(file_binary, self.bucket_name, bucket_key)
+            if isinstance(file_or_binary, str):
+                self._s3.upload_file(file_or_binary, self.bucket_name, bucket_key)
+            else:
+                self._s3.upload_fileobj(file_or_binary, self.bucket_name, bucket_key)
         except Exception as e:
             logger.warning(e)
             return False

src/tests/test_services.py

@@ -1,3 +1,5 @@
+import pytest
+
 from app.services import bucket_service, resolve_bucket_key
 from app.services.bucket import S3Bucket
 from app.services.utils import cfg
@@ -23,3 +25,14 @@ def test_resolve_bucket_key(monkeypatch):
 
 def test_bucket_service():
     assert isinstance(bucket_service, S3Bucket)
+
+
+@pytest.mark.parametrize("filename, is_binary", [("test.txt", False), ("test.binary", True)])
+@pytest.mark.asyncio
+async def test_upload_file(filename, is_binary, tmp_path):
+    fname = str(tmp_path / filename)
+    with open(fname, "wb" if is_binary else "w") as f:
+        f.write(bytearray(b"This is a test") if is_binary else "This is a test")
+    assert await bucket_service.upload_file(f"test/{filename}", open(fname, "rb") if is_binary else fname)
+    assert await bucket_service.get_file_metadata(f"test/{filename}")
+    await bucket_service.delete_file(f"test/{filename}")