From b071de88cef2aeba64be57339585bdd1b3a0d290 Mon Sep 17 00:00:00 2001 From: dedenbangkit Date: Thu, 7 Sep 2023 15:44:04 +0700 Subject: [PATCH 1/4] [#16] Init storage service util --- backend/akvo/utils/storage.py | 30 ++++++++++++++++++++++ backend/akvo/utils/tests/__init__.py | 0 backend/akvo/utils/tests/test_storage.py | 32 ++++++++++++++++++++++++ docker-compose.override.yml | 4 +++ docker-compose.yml | 1 + 5 files changed, 67 insertions(+) create mode 100644 backend/akvo/utils/storage.py create mode 100644 backend/akvo/utils/tests/__init__.py create mode 100644 backend/akvo/utils/tests/test_storage.py diff --git a/backend/akvo/utils/storage.py b/backend/akvo/utils/storage.py new file mode 100644 index 0000000..b0504e0 --- /dev/null +++ b/backend/akvo/utils/storage.py @@ -0,0 +1,30 @@ +import os +from pathlib import Path +import shutil + + +class Storage: + def __init__(self, storage_path: str): + self.storage_path = storage_path + + def upload(self, file: str, folder: str = None, filename: str = None): + storage_location = self.storage_path + if folder: + Path(f"{storage_location}/{folder}").mkdir(parents=True, exist_ok=True) + storage_location = f"{storage_location}/{folder}" + if not filename: + filename = file.split("/")[-1] + location = f"{storage_location}/{filename}" + shutil.copy2(file, location) + return location + + def delete(self, url: str): + os.remove(f"{self.storage_path}/{url}") + return url + + def check(self, url: str): + path = Path(f"{self.storage_location}/{url}") + return path.is_file() + + def download(self, url: str): + return f"{self.storage_path}/{url}" diff --git a/backend/akvo/utils/tests/__init__.py b/backend/akvo/utils/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/akvo/utils/tests/test_storage.py b/backend/akvo/utils/tests/test_storage.py new file mode 100644 index 0000000..547476a --- /dev/null +++ b/backend/akvo/utils/tests/test_storage.py @@ -0,0 +1,32 @@ +import os +from django.test import TestCase +from storage import Storage + + +class StorageTestCase(TestCase): + def setUp(self): + # Create a temporary directory if it doesn't exist + if not os.path.exists("/tmp"): + os.mkdir("/tmp") + # create a file + self.file = "./test.txt" + with open(self.file, "w") as f: + f.write("test") + self.storage = Storage(storage_path="/tmp") + + def test_upload(self): + self.storage.upload(file=self.file) + self.assertTrue(os.path.exists(self.file)) + + def test_delete(self): + self.storage.upload(file=self.file) + self.storage.delete(url="test.txt") + self.assertFalse(os.path.exists(self.file)) + + def test_check(self): + self.storage.upload(file=self.file) + self.assertTrue(self.storage.check(url="test.txt")) + + def test_download(self): + self.storage.upload(file=self.file) + self.assertEqual(self.storage.download(url="test.txt"), self.file) diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 2476522..7e28da7 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -12,10 +12,14 @@ services: - 8888:8888 frontend: network_mode: service:mainnetwork + volumes: + - ${STORAGE_PATH}/images:/app/public/images:delegated backend: environment: - WEBDOMAIN=https://afs.akvotest.org network_mode: service:mainnetwork + volumes: + - ${STORAGE_PATH}:/app/storage:delegated pgadmin: image: dpage/pgadmin4:5.7 environment: diff --git a/docker-compose.yml b/docker-compose.yml index d397546..fb1882e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ services: - DB_HOST=db - DEBUG=True - DJANGO_SECRET=local-secret + - STORAGE_PATH depends_on: - db frontend: From 49b39b96dcf8535466eae31209e5a191838875a2 Mon Sep 17 00:00:00 2001 From: dedenbangkit Date: Thu, 7 Sep 2023 17:52:30 +0700 Subject: [PATCH 2/4] [#16] Add utils to afs settings --- backend/afs/settings.py | 3 ++- backend/akvo/utils/storage.py | 2 +- backend/akvo/utils/tests/test_storage.py | 30 +++++++++++++++++------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/backend/afs/settings.py b/backend/afs/settings.py index 6c02da8..f0b396b 100644 --- a/backend/afs/settings.py +++ b/backend/afs/settings.py @@ -26,7 +26,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = ['*'] +ALLOWED_HOSTS = ["*"] # Application definition @@ -44,6 +44,7 @@ "akvo.core_forms", "akvo.core_node", "akvo.core_data", + "akvo.utils", ] MIDDLEWARE = [ diff --git a/backend/akvo/utils/storage.py b/backend/akvo/utils/storage.py index b0504e0..4e27a3a 100644 --- a/backend/akvo/utils/storage.py +++ b/backend/akvo/utils/storage.py @@ -23,7 +23,7 @@ def delete(self, url: str): return url def check(self, url: str): - path = Path(f"{self.storage_location}/{url}") + path = Path(f"{self.storage_path}/{url}") return path.is_file() def download(self, url: str): diff --git a/backend/akvo/utils/tests/test_storage.py b/backend/akvo/utils/tests/test_storage.py index 547476a..aad3e6d 100644 --- a/backend/akvo/utils/tests/test_storage.py +++ b/backend/akvo/utils/tests/test_storage.py @@ -5,28 +5,42 @@ class StorageTestCase(TestCase): def setUp(self): - # Create a temporary directory if it doesn't exist - if not os.path.exists("/tmp"): - os.mkdir("/tmp") # create a file + self.path = "/tmp" self.file = "./test.txt" with open(self.file, "w") as f: f.write("test") - self.storage = Storage(storage_path="/tmp") + self.storage = Storage(storage_path=self.path) + + def tearDown(self): + # delete the file + os.remove(self.file) def test_upload(self): self.storage.upload(file=self.file) - self.assertTrue(os.path.exists(self.file)) + self.assertTrue(os.path.exists(f"{self.path}/{self.file}")) + + # test upload with folder + self.storage.upload(file=self.file, folder="testing") + self.assertTrue(os.path.exists(f"{self.path}/testing/{self.file}")) + # test upload with filename + self.storage.upload(file=self.file, filename="test2.txt") + self.assertTrue(os.path.exists(f"{self.path}/test2.txt")) def test_delete(self): self.storage.upload(file=self.file) self.storage.delete(url="test.txt") - self.assertFalse(os.path.exists(self.file)) + self.assertFalse(os.path.exists(f"{self.path}/{self.file}")) def test_check(self): self.storage.upload(file=self.file) self.assertTrue(self.storage.check(url="test.txt")) def test_download(self): - self.storage.upload(file=self.file) - self.assertEqual(self.storage.download(url="test.txt"), self.file) + self.storage.upload(file=self.file, folder="testing") + self.assertTrue(os.path.exists(f"{self.path}/testing/{self.file}")) + filename = self.file.split("/")[-1] + self.assertEqual( + self.storage.download(url="testing/test.txt"), + f"{self.path}/testing/{filename}", + ) From 7d42d7604f469aaf9996f950c7ad1abf709c1ebb Mon Sep 17 00:00:00 2001 From: dedenbangkit Date: Mon, 11 Sep 2023 07:16:36 +0700 Subject: [PATCH 3/4] [#16] Directly open file instead return url --- backend/akvo/utils/storage.py | 5 ++++- backend/akvo/utils/tests/test_storage.py | 11 +++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/akvo/utils/storage.py b/backend/akvo/utils/storage.py index 4e27a3a..4b5db29 100644 --- a/backend/akvo/utils/storage.py +++ b/backend/akvo/utils/storage.py @@ -27,4 +27,7 @@ def check(self, url: str): return path.is_file() def download(self, url: str): - return f"{self.storage_path}/{url}" + if not self.check(url): + return None + with open(f"{self.storage_path}/{url}", "rb") as f: + return f.read() diff --git a/backend/akvo/utils/tests/test_storage.py b/backend/akvo/utils/tests/test_storage.py index aad3e6d..a94f1cc 100644 --- a/backend/akvo/utils/tests/test_storage.py +++ b/backend/akvo/utils/tests/test_storage.py @@ -40,7 +40,10 @@ def test_download(self): self.storage.upload(file=self.file, folder="testing") self.assertTrue(os.path.exists(f"{self.path}/testing/{self.file}")) filename = self.file.split("/")[-1] - self.assertEqual( - self.storage.download(url="testing/test.txt"), - f"{self.path}/testing/{filename}", - ) + file = self.storage.download(url="testing/test.txt") + with open(f"{self.path}/testing/{filename}", "rb") as f: + self.assertEqual(f.read(), file) + + # Failed Download + file = self.storage.download(url="testing/test2.txt") + self.assertFalse(file) From b465aff5d7a2fa8bec18ac2ce362c35b78e55e62 Mon Sep 17 00:00:00 2001 From: dedenbangkit Date: Mon, 11 Sep 2023 07:26:14 +0700 Subject: [PATCH 4/4] [#16] Add images folder to nginx --- frontend/nginx/conf.d/default.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/nginx/conf.d/default.conf b/frontend/nginx/conf.d/default.conf index 5908a3e..6e38f93 100644 --- a/frontend/nginx/conf.d/default.conf +++ b/frontend/nginx/conf.d/default.conf @@ -49,6 +49,12 @@ server { proxy_pass http://localhost:8000; } + location /images/ { + rewrite ^/images/(.*)$ /storage/images/$1 last; + expires 7d; + add_header Cache-Control "max-age=604800, public"; + } + location / { try_files $uri $uri/ /index.html; }