Merge pull request #1281 from TiagoRascazzi/develop

PDF export improvement

.directory

@@ -0,0 +1,3 @@
+[Dolphin]
+Timestamp=2022,1,7,19,23,46.14
+Version=4

.env.template

@@ -150,3 +150,7 @@ REVERSE_PROXY_AUTH=0
 # Enables exporting PDF (see export docs)
 # Disabled by default, uncomment to enable
 # ENABLE_PDF_EXPORT=1
+
+# Duration to keep the cached export file (in seconds)
+EXPORT_FILE_CACHE_DURATION=300
+

cookbook/integration/default.py

@@ -32,11 +32,12 @@ def decode_recipe(self, string):
         return None
 
     def get_file_from_recipe(self, recipe):
+
         export = RecipeExportSerializer(recipe).data
 
         return 'recipe.json', JSONRenderer().render(export).decode("utf-8")
 
-    def get_files_from_recipes(self, recipes, cookie):
+    def get_files_from_recipes(self, recipes, el, cookie):
         export_zip_stream = BytesIO()
         export_zip_obj = ZipFile(export_zip_stream, 'w')
 
@@ -50,13 +51,20 @@ def get_files_from_recipes(self, recipes, cookie):
                 recipe_stream.write(data)
                 recipe_zip_obj.writestr(filename, recipe_stream.getvalue())
                 recipe_stream.close()
+
                 try:
                     recipe_zip_obj.writestr(f'image{get_filetype(r.image.file.name)}', r.image.file.read())
                 except ValueError:
                     pass
 
                 recipe_zip_obj.close()
+
                 export_zip_obj.writestr(str(r.pk) + '.zip', recipe_zip_stream.getvalue())
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(r)
+            el.save()
+
         export_zip_obj.close()
 
-        return [[ 'export.zip', export_zip_stream.getvalue() ]]
+        return [[ self.get_export_file_name(), export_zip_stream.getvalue() ]]

cookbook/integration/integration.py

@@ -1,9 +1,12 @@
+import time
 import datetime
 import json
 import traceback
 import uuid
 from io import BytesIO, StringIO
 from zipfile import BadZipFile, ZipFile
+from django.core.cache import cache
+import datetime
 
 from bs4 import Tag
 from django.core.exceptions import ObjectDoesNotExist
@@ -18,6 +21,7 @@
 from cookbook.helper.image_processing import get_filetype, handle_image
 from cookbook.models import Keyword, Recipe
 from recipes.settings import DEBUG
+from recipes.settings import EXPORT_FILE_CACHE_DURATION
 
 
 class Integration:
@@ -61,35 +65,44 @@ def __init__(self, request, export_type):
                 space=request.space
             )
 
-    def do_export(self, recipes):
-        """
-        Perform the export based on a list of recipes
-        :param recipes: list of recipe objects
-        :return: HttpResponse with the file of the requested export format that is directly downloaded (When that format involve multiple files they are zipped together) 
-        """
 
-        files = self.get_files_from_recipes(recipes, self.request.COOKIES)
 
-        if len(files) == 1:
-            filename, file = files[0]
-            export_filename = filename
-            export_file = file
+    def do_export(self, recipes, el):
+
+        with scope(space=self.request.space):
+            el.total_recipes = len(recipes) 
+            el.cache_duration = EXPORT_FILE_CACHE_DURATION
+            el.save()
+
+            files = self.get_files_from_recipes(recipes, el, self.request.COOKIES)
+
+            if len(files) == 1:
+                filename, file = files[0]
+                export_filename = filename
+                export_file = file
 
-        else:
-            export_filename = "export.zip"
-            export_stream = BytesIO()
-            export_obj = ZipFile(export_stream, 'w')
+            else:
+                #zip the files if there is more then one file
+                export_filename = self.get_export_file_name()
+                export_stream = BytesIO()
+                export_obj = ZipFile(export_stream, 'w')
+
+                for filename, file in files:
+                    export_obj.writestr(filename, file)
 
-            for filename, file in files:
-                export_obj.writestr(filename, file)
+                export_obj.close()
+                export_file = export_stream.getvalue()
 
-            export_obj.close()
-            export_file = export_stream.getvalue()
+
+            cache.set('export_file_'+str(el.pk), {'filename': export_filename, 'file': export_file}, EXPORT_FILE_CACHE_DURATION)
+            el.running = False
+            el.save()
 
         response = HttpResponse(export_file, content_type='application/force-download')
         response['Content-Disposition'] = 'attachment; filename="' + export_filename + '"'
         return response
 
+
     def import_file_name_filter(self, zip_info_object):
         """
         Since zipfile.namelist() returns all files in all subdirectories this function allows filtering of files
@@ -126,7 +139,7 @@ def do_import(self, files, il, import_duplicates):
                             for d in data_list:
                                 recipe = self.get_recipe_from_file(d)
                                 recipe.keywords.add(self.keyword)
-                                il.msg += f'{recipe.pk} - {recipe.name} \n'
+                                il.msg += self.get_recipe_processed_msg(recipe)
                                 self.handle_duplicates(recipe, import_duplicates)
                                 il.imported_recipes += 1
                                 il.save()
@@ -151,7 +164,7 @@ def do_import(self, files, il, import_duplicates):
                                 else:
                                     recipe = self.get_recipe_from_file(BytesIO(import_zip.read(z.filename)))
                                 recipe.keywords.add(self.keyword)
-                                il.msg += f'{recipe.pk} - {recipe.name} \n'
+                                il.msg += self.get_recipe_processed_msg(recipe)
                                 self.handle_duplicates(recipe, import_duplicates)
                                 il.imported_recipes += 1
                                 il.save()
@@ -166,7 +179,7 @@ def do_import(self, files, il, import_duplicates):
                             try:
                                 recipe = self.get_recipe_from_file(d)
                                 recipe.keywords.add(self.keyword)
-                                il.msg += f'{recipe.pk} - {recipe.name} \n'
+                                il.msg += self.get_recipe_processed_msg(recipe)
                                 self.handle_duplicates(recipe, import_duplicates)
                                 il.imported_recipes += 1
                                 il.save()
@@ -183,7 +196,7 @@ def do_import(self, files, il, import_duplicates):
                                     try:
                                         recipe = self.get_recipe_from_file(d)
                                         recipe.keywords.add(self.keyword)
-                                        il.msg += f'{recipe.pk} - {recipe.name} \n'
+                                        il.msg += self.get_recipe_processed_msg(recipe)
                                         self.handle_duplicates(recipe, import_duplicates)
                                         il.imported_recipes += 1
                                         il.save()
@@ -193,7 +206,7 @@ def do_import(self, files, il, import_duplicates):
                     else:
                         recipe = self.get_recipe_from_file(f['file'])
                         recipe.keywords.add(self.keyword)
-                        il.msg += f'{recipe.pk} - {recipe.name} \n'
+                        il.msg += self.get_recipe_processed_msg(recipe)
                         self.handle_duplicates(recipe, import_duplicates)
             except BadZipFile:
                 il.msg += 'ERROR ' + _(
@@ -260,7 +273,7 @@ def get_file_from_recipe(self, recipe):
         """
         raise NotImplementedError('Method not implemented in integration')
 
-    def get_files_from_recipes(self, recipes, cookie):
+    def get_files_from_recipes(self, recipes, el, cookie):
         """
         Takes a list of recipe object and converts it to a array containing each file.
         Each file is represented as an array [filename, data] where data is a string of the content of the file.
@@ -279,3 +292,10 @@ def handle_exception(exception, log=None, message=''):
                 log.msg += exception.msg
         if DEBUG:
             traceback.print_exc()
+
+
+    def get_export_file_name(self, format='zip'):
+        return "export_{}.{}".format(datetime.datetime.now().strftime("%Y-%m-%d"), format)
+
+    def get_recipe_processed_msg(self, recipe):
+        return f'{recipe.pk} - {recipe.name} \n'

cookbook/integration/pdfexport.py

@@ -11,22 +11,25 @@
 from cookbook.integration.integration import Integration
 from cookbook.serializer import RecipeExportSerializer
 
-import django.core.management.commands.runserver as runserver
+from cookbook.models import ExportLog
+from asgiref.sync import sync_to_async
 
+import django.core.management.commands.runserver as runserver
+import logging
 
 class PDFexport(Integration):
 
     def get_recipe_from_file(self, file):
         raise NotImplementedError('Method not implemented in storage integration')
 
-    async def get_files_from_recipes_async(self, recipes, cookie):
+    async def get_files_from_recipes_async(self, recipes, el, cookie):
         cmd = runserver.Command()
 
         browser = await launch(
             handleSIGINT=False,
             handleSIGTERM=False,
             handleSIGHUP=False,
-            ignoreHTTPSErrors=True
+            ignoreHTTPSErrors=True,
         )
 
         cookies = {'domain': cmd.default_addr, 'name': 'sessionid', 'value': cookie['sessionid'], }
@@ -39,17 +42,28 @@ async def get_files_from_recipes_async(self, recipes, cookie):
                    }
                    }
 
-        page = await browser.newPage()
-        await page.emulateMedia('print')
-        await page.setCookie(cookies)
 
         files = []
         for recipe in recipes:
-            await page.goto('http://' + cmd.default_addr + ':' + cmd.default_port + '/view/recipe/' + str(recipe.id), {'waitUntil': 'networkidle0', })
+
+            page = await browser.newPage()
+            await page.emulateMedia('print')
+            await page.setCookie(cookies)
+
+            await page.goto('http://'+cmd.default_addr+':'+cmd.default_port+'/view/recipe/'+str(recipe.id), {'waitUntil': 'domcontentloaded'})
+            await page.waitForSelector('#printReady');
+
             files.append([recipe.name + '.pdf', await page.pdf(options)])
+            await page.close();
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(recipe)
+            await sync_to_async(el.save, thread_sensitive=True)()
+
 
         await browser.close()
         return files
 
-    def get_files_from_recipes(self, recipes, cookie):
-        return asyncio.run(self.get_files_from_recipes_async(recipes, cookie))
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        return asyncio.run(self.get_files_from_recipes_async(recipes, el, cookie))

cookbook/integration/recipesage.py

@@ -88,12 +88,16 @@ def get_file_from_recipe(self, recipe):
 
         return data
 
-    def get_files_from_recipes(self, recipes, cookie):
+    def get_files_from_recipes(self, recipes, el, cookie):
         json_list = []
         for r in recipes:
             json_list.append(self.get_file_from_recipe(r))
 
-        return [['export.json', json.dumps(json_list)]]
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(r)
+            el.save()
+
+        return [[self.get_export_file_name('json'), json.dumps(json_list)]]
 
     def split_recipe_file(self, file):
         return json.loads(file.read().decode("utf-8"))

cookbook/integration/saffron.py

@@ -87,10 +87,14 @@ def get_file_from_recipe(self, recipe):
 
         return recipe.name+'.txt', data
 
-    def get_files_from_recipes(self, recipes, cookie):
+    def get_files_from_recipes(self, recipes, el, cookie):
         files = []
         for r in recipes:
             filename, data = self.get_file_from_recipe(r)
             files.append([ filename, data ])
 
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(r)
+            el.save()
+
         return files

cookbook/migrations/0164_exportlog.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.2.11 on 2022-01-07 20:29
+
+import cookbook.models
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('cookbook', '0163_auto_20220105_0758'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ExportLog',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('type', models.CharField(max_length=32)),
+                ('running', models.BooleanField(default=True)),
+                ('msg', models.TextField(default='')),
+                ('total_recipes', models.IntegerField(default=0)),
+                ('exported_recipes', models.IntegerField(default=0)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('space', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cookbook.space')),
+            ],
+            bases=(models.Model, cookbook.models.PermissionModelMixin),
+        ),
+    ]

cookbook/migrations/0165_exportlog_cache_duration.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.11 on 2022-01-08 00:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookbook', '0164_exportlog'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='exportlog',
+            name='cache_duration',
+            field=models.IntegerField(default=0),
+        ),
+    ]

cookbook/migrations/0166_exportlog_possibly_not_expired.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.11 on 2022-01-08 00:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookbook', '0165_exportlog_cache_duration'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='exportlog',
+            name='possibly_not_expired',
+            field=models.BooleanField(default=True),
+        ),
+    ]

cookbook/models.py

@@ -995,6 +995,25 @@ class ImportLog(models.Model, PermissionModelMixin):
     def __str__(self):
         return f"{self.created_at}:{self.type}"
 
+class ExportLog(models.Model, PermissionModelMixin):
+    type = models.CharField(max_length=32)
+    running = models.BooleanField(default=True)
+    msg = models.TextField(default="")
+
+    total_recipes = models.IntegerField(default=0)
+    exported_recipes = models.IntegerField(default=0)
+    cache_duration = models.IntegerField(default=0)
+    possibly_not_expired = models.BooleanField(default=True)
+
+    created_at = models.DateTimeField(auto_now_add=True)
+    created_by = models.ForeignKey(User, on_delete=models.CASCADE)
+
+    objects = ScopedManager(space='space')
+    space = models.ForeignKey(Space, on_delete=models.CASCADE)
+
+    def __str__(self):
+        return f"{self.created_at}:{self.type}"
+
 
 class BookmarkletImport(ExportModelOperationsMixin('bookmarklet_import'), models.Model, PermissionModelMixin):
     html = models.TextField()