Add configurable limit to CSV file size

omero_parade/csv_filters/data_providers.py

@@ -20,19 +20,24 @@
 from io import StringIO
 
 from omero.model import FileAnnotationI
+from omero_parade import parade_settings
 
 logger = logging.getLogger(__name__)
+MAX_CSV_SIZE = parade_settings.MAX_CSV_SIZE
 
 
-def get_csv_files(conn, obj_type, obj_id):
+def get_csv_annotations(conn, obj_type, obj_id):
 
     csv_files = []
-
     obj = conn.getObject(obj_type, obj_id)
+    if obj is None:
+        return []
+
     for ann in obj.listAnnotations():
-        if (ann.OMERO_TYPE == FileAnnotationI):
-            if ann.file.name.val.endswith('.csv'):
-                csv_files.append(ann)
+        if (ann.OMERO_TYPE == FileAnnotationI and
+                ann.file.size.val < MAX_CSV_SIZE and
+                ann.file.name.val.endswith('.csv')):
+            csv_files.append(ann)
 
     return csv_files
 
@@ -47,12 +52,11 @@ def get_csv_column_names(file_ann):
 
 
 def get_names(conn, obj_type, obj_id):
-    csv_files = get_csv_files(conn, obj_type, obj_id)
+    csv_files = get_csv_annotations(conn, obj_type, obj_id)
 
     names = []
     for f in csv_files:
         fname = f.file.name.val
-        print(fname)
         for col in get_csv_column_names(f):
             names.append("%s %s" % (fname, col))
     return names
@@ -77,7 +81,7 @@ def get_data(request, data_name, conn):
     for dtype in obj_types:
         if request.GET.get(dtype) is not None:
             obj_id = request.GET.get(dtype)
-            for file_ann in get_csv_files(conn, dtype, obj_id):
+            for file_ann in get_csv_annotations(conn, dtype, obj_id):
                 if data_name.startswith(file_ann.getFile().name):
                     csv_file = file_ann.getFile()
                     break

omero_parade/csv_filters/omero_filters.py

@@ -26,11 +26,17 @@
 from omero.model import FileAnnotationI
 from omero_parade.views import NUMPY_GT_1_11_0
 from omero_parade.utils import get_well_image_ids
+from .data_providers import get_csv_annotations
 
 logger = logging.getLogger(__name__)
 
 
 def name_to_word(name):
+    """
+    Replace any non alpha-numeric character or . with underscore.
+
+    This allows us to use the filter name in URL to load filter script
+    """
     w = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890.'
     return "".join([l if l in w else '_' for l in name])
 
@@ -39,23 +45,17 @@ def get_filters(request, conn):
 
     obj_types = ["project", "dataset", "screen", "plate"]
 
-    obj = None
-    for ob_type in obj_types:
-        if request.GET.get(ob_type) is not None:
-            obj_id = request.GET.get(ob_type)
-            obj = conn.getObject(ob_type, obj_id)
+    obj_type = None
+    obj_id = None
+    for o in obj_types:
+        if request.GET.get(o) is not None:
+            obj_id = request.GET.get(o)
+            obj_type = o
             break
 
-    if obj is None:
-        return []
-
-    csv_names = []
-    for ann in obj.listAnnotations():
-        if (ann.OMERO_TYPE == FileAnnotationI):
-            if ann.file.name.val.endswith('csv'):
-                csv_names.append(ann.file.name.val)
-
-    csv_names = [name_to_word(name) for name in csv_names]
+    csv_anns = get_csv_annotations(conn, obj_type, obj_id)
+    # Use name_to_word so that filter can be used in URL
+    csv_names = [name_to_word(ann.file.name.val) for ann in csv_anns]
     return csv_names
 
 
@@ -86,8 +86,6 @@ def get_script(request, script_name, conn):
         return JsonResponse(
             {'Error': 'Specify object e.g. ?project=1'})
 
-    print('name', script_name)
-
     # find csv file by name
     file_ann = None
     for ann in obj.listAnnotations():

omero_parade/parade_settings.py

@@ -24,6 +24,10 @@
 # load settings
 PARADE_SETTINGS_MAPPING = {
 
+    "omero.web.parade.max_csv_size": ["MAX_CSV_SIZE", 1000000, int,
+    ("Maximum supported size of CSV files in bytes, to avoid trying \
+     to load very large files for filtering or plotting")],
+
     "omero.web.parade.filters":
         ["PARADE_FILTERS",
          '["omero_parade", "omero_parade.annotation_filters", '