Update memory usage after optimization (#270)

* Update memory usage after optimization

* Update memory requirements

README.md

@@ -103,7 +103,7 @@ work_dir = /scratch/$USER/workflows
 x509_user_proxy = $HOME/.xenon_service_proxy
 user_install_package = strax, straxen, cutax, rucio, utilix, admix, outsource
 check_user_install_package = strax, straxen, cutax, rucio, utilix, admix, outsource
-# sites to exclude (GLIDEIN_Site), comma seprated list
+# sites to exclude (GLIDEIN_Site), comma separated list
 exclude_sites = SU-ITS, NotreDame, UConn-HPC, Purdue Geddes, Chameleon, WSU-GRID, SIUE-CC-production, Lancium
 # countries of EU only jobs
 eu_only_countries = NL, FR, IT
@@ -124,6 +124,7 @@ max_run_number = 999999
 max_num = 500
 chunks_per_job = 10
 rough_disk = 16000
+max_memory = 8000
 dagman_retry = 2
 dagman_static_retry = 0
 resources_increment = 0.2

outsource/config.py

@@ -84,7 +84,7 @@
     "CCIN2P32_USERDISK": "CCIN2P3",
 }
 
-MAX_MEMORY = 30_000  # in MB
+MAX_MEMORY = uconfig.getint("Outsource", "max_memory", fallback=12_000)  # in MB
 MIN_DISK = 200  # in MB
 
 logger = setup_logger("outsource", uconfig.get("Outsource", "logging_level", fallback="WARNING"))

outsource/meta.py

@@ -58,7 +58,7 @@
             "afterpulses": None,
         },
         "memory": {
-            "lower": [3.2, 1.2e3],
+            "lower": [1.0, 1.6e3],
             "combine": [0.0, 2.0e3],
             "upper": [0.007, 4.0e3],
         },
@@ -85,7 +85,7 @@
             "events_nv": 0.15,
         },
         "memory": {
-            "lower": [4.0, 0.0e3],
+            "lower": [0.1, 7.0e3],
             "combine": [0.0, 2.0e3],
             "upper": [0.0, 4.0e3],
         },
@@ -107,7 +107,7 @@
             "events_mv": 0.003,
         },
         "memory": {
-            "lower": [4.0, 1.0e3],
+            "lower": [1.1, 1.05e3],
             "combine": [0.0, 2.0e3],
             "upper": [0.0, 4.0e3],
         },

outsource/workflow/process.py

@@ -5,18 +5,23 @@
 import gc
 from utilix import uconfig
 from utilix.config import setup_logger
+import straxen
 
 from outsource.utils import get_context, get_processing_order, per_chunk_storage_root_data_type
 from outsource.upload import upload_to_rucio
 
 from rframe.interfaces.mongo import MongoAggregation
 from rframe.interfaces.mongo import MultiMongoAggregation
 
+# Allow disk use for mongo aggregation
 MongoAggregation.allow_disk_use = True
 MultiMongoAggregation.allow_disk_use = True
 
 logger = setup_logger("outsource", uconfig.get("Outsource", "logging_level", fallback="WARNING"))
 
+if not straxen.HAVE_ADMIX:
+    raise ImportError("straxen must be installed with admix to use this script")
+
 
 def get_chunk_number(st, run_id, data_type, chunks):
     """Get chunk_number for per-chunk storage."""