Add scripts of outputs management (#269)

* Add scripts of outputs management

* Debug

outsource/scripts/decompression.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+workflow=$1
+
+mkdir -p $workflow/outputs/strax_data
+
+for tarball in $(ls $workflow/outputs | grep tar); do
+    # run_id=$( echo $tarball | cut -d '-' -f 1 )
+    # run_id=$( echo $run_id | sed 's/^0*//' )
+    # if [[ $run_id -lt 54000 ]]; then
+    #     continue
+    # fi
+    # echo $tarball
+    tar -xvzf $workflow/outputs/$tarball -C $workflow/outputs/strax_data --strip-components=1
+    # Move resources testing files to the outputs folder
+    mv $workflow/outputs/strax_data/*.npy $workflow/outputs/.
+    mv $workflow/outputs/strax_data/*.json $workflow/outputs/.
+done

outsource/scripts/gfal_download.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+list=$1
+workflow=$2
+
+mkdir -p $workflow/outputs
+
+find $workflow/outputs -type f -size 0 -delete
+
+for tarball in $(cat $list); do
+    _tarball=$(basename $tarball)
+    if [ ! -f $workflow/outputs/$_tarball ]; then
+        gfal-copy -f -p -t 7200 -T 7200 $tarball $workflow/outputs/.
+    fi
+done

outsource/scripts/rm_unfinished.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+workflow=$1
+
+# Define directories
+outputs=$workflow/outputs
+del=$workflow/del
+lower_log=lower_done.log
+upper_log=upper_done.log
+
+# Get the run IDs from the files
+ls $outputs | grep let | cut -d'/' -f11 | cut -d'-' -f1 | sort | uniq > $lower_log
+ls $outputs | grep event | cut -d'/' -f11 | cut -d'-' -f1 | sort | uniq > $upper_log
+
+# Ensure the delete folder exists
+mkdir -p $del
+
+# Read each run ID from the file
+while IFS= read -r run_id; do
+    # Find files matching the run ID
+    matching_files=($(ls $outputs | grep $run_id))
+
+    # Check the number of matching files
+    if [ ${#matching_files[@]} -ne 2 ]; then
+        for file in "${matching_files[@]}"; do
+            mv $outputs/$file $del/
+            echo "Moved: $outputs/$file -> $del/"
+        done
+    fi
+done < <(cat $lower_log $upper_log | sort | uniq)