diff --git a/outsource/scripts/decompression.sh b/outsource/scripts/decompression.sh new file mode 100644 index 0000000..a962eb3 --- /dev/null +++ b/outsource/scripts/decompression.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +workflow=$1 + +mkdir -p $workflow/outputs/strax_data + +for tarball in $(ls $workflow/outputs | grep tar); do + # run_id=$( echo $tarball | cut -d '-' -f 1 ) + # run_id=$( echo $run_id | sed 's/^0*//' ) + # if [[ $run_id -lt 54000 ]]; then + # continue + # fi + # echo $tarball + tar -xvzf $workflow/outputs/$tarball -C $workflow/outputs/strax_data --strip-components=1 + # Move resources testing files to the outputs folder + mv $workflow/outputs/strax_data/*.npy $workflow/outputs/. + mv $workflow/outputs/strax_data/*.json $workflow/outputs/. +done diff --git a/outsource/scripts/gfal_download.sh b/outsource/scripts/gfal_download.sh new file mode 100644 index 0000000..13512d1 --- /dev/null +++ b/outsource/scripts/gfal_download.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +list=$1 +workflow=$2 + +mkdir -p $workflow/outputs + +find $workflow/outputs -type f -size 0 -delete + +for tarball in $(cat $list); do + _tarball=$(basename $tarball) + if [ ! -f $workflow/outputs/$_tarball ]; then + gfal-copy -f -p -t 7200 -T 7200 $tarball $workflow/outputs/. + fi +done diff --git a/outsource/scripts/rm_unfinished.sh b/outsource/scripts/rm_unfinished.sh new file mode 100644 index 0000000..917a0d8 --- /dev/null +++ b/outsource/scripts/rm_unfinished.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +workflow=$1 + +# Define directories +outputs=$workflow/outputs +del=$workflow/del +lower_log=lower_done.log +upper_log=upper_done.log + +# Get the run IDs from the files +ls $outputs | grep let | cut -d'/' -f11 | cut -d'-' -f1 | sort | uniq > $lower_log +ls $outputs | grep event | cut -d'/' -f11 | cut -d'-' -f1 | sort | uniq > $upper_log + +# Ensure the delete folder exists +mkdir -p $del + +# Read each run ID from the file +while IFS= read -r run_id; do + # Find files matching the run ID + matching_files=($(ls $outputs | grep $run_id)) + + # Check the number of matching files + if [ ${#matching_files[@]} -ne 2 ]; then + for file in "${matching_files[@]}"; do + mv $outputs/$file $del/ + echo "Moved: $outputs/$file -> $del/" + done + fi +done < <(cat $lower_log $upper_log | sort | uniq)