dec2023_FD_MC_production.jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


##TODO -- write usage



DETECTOR="${DETPROD:-HD}"
HC="${HCPROD:-FHC}"

## Build up the fcls to use
if [ "${DETECTOR}" = "HD" ]; then
  ANAFCL="standard_ana_dune10kt_1x2x6.fcl"
  MDDETTYPE="fardet-hd"

  if [ "$HC" = "FHC" ]; then
    RECOFCL="standard_reco2_dune10kt_nu_1x2x6.fcl"
  elif [ "$HC" = "RHC" ]; then
    RECOFCL="standard_reco2_dune10kt_anu_1x2x6.fcl"
  fi

elif [ "${DETECTOR}" = "VD" ]; then
  ANAFCL="anatree_dunevd10kt_1x8x6_3view_30deg_geov3.fcl"
  MDDETTYPE="fardet-vd"

  if [ "$HC" = "FHC" ]; then
    RECOFCL="reco2_dunevd10kt_nu_1x8x6_3view_30deg_geov3.fcl"
  elif [ "$HC" = "RHC" ]; then
    RECOFCL="reco2_dunevd10kt_anu_1x8x6_3view_30deg_geov3.fcl"
  fi

elif [ "${DETECTOR}" = "TEST" ]; then
  RECOFCL="${INPUT_TAR_DIR_LOCAL}/test.fcl"

elif [ "${DETECTOR}" = "TEST2" ]; then
  RECOFCL="${INPUT_TAR_DIR_LOCAL}/test2.fcl"

else
  echo "NEED TO REQUEST EITHER HD OR VD. USER REQUESTED ${DETECTOR}"
  exit 1
fi

echo "Using Reco fcl: ${RECOFCL}"
echo "and Ana fcl: ${ANAFCL}"


#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_81_00d02}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
echo "printing env"
env


##Force certain env vars to force good behavior
#export TF_NUM_THREADS=$JUSTIN_PROCESSORS    
#export OPENBLAS_NUM_THREADS=$JUSTIN_PROCESSORS  
#export JULIA_NUM_THREADS=$JUSTIN_PROCESSORS  
#export MKL_NUM_THREADS=$JUSTIN_PROCESSORS  
#export NUMEXPR_NUM_THREADS=$JUSTIN_PROCESSORS  
#export OMP_NUM_THREADS=$JUSTIN_PROCESSORS   

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
#getfileExit=$?

#if [ $getfileExit -ne 0 ] ; then
#  # Success !
#  # Error -- exit immediately 
#  jobscriptExit=1
#  echo "Error in justin-get-file"
#  exit $jobscriptExit
#fi

##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "DID: ${DID}"

metacat file show -mj ${DID} > old_md.json
mcExit=$?
if [ $mcExit -eq 0 ] ; then
  echo "old metadata:"
  cat old_md.json
else
  echo "Could not retrieve old metadata"
  exit 1 
fi

#
#
now=$(date -u +"%Y%m%dT%H%M%SZ")
#OUTFILE="${OUTPREFIX:-fd_mc_prod_test}.${JUSTIN_REQUEST_ID}"
#OUTFILE="$OUTFILE.$JUSTIN_JOBSUB_ID.${now}"
#OUTFILE=`echo $OUTFILE | sed -e 's/@/./'`
#
####Run reco2
lar -c "${RECOFCL}" \
    -n "${NEVENTS:--1}" \
    "$FILE" >reco2.log 2>&1
    #-o "$OUTFILE.root" \
    #-n "${NEVENTS:--1}" \
    #"$FILE" >$OUTFILE.log 2>&1

larExit=$?
echo "Reco step lar exit code $larExit"

if [ $larExit -eq 0 ] ; then
  # Success !
  # Log the file for justin and move on to the next step
  #echo "$FILE" > justin-processed-pfns.txt
  echo "Moving on to analysis "
else
  # Error -- exit immediately 
  #jobscriptExit=1
  tail -100 reco2.log
  exit $larExit 
fi

if [ "${DETECTOR}" = "TEST" ] || [ "${DETECTOR}" = "TEST2" ]; then
  echo "Done testing. Exiting"
  exit 0
fi
ORIG_OUTFILE=`ls *reco2.root`
OUTFILE=`echo "${ORIG_OUTFILE}_${now}_reco2" | sed -e 's/reco2.root//'`

mv ${ORIG_OUTFILE} ${OUTFILE}.root

### Get the metadata
##TODO -- convert this is the right file extensions
extractor_prod.py --infile ${OUTFILE}.root --campaign ${CAMPAIGN:-fd_mc_2023a_reco2} \
                  --requestid ritm1780305 --no_crc > ${OUTFILE}.root.json
extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

### Convert the metadata to metacat
##TODO -- make sure the RCDS behavior is correct
python $CONVERT_DIR/convert_metadata.py -i ${OUTFILE}.root.json \
                                                -c ${RECOFCL} -j old_md.json \
                                                --app "art.reco2" \
                                                --app_ver "${DUNE_VERSION:-v09_81_00d02}" \
                                                --det "${MDDETTYPE}" \
                                                --parent ${DID}
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "Converted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to convert md"
  exit $converterExit
fi

##If running VD, there will also be pandora files. Rename these
if [ "${DETECTOR}" = "VD" ]; then
  all_good=true
  mv Validation.root "${OUTFILE}_Validation.root" || all_good=false
  mv Pandora_Events.pndr "${OUTFILE}_Pandora_Events.pndr" || all_good=false

  ##Copy over the metadata for pndr
  ##Edit the fields corresepondingly
  sed ${OUTFILE}.root.json -e 's/full-reconstructed/pandora-info/' > ${OUTFILE}_Pandora_Events.pndr.json || all_good=false
  sed -i ${OUTFILE}_Pandora_Events.pndr.json -e 's/artroot/binary/' || all_good=false
  sed -i ${OUTFILE}_Pandora_Events.pndr.json -e '/art.file_format/d' || all_good=false
  
  ##Copy over the metadata for validation.root 
  ##Edit the fields corresepondingly
  sed ${OUTFILE}.root.json -e 's/full-reconstructed/pandora-info/' > ${OUTFILE}_Validation.root.json || all_good=false
  sed -i ${OUTFILE}_Validation.root.json -e 's/artroot/root/' || all_good=false
  sed -i ${OUTFILE}_Validation.root.json -e '/art.file_format/d' || all_good=false

  if [ $all_good = false ]; then
    echo "Something failed when editing pandora metadata"
    jobscriptExit=1
    exit $jobscriptExit
  fi

fi

ANAOUTFILE="${OUTFILE}_ana"
echo "Will output ana to $ANAOUTFILE"

###Run ana 
anastart=`date +"%s"`.0
lar -c "${ANAFCL}" \
    -n "${NEVENTS:--1}" \
    -T "$ANAOUTFILE.root" \
    "$OUTFILE.root" >ana.log 2>&1


larExit=$?
echo "Ana step lar exit code $larExit"
anaend=`date +"%s"`.0
echo "ANA start and end ${anastart} ${anaend}"

### TODO: Handle this correctly
if [ $larExit -eq 0 ] ; then
  # Success !
  # Log the file for justin and move on to the next step
  echo "$FILE" > justin-processed-pfns.txt
else
  # Error -- exit immediately 
  jobscriptExit=1
  tail -100 ana.log
  exit $larExit
fi

sed_good=true
sed ${OUTFILE}.root.json -e 's/full-reconstructed/root-tuple-virtual/' > ${ANAOUTFILE}.root.json || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e 's/artroot/root/' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/art.file_format/d' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/art.process/d' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "s/${RECOFCL}/${ANAFCL}/" || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/campaign/!s/reco2/anatree/' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "s/${DID}/${JUSTIN_SCOPE}:${OUTFILE}.root/" || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "/core.start_time/c\\\"core.start_time\": ${anastart},"
sed -i ${ANAOUTFILE}.root.json -e "/core.end_time/c\\\"core.end_time\": ${anaend},"

if [ $sed_good = false ]; then
  echo "Something failed when editing ana metadata"
  jobscriptExit=1
  exit $jobscriptExit
fi

ls
exit 0