orca.pbs

#!/bin/bash

## University of New Mexico Center for Advanced Research Computing
## Ryan Johnson, May 13, 2015
## PBS submission script to run orca code

#PBS -l walltime=01:00:00
#PBS -l nodes=2:ppn=8
#PBS -N TestIt

# User defined variables
# Add any files here which are needed by ORCA.
# At a minimum, you need the .inp file, which must be first.
files=( "inp.inp" "inp.xyz" )

cwd=$PBS_O_WORKDIR

export RSH_COMMAND="ssh"

# Load Orca and included openmpi modules
module load orca/4.0.1

# Get the full path to orca (helps with mpi problems)
ORCA_EXEC=$(which orca)

# Internal variables for creating scratch dir
PBS_JOBNUM=$(echo PBS_JOBID | cut -d"." -f1 )
input=${files[0]}
NodeList=( $(cat "$PBS_NODEFILE" | sort | uniq -c) )
ExecHost=$(hostname -s)
JobID=$(echo -n "$PBS_JOBID"|cut -d'.' -f1)

# Choosing scratch location based on machine
# Since wheeler does not have hard drives on compute nodes
case "$HOST" in 
    *wheeler*)
        # Define and create the scratch directry
        SCRDIR="/wheeler/scratch/$USER/.orcajobs/$JobID" 
        mkdir -p "$SCRDIR" ;;
    *)
        # Define and create the scratch directory
        SCRDIR="/tmp/orca/$(whoami)/$PBS_JOBNUM" 
        for (( i=0 ; i < ${#NodeList[@]} ; i++ ))
        do
                ((i++))
                if [[ ${NodeList[$i]} == "$ExecHost" ]] ; then
                        mkdir -p "$SCRDIR"
                else
                        ssh -x -n "${NodeList[$i]}" "mkdir -p $SCRDIR"
                fi
        done  ;;
esac 


# Create .nodes file and copy to SCRDIR
cp "$PBS_NODEFILE" "$cwd/${input:0:${#input}-4}.nodes"
cp "$cwd/${input:0:${#input}-4}.nodes" "$SCRDIR"

for file in "${files[@]}"
do
	cp "$cwd/$file" "$SCRDIR/"
done


# cd to scratch dir and start orca, redirect output to the PBS_O_WORKDIR

cd "$SCRDIR"

echo Starting job: "$(date)"
echo Job number: "$(echo "$JobID")"

$ORCA_EXEC "$input" > "$cwd/${input:0:${#input}-4}.out"


# orca finished, copy output to user dir
cp "$SCRDIR"/*.* "$cwd"

# remove scratch dir
for (( i=0 ; i < ${#NodeList[@]} ; i++ ))
do
        ((i++))
        if [[ ${NodeList[$i]} == "$ExecHost" ]] ; then
                rm -rf "$SCRDIR"
        else
                ssh -x -n "${NodeList[$i]}" "rm -rf $SCRDIR"
        fi
done

# All done
echo Finish: "$(date)"