run

#!/bin/bash

set -e

function usage {
	echo "p/run [--print] [--wait] model example.xml"
	exit 2
}

PP=$(dirname $0)
test -f $PP/conf.ini || $PP/config
test -f $PP/conf.ini || exit -1
. $PP/lib.bash
source_conf

ENGINE="$ENGINE_RUN"
ONLY_PRINT=false
RUN_WAIT=false
TIME="1:00:00"
for arg
do
	shift
	case "$arg" in
		-h|--help) usage;;
		--slurm) ENGINE="slurm" ;;
		--pbs) ENGINE="pbs" ;;
		--local) ENGINE="local" ;;
		--print) ONLY_PRINT=true;;
		--wait) RUN_WAIT=true;;
		--time=*) TIME="${arg#--time=}";;
		--name=*) NAME="${arg#--name=}";;
		--queue=*) MAIN_PARTITION="${arg#--queue=}";;
		--xpath=*) XPATH="${arg#--xpath=}";;
		*) set -- "$@" "$arg";;
	esac
done
set -e

MODEL=$1
if test -z "$MODEL"
then
	usage
fi
SOLVER=$TCLB/CLB/$MODEL/main
if [ ! -f "$SOLVER" ]
then
	echo "Model '$MODEL' not found ('$SOLVER' doesn't exist)"
	usage
fi
shift

CASE=$1
if test -z "$CASE"
then
	usage
fi
if [ ! -f "$CASE" ]
then
	echo "Case file '$CASE' doesn't exist"
	usage
fi
shift

TASKS=$1
if [ -z "$TASKS" ]
then
	TASKS=1
else
	shift
fi

if [ $TASKS -gt $MAX_TASKS_PER_NODE ]
then
	TASKS_PER_NODE=$MAX_TASKS_PER_NODE
	NODES=$[$TASKS/$TASKS_PER_NODE]
	CORES_PER_TASK=$CORES_PER_TASK_FULL
else
	TASKS_PER_NODE=$TASKS
	NODES=1
fi

# REMAINDER=$[($NODES*$TASKS_PER_NODE)%$TASKS]
if [ $TASKS -ne $[$NODES*$TASKS_PER_NODE] ]
then
	echo Requested number of cores/gpus \($TASKS\) is not $NODES x $TASKS_PER_NODE
	exit -1
fi

CORES=$[$NODES*$TASKS_PER_NODE*$CORES_PER_TASK]

if test -z "$NAME"
then
	NAME="TCLB:$CASE"
fi

GPU_PER_TASK=0
if test "$RUN_GPU" == "y"
then
	GPU_PER_TASK=1
fi

echo "Running:"
echo "  case file: $CASE"
echo "  model    : $MODEL"
echo "  MPI ranks: $TASKS"
echo "    CPU cores per rank: $CORES_PER_TASK"
echo "    GPUs per rank     : $GPU_PER_TASK"
echo "    memory per rank   : $MEMORY_PER_TASK"
echo "  Walltime : $TIME"
if ! test -z "$1"
then
	echo "  Options  :" "$@"
fi
echo

source_engine $ENGINE
(
	set -e
	q_header $Q_HEADER_SHELL_FLAGS
	q_name $NAME
	q_queue $MAIN_PARTITION
	q_grant $GRANT
	q_qos $MAIN_QOS
	q_units $NODES $TASKS_PER_NODE $CORES_PER_TASK $GPU_PER_TASK
	q_mem $NODES $TASKS_PER_NODE $MEMORY_PER_TASK
	q_walltime $TIME
	
	echo
	echo "ulimit -l unlimited"
	echo "echo \"running on:\""
	echo "hostname"
	echo
	echo "date"
	
	env_prepare
	echo "cd $PWD"
	echo "$RUN_COMMAND $MPI_OPTS $SINGULARITY_COMMAND $SOLVER $CASE $XPATH"
) >tmp.job.scr

q_run "$@" "tmp.job.scr"