-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathrun_gpu.sh
executable file
·113 lines (87 loc) · 3.91 KB
/
run_gpu.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
#!/bin/bash
#SBATCH --account=proj16
# SBATCH --partition=prod
#SBATCH --time=01:00:00
#SBATCH --nodes=1
#SBATCH --constraint=volta
#SBATCH --gres=gpu:4
#SBATCH --ntasks-per-node=36
#SBATCH --cpus-per-task=2
#SBATCH --exclusive
#SBATCH --mem=0
# Stop on error
#set -e
# =============================================================================
# SIMULATION PARAMETERS TO EDIT
# =============================================================================
# Using top level source and install directory, set the HOC_LIBRARY_PATH for simulator
BASE_DIR=$(pwd)/benchmark
INSTALL_DIR=$BASE_DIR/install
SOURCE_DIR=$BASE_DIR/sources
export HOC_LIBRARY_PATH=$BASE_DIR/channels/lib/hoclib
. $SOURCE_DIR/venv/bin/activate
export PYTHONPATH=$INSTALL_DIR/NRN/lib/python:$PYTHONPATH
#Change this according to the desired runtime of the benchmark
export SIM_TIME=10
# Number of cells ((LCM of #cores_system1, #core_system2)*#cell_types)
export NUM_CELLS=$((360*22))
# GID for prcellstate (-1 for none)
export PRCELL_GID=-1
# =============================================================================
# Enter the channel benchmark directory
cd $BASE_DIR/channels
rm -rf coredat_gpu
rm NRN_GPU.spk GPU_MOD2C.spk GPU_NMODL.spk
rm NRN_GPU.log GPU_MOD2C.log GPU_NMODL.log
echo "----------------- NEURON SIM (CPU) ----------------"
srun dplace $INSTALL_DIR/NRN/special/x86_64/special -mpi -c arg_tstop=$SIM_TIME -c arg_target_count=$NUM_CELLS -c arg_prcell_gid=$PRCELL_GID $HOC_LIBRARY_PATH/init.hoc 2>&1 | tee NRN_GPU.log
# Sort the spikes
cat out.dat | sort -k 1n,1n -k 2n,2n > NRN_GPU.spk
rm out.dat
echo "----------------- Produce coredat ----------------"
srun dplace $INSTALL_DIR/NRN/special/x86_64/special -mpi -c arg_dump_coreneuron_model=1 -c arg_tstop=$SIM_TIME -c arg_target_count=$NUM_CELLS $HOC_LIBRARY_PATH/init.hoc
mv coredat coredat_gpu
# =============================================================================
nvidia-cuda-mps-control -d # Start the daemon
echo "----------------- CoreNEURON SIM (GPU_MOD2C) ----------------"
srun dplace $INSTALL_DIR/GPU_MOD2C/special/x86_64/special-core --mpi --voltage 1000. --gpu --cell-permute 2 --tstop $SIM_TIME -d coredat_gpu --prcellgid $PRCELL_GID 2>&1 | tee GPU_MOD2C.log
# Sort the spikes
cat out.dat | sort -k 1n,1n -k 2n,2n > GPU_MOD2C.spk
rm out.dat
#echo "----------------- CoreNEURON SIM (GPU_NMODL) ----------------"
#srun dplace $INSTALL_DIR/GPU_NMODL/special/x86_64/special-core --mpi --voltage 1000. --gpu --cell-permute 2 --tstop $SIM_TIME -d coredat_gpu --prcellgid $PRCELL_GID 2>&1 | tee GPU_NMODL.log
## Sort the spikes
#cat out.dat | sort -k 1n,1n -k 2n,2n > GPU_NMODL.spk
#rm out.dat
echo quit | nvidia-cuda-mps-control
# =============================================================================
echo "---------------------------------------------"
echo "-------------- Compare Spikes ---------------"
echo "---------------------------------------------"
DIFF=$(diff NRN_GPU.spk GPU_MOD2C.spk)
if [ "$DIFF" != "" ]
then
echo "NRN_GPU.spk GPU_MOD2C.spk are not the same"
else
echo "NRN_GPU.spk GPU_MOD2C.spk are the same"
fi
#DIFF=$(diff NRN_GPU.spk GPU_NMODL.spk)
#if [ "$DIFF" != "" ]
#then
# echo "NRN_GPU.spk GPU_NMODL.spk are not the same"
#else
# echo "NRN_GPU.spk GPU_NMODL.spk are the same"
#fi
# =============================================================================
echo "---------------------------------------------"
echo "----------------- SIM STATS -----------------"
echo "---------------------------------------------"
echo "Number of cells: $NUM_CELLS"
echo "----------------- NEURON SIM STATS (CPU) ----------------"
grep "psolve" NRN_GPU.log
echo "----------------- CoreNEURON SIM (GPU_MOD2C) STATS ----------------"
grep "Solver Time" GPU_MOD2C.log
#echo "----------------- CoreNEURON SIM (GPU_NMODL) STATS ----------------"
#grep "Solver Time" GPU_NMODL.log
echo "---------------------------------------------"
echo "---------------------------------------------"