Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
#!/usr/bin/env bash
#SBATCH -J sette
#SBATCH -o sette.%j.out
#SBATCH -e sette.%j.err
#SBATCH -N 1
#SBATCH --time=0-00:20:00
#SBATCH --account=frtomerc
#SBATCH --partition=par
#SBATCH --qos=np
##SBATCH --ntasks=NPROCS
#SBATCH --ntasks=TOTAL_NPROCS
set -x
# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
# (via sed operating on this template job file).
#
echo " ";
OCORES=NPROCS
XCORES=NXIOPROCS
O_PER_NODE=32
X_PER_NODE=8
if [ $XCORES -le $X_PER_NODE ]; then X_PER_NODE=$XCORES; fi
if [ $OCORES -le $O_PER_NODE ]; then O_PER_NODE=$OCORES; fi
export SETTE_DIR=DEF_SETTE_DIR
###############################################################
#
export MPIRUN="srun"
#
# load sette functions (only post_test_tidyup needed)
#
. ${SETTE_DIR}/all_functions.sh
###############################################################
#
# modules to load
module purge &> /dev/null
module use /home/ar5/modules
module load prgenv/intel intel/2021.4.0 openmpi/4.1.1.1 hdf5-parallel/1.10.6 netcdf4-parallel/4.7.4 xios/trunk/r2320_intel-ompi
module list
# Don't remove neither change the following line
# BODY
#
# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
# (via sed operating on this template job file). Note that the number of compute nodes required
# is also set by the fcm_job.sh on the PBS select header line above.
#
# These variables are needed by post_test_tidyup function in all_functions.sh
#
export XIO_HOME=${XIOS_DIR}
export INPUT_DIR=DEF_INPUT_DIR
export CONFIG_DIR=DEF_CONFIG_DIR
export TOOLS_DIR=DEF_TOOLS_DIR
export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION
export NEW_CONF=DEF_NEW_CONF
export CMP_NAM=DEF_CMP_NAM
export TEST_NAME=DEF_TEST_NAME
export EXE_DIR=DEF_EXE_DIR
ulimit -a
ulimit -s unlimited
#
# end of set up
###############################################################
#
# change to the working directory
#
cd ${EXE_DIR}
echo Running on host `hostname`
echo Time is `date`
echo Directory is `pwd`
sed -i "s/one_file/multiple_file/" file_def_nemo-*.xml
#
# Run the parallel MPI executable
#
echo "Running time ${MPIRUN} ./nemo"
#
if [ $XCORES -gt 0 ]; then
#
# Run MPMD case
#
#XIOS will run on a separate node so will run in parallel queue
if [ ! -f ./xios_server.exe ] && [ -f ${XIO_HOME}/bin/xios_server.exe ]; then
cp ${XIO_HOME}/bin/xios_server.exe .
fi
if [ ! -f ./xios_server.exe ]; then
echo "./xios_server.exe not found"
echo "run aborted"
exit
fi
cat > ./config.file <<-EOF
0-$((OCORES-1)) ./nemo
${OCORES}-$((TOTAL_NPROCS-1)) ./xios_server.exe
EOF
$MPIRUN --ntasks=TOTAL_NPROCS --multi-prog config.file
else
#
# Run SPMD case
#
time ./nemo
fi
#
#
post_test_tidyup
# END_BODY
# Don't remove neither change the previous line
exit