Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
#!/bin/bash
# set -vx
# Simple script to robustly run a full suite of SETTE tests
#
########################################
function wait_on_q()
{
# SETTE testing on ARCHER2 uses the test queue in which users are limited to
# 16 queued jobs (including a maximum of 4 running). To prevent sette testing
# breaching this limit each configuration is processed separately and
# processing only begins when the user has no more than 12 jobs already queued.
# The supposition here is that each config forks no more than 4 tests - may need
# re-revaluating if used for PHYSICS tests.
#
# This function checks the queue usage and waits if necessary until the queue
# has drained sufficiently for the next test.
NRUN=999
NIT=0
BATCH_STAT="squeue -u $USER"
BATCH_NAME=sette
echo "Checking queues"
while [[ $NRUN -gt 12 && $nit -le 1080 ]]; do
nit=$((nit+1))
NRUN=$( ${BATCH_STAT} | grep ${BATCH_NAME} | wc -l )
if [[ $NRUN -gt 12 ]]; then
printf "%-3d %s\r" $NRUN 'nemo_sette runs still in queue or running ...';
else
printf "%-50s\n" "Queues sufficiently drained"
return 99
fi
sleep 10
done
echo "Something has gone wrong. Excessive wait time has been exceeded"
exit
}
#
########################################
# Start of main script
########################################
FULLSET=( ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 AGRIF WED025 GYRE_PISCES SAS ORCA2_ICE_OBS SWG ICE_AGRIF OVERFLOW LOCK_EXCHANGE VORTEX ISOMIP+ )
#
GROUP_SETS=( "-r" )
#
# These groups sets correspond to the following test regimes:
#
# A. complete sets with various combinations of options:
#
printf "%-93s %s\n" "Full tests - <branch_name> (using *_ST config dirs) : " "${GROUP_SETS[0]}"
#
# A. Full tests
for gs in 0
do
for n in `seq 0 1 $(( ${#FULLSET[@]} - 1 ))`
do
confstr="${FULLSET[$n]}"
# compile seperately since the final link sometimes has a bus error on ARCHER2 (which never happens on the 2nd attempt)
echo ./sette.sh ${GROUP_SETS[$gs]} -x "COMPILE" -n "$confstr"
./sette.sh ${GROUP_SETS[$gs]} -x "COMPILE" -n "$confstr"
# Now run the test (and finish linking if necessary)
echo ./sette.sh ${GROUP_SETS[$gs]} -x "RESTART REPRO CORRUPT" -n "$confstr"
./sette.sh ${GROUP_SETS[$gs]} -x "RESTART REPRO CORRUPT" -n "$confstr"
wait_on_q
done
done
exit