From c2a4e974505594bc1a5c54212b0df80ba8b66eba Mon Sep 17 00:00:00 2001
From: accowa <acc@noc.ac.uk>
Date: Thu, 9 Dec 2021 20:48:37 +0000
Subject: [PATCH] Some ARCHER2-specific changes (arch files and sette scripts
only)
---
arch/NOC/arch-X86_ARCHER2-Gnu.fcm | 62 ++++++++++++++++++++
sette/BATCH_TEMPLATE/batch-X86_ARCHER2-Cray | 2 +-
sette/BATCH_TEMPLATE/batch-X86_ARCHER2-Gnu | 6 ++
sette/prepare_job.sh | 5 +-
sette/super_sette_waitq.sh | 65 +++++++++++++++++++++
5 files changed, 138 insertions(+), 2 deletions(-)
create mode 100644 arch/NOC/arch-X86_ARCHER2-Gnu.fcm
create mode 100644 sette/BATCH_TEMPLATE/batch-X86_ARCHER2-Gnu
create mode 100755 sette/super_sette_waitq.sh
diff --git a/arch/NOC/arch-X86_ARCHER2-Gnu.fcm b/arch/NOC/arch-X86_ARCHER2-Gnu.fcm
new file mode 100644
index 000000000..eaf9fbc0c
--- /dev/null
+++ b/arch/NOC/arch-X86_ARCHER2-Gnu.fcm
@@ -0,0 +1,62 @@
+# compiler options for Archer CRAY XC-30 (using crayftn compiler)
+#
+# NCDF_HOME root directory containing lib and include subdirectories for netcdf4
+# HDF5_HOME root directory containing lib and include subdirectories for HDF5
+# XIOS_HOME root directory containing lib for XIOS
+# OASIS_HOME root directory containing lib for OASIS
+#
+# NCDF_INC netcdf4 include file
+# NCDF_LIB netcdf4 library
+# XIOS_INC xios include file (taken into accound only if key_xios is activated)
+# XIOS_LIB xios library (taken into accound only if key_xios is activated)
+# OASIS_INC oasis include file (taken into accound only if key_oasis3 is activated)
+# OASIS_LIB oasis library (taken into accound only if key_oasis3 is activated)
+#
+# FC Fortran compiler command
+# FCFLAGS Fortran compiler flags
+# FFLAGS Fortran 77 compiler flags
+# LD linker
+# LDFLAGS linker flags, e.g. -L<lib dir> if you have libraries
+# FPPFLAGS pre-processing flags
+# AR assembler
+# ARFLAGS assembler flags
+# MK make
+# USER_INC complete list of include files
+# USER_LIB complete list of libraries to pass to the linker
+# CC C compiler used to compile conv for AGRIF
+# CFLAGS compiler flags used with CC
+#
+# Note that:
+# - unix variables "$..." are accpeted and will be evaluated before calling fcm.
+# - fcm variables are starting with a % (and not a $)
+#
+%NCDF_HOME $NETCDF_DIR
+%HDF5_HOME $HDF5_DIR
+%XIOS_HOME /work/n01/shared/nemo/xios-trunk-gnu
+#OASIS_HOME
+
+%NCDF_INC -I%NCDF_HOME/include -I%HDF5_HOME/include
+%NCDF_LIB -L%HDF5_HOME/lib -L%NCDF_HOME/lib -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lz
+%XIOS_INC -I%XIOS_HOME/inc
+%XIOS_LIB -L%XIOS_HOME/lib -lxios
+#OASIS_INC -I%OASIS_HOME/build/lib/mct -I%OASIS_HOME/build/lib/psmile.MPI1
+#OASIS_LIB -L%OASIS_HOME/lib -lpsmile.MPI1 -lmct -lmpeu -lscrip
+
+%CPP cpp -Dkey_nosignedzero
+%FC ftn
+%FCFLAGS -O2 -cpp -fallow-argument-mismatch -fdefault-real-8 -fcray-pointer -ffree-line-length-none
+%FFLAGS -O2 -cpp -fallow-argument-mismatch -fdefault-real-8 -fcray-pointer -ffree-line-length-none
+%LD CC
+%FPPFLAGS -P -traditional
+%LDFLAGS -lmpichf90
+%AR ar
+%ARFLAGS -r
+%MK gmake
+%USER_INC %XIOS_INC %NCDF_INC
+%USER_LIB %XIOS_LIB %NCDF_LIB
+#USER_INC %XIOS_INC %OASIS_INC %NCDF_INC
+#USER_LIB %XIOS_LIB %OASIS_LIB %NCDF_LIB
+
+%CC cc -Wl,"--allow-multiple-definition"
+%CFLAGS -O2 -Wl,"--allow-multiple-definition"
+bld::tool::fc_modsearch -J
diff --git a/sette/BATCH_TEMPLATE/batch-X86_ARCHER2-Cray b/sette/BATCH_TEMPLATE/batch-X86_ARCHER2-Cray
index 33dfbf16f..a3223fdf3 100644
--- a/sette/BATCH_TEMPLATE/batch-X86_ARCHER2-Cray
+++ b/sette/BATCH_TEMPLATE/batch-X86_ARCHER2-Cray
@@ -1,6 +1,6 @@
#!/bin/bash
#
# A batch script will be generated using:
-# /work/n01/shared/acc/mkslurm_settejob -S $NXIO_PROC -s 8 -m 4 -C $NB_PROC -g 2 -a n01-CLASS -j sette_job -t 20:00 > ${SETTE_DIR}/job_batch_template
+# /work/n01/shared/acc/mkslurm_settejob_4.2 -S $NXIO_PROC -s 8 -m 4 -C $NB_PROC -g 2 -a n01-CLASS -j sette_job -t 20:00 > ${SETTE_DIR}/job_batch_template
# by prepare_job.sh
#
diff --git a/sette/BATCH_TEMPLATE/batch-X86_ARCHER2-Gnu b/sette/BATCH_TEMPLATE/batch-X86_ARCHER2-Gnu
new file mode 100644
index 000000000..88ef23b40
--- /dev/null
+++ b/sette/BATCH_TEMPLATE/batch-X86_ARCHER2-Gnu
@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+# A batch script will be generated using:
+# /work/n01/shared/nemo/mkslurm_settejob_4.2_Gnu -S $NXIO_PROC -s 8 -m 4 -C $NB_PROC -g 2 -a n01-CLASS -j sette_job -t 20:00 > ${SETTE_DIR}/job_batch_template
+# by prepare_job.sh
+#
diff --git a/sette/prepare_job.sh b/sette/prepare_job.sh
index d4339d1e0..2cc347146 100755
--- a/sette/prepare_job.sh
+++ b/sette/prepare_job.sh
@@ -198,9 +198,12 @@ fi
NB_NODES=$( echo $NB_PROC $NXIO_PROC | awk '{printf("%d",($1 + $2 ) / 16 + 1 )}')
fi
;;
- X86_ARCHER2*)
+ X86_ARCHER2-Cray)
MK_TEMPLATE=$( /work/n01/shared/nemo/mkslurm_settejob_4.2 -S $NXIO_PROC -s 8 -m 4 -C $NB_PROC -g 2 -a n01-CLASS -j sette_job -t 20:00 > ${SETTE_DIR}/job_batch_template )
;;
+ X86_ARCHER2-Gnu)
+ MK_TEMPLATE=$( /work/n01/shared/nemo/mkslurm_settejob_4.2_Gnu -S $NXIO_PROC -s 8 -m 4 -C $NB_PROC -g 2 -a n01-CLASS -j sette_job -t 20:00 > ${SETTE_DIR}/job_batch_template )
+ ;;
XC40_METO*) #Setup for Met Office XC40 with any compiler
# ocean cores are packed 32 to a node
# If we need more than one node then have to use parallel queue and XIOS must have a node to itself
diff --git a/sette/super_sette_waitq.sh b/sette/super_sette_waitq.sh
new file mode 100755
index 000000000..78a7efd93
--- /dev/null
+++ b/sette/super_sette_waitq.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# set -vx
+# Simple script to robustly run a full suite of SETTE tests
+#
+########################################
+function wait_on_q()
+{
+# SETTE testing on ARCHER2 uses the test queue in which users are limited to
+# 16 queued jobs (including a maximum of 4 running). To prevent sette testing
+# breaching this limit each configuration is processed separately and
+# processing only begins when the user has no more than 12 jobs already queued.
+# The supposition here is that each config forks no more than 4 tests - may need
+# re-revaluating if used for PHYSICS tests.
+#
+# This function checks the queue usage and waits if necessary until the queue
+# has drained sufficiently for the next test.
+NRUN=999
+NIT=0
+BATCH_STAT="squeue -u $USER"
+BATCH_NAME=sette
+echo "Checking queues"
+while [[ $NRUN -gt 12 && $nit -le 1080 ]]; do
+ nit=$((nit+1))
+ NRUN=$( ${BATCH_STAT} | grep ${BATCH_NAME} | wc -l )
+ if [[ $NRUN -gt 12 ]]; then
+ printf "%-3d %s\r" $NRUN 'nemo_sette runs still in queue or running ...';
+ else
+ printf "%-50s\n" "Queues sufficiently drained"
+ return 99
+ fi
+ sleep 10
+done
+echo "Something has gone wrong. Excessive wait time has been exceeded"
+exit
+}
+#
+########################################
+# Start of main script
+########################################
+FULLSET=( ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 AGRIF WED025 GYRE_PISCES SAS ORCA2_ICE_OBS SWG ICE_AGRIF OVERFLOW LOCK_EXCHANGE VORTEX ISOMIP+ )
+#
+GROUP_SETS=( "-r" )
+#
+# These groups sets correspond to the following test regimes:
+#
+# A. complete sets with various combinations of options:
+#
+ printf "%-93s %s\n" "Full tests - <branch_name> (using *_ST config dirs) : " "${GROUP_SETS[0]}"
+#
+# A. Full tests
+for gs in 0
+do
+ for n in `seq 0 1 $(( ${#FULLSET[@]} - 1 ))`
+ do
+ confstr="${FULLSET[$n]}"
+ # compile seperately since the final link sometimes has a bus error on ARCHER2 (which never happens on the 2nd attempt)
+ echo ./sette.sh ${GROUP_SETS[$gs]} -x "COMPILE" -n "$confstr"
+ ./sette.sh ${GROUP_SETS[$gs]} -x "COMPILE" -n "$confstr"
+ # Now run the test (and finish linking if necessary)
+ echo ./sette.sh ${GROUP_SETS[$gs]} -x "RESTART REPRO CORRUPT" -n "$confstr"
+ ./sette.sh ${GROUP_SETS[$gs]} -x "RESTART REPRO CORRUPT" -n "$confstr"
+ wait_on_q
+ done
+done
+exit
--
GitLab