From d5bd06feea4d6837a24108b6d40385b2fdd4ce02 Mon Sep 17 00:00:00 2001
From: Guillaume S <gsamson@mercator-ocean.fr>
Date: Fri, 16 Sep 2022 16:48:03 +0200
Subject: [PATCH] ARCH + BATCH files for INTEL + OPENMPI ON ECMWF AA
---
arch/MERCATOR/arch-X64_AA_INTEL_OMPI.fcm | 62 ++++++++++
.../MERCATOR/arch-X64_AA_INTEL_OMPI_DEBUG.fcm | 62 ++++++++++
sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI | 110 ++++++++++++++++++
.../batch-X64_AA_INTEL_OMPI_DEBUG | 1 +
.../batch-mpmd-X64_AA_INTEL_OMPI | 1 +
.../batch-mpmd-X64_AA_INTEL_OMPI_DEBUG | 1 +
6 files changed, 237 insertions(+)
create mode 100644 arch/MERCATOR/arch-X64_AA_INTEL_OMPI.fcm
create mode 100644 arch/MERCATOR/arch-X64_AA_INTEL_OMPI_DEBUG.fcm
create mode 100644 sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI
create mode 120000 sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI_DEBUG
create mode 120000 sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI
create mode 120000 sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI_DEBUG
diff --git a/arch/MERCATOR/arch-X64_AA_INTEL_OMPI.fcm b/arch/MERCATOR/arch-X64_AA_INTEL_OMPI.fcm
new file mode 100644
index 00000000..2e8a3139
--- /dev/null
+++ b/arch/MERCATOR/arch-X64_AA_INTEL_OMPI.fcm
@@ -0,0 +1,62 @@
+# compiler options for AA (using INTEL compiler & OpenMPI)
+#
+# module purge
+# module use /home/ar0s/modules
+# module load prgenv/intel intel/2021.4.0 openmpi/4.1.1.1 hdf5-parallel/1.10.6 netcdf4-parallel/4.7.4 xios/trunk/rev2320-nmpi
+#
+#
+# NCDF_INC netcdf4 include file
+# NCDF_LIB netcdf4 library
+# XIOS_INC xios include file (taken into accound only if key_xios is activated)
+# XIOS_LIB xios library (taken into accound only if key_xios is activated)
+# OASIS_INC oasis include file (taken into accound only if key_oasis3 is activated)
+# OASIS_LIB oasis library (taken into accound only if key_oasis3 is activated)
+#
+# FC Fortran compiler command
+# FCFLAGS Fortran compiler flags
+# FFLAGS Fortran 77 compiler flags
+# LD linker
+# LDFLAGS linker flags, e.g. -L<lib dir> if you have libraries
+# FPPFLAGS pre-processing flags
+# AR assembler
+# ARFLAGS assembler flags
+# MK make
+# USER_INC complete list of include files
+# USER_LIB complete list of libraries to pass to the linker
+# CC C compiler used to compile conv for AGRIF
+# CFLAGS compiler flags used with CC
+#
+# Note that:
+# - unix variables "$..." are accpeted and will be evaluated before calling fcm.
+# - fcm variables are starting with a % (and not a $)
+#
+#---------------------------------------------------------------------------------------------
+#---------------------------------------------------------------------------------------------
+# All NETCDF and HDF paths are empty as they are automatically defined through environment
+# variables by the load of modules
+#---------------------------------------------------------------------------------------------
+#---------------------------------------------------------------------------------------------
+#
+#
+%NCDF_INC ${NETCDF4_INCLUDE}
+%NCDF_LIB ${NETCDF4_LIB} -L${HDF5_DIR}/lib -Wl,-rpath,${HDF5_DIR}/lib -lhdf5_hl -lhdf5 -lz
+%XIOS_INC -I${XIOS_INC}
+%XIOS_LIB -L${XIOS_LIB} -lxios -lstdc++
+%OASIS_INC -I${OASIS_DIR}/build/lib/mct -I${OASIS_DIR}/build/lib/psmile.MPI1
+%OASIS_LIB -L${OASIS_DIR}/lib -lpsmile.MPI1 -lmct -lmpeu -lscrip
+
+%CPP cpp
+%FC mpifort -c -cpp
+%FCFLAGS -march=core-avx2 -i4 -r8 -O3 -fp-model strict -fno-alias -align array64byte
+%FFLAGS %FCFLAGS
+%LD mpifort
+%LDFLAGS
+%FPPFLAGS -P -traditional
+%AR ar
+%ARFLAGS rs
+%MK gmake
+%USER_INC %XIOS_INC %OASIS_INC %NCDF_INC
+%USER_LIB %XIOS_LIB %OASIS_LIB %NCDF_LIB
+
+%CC mpicc
+%CFLAGS -O0
diff --git a/arch/MERCATOR/arch-X64_AA_INTEL_OMPI_DEBUG.fcm b/arch/MERCATOR/arch-X64_AA_INTEL_OMPI_DEBUG.fcm
new file mode 100644
index 00000000..dd099479
--- /dev/null
+++ b/arch/MERCATOR/arch-X64_AA_INTEL_OMPI_DEBUG.fcm
@@ -0,0 +1,62 @@
+# compiler options for AA (using INTEL compiler & OpenMPI)
+#
+# module purge
+# module use /home/ar0s/modules
+# module load prgenv/intel intel/2021.4.0 openmpi/4.1.1.1 hdf5-parallel/1.10.6 netcdf4-parallel/4.7.4 xios/trunk/rev2320-nmpi
+#
+#
+# NCDF_INC netcdf4 include file
+# NCDF_LIB netcdf4 library
+# XIOS_INC xios include file (taken into accound only if key_xios is activated)
+# XIOS_LIB xios library (taken into accound only if key_xios is activated)
+# OASIS_INC oasis include file (taken into accound only if key_oasis3 is activated)
+# OASIS_LIB oasis library (taken into accound only if key_oasis3 is activated)
+#
+# FC Fortran compiler command
+# FCFLAGS Fortran compiler flags
+# FFLAGS Fortran 77 compiler flags
+# LD linker
+# LDFLAGS linker flags, e.g. -L<lib dir> if you have libraries
+# FPPFLAGS pre-processing flags
+# AR assembler
+# ARFLAGS assembler flags
+# MK make
+# USER_INC complete list of include files
+# USER_LIB complete list of libraries to pass to the linker
+# CC C compiler used to compile conv for AGRIF
+# CFLAGS compiler flags used with CC
+#
+# Note that:
+# - unix variables "$..." are accpeted and will be evaluated before calling fcm.
+# - fcm variables are starting with a % (and not a $)
+#
+#---------------------------------------------------------------------------------------------
+#---------------------------------------------------------------------------------------------
+# All NETCDF and HDF paths are empty as they are automatically defined through environment
+# variables by the load of modules
+#---------------------------------------------------------------------------------------------
+#---------------------------------------------------------------------------------------------
+#
+#
+%NCDF_INC ${NETCDF4_INCLUDE}
+%NCDF_LIB ${NETCDF4_LIB} -L${HDF5_DIR}/lib -Wl,-rpath,${HDF5_DIR}/lib -lhdf5_hl -lhdf5 -lz
+%XIOS_INC -I${XIOS_INC}
+%XIOS_LIB -L${XIOS_LIB} -lxios -lstdc++
+%OASIS_INC -I${OASIS_DIR}/build/lib/mct -I${OASIS_DIR}/build/lib/psmile.MPI1
+%OASIS_LIB -L${OASIS_DIR}/lib -lpsmile.MPI1 -lmct -lmpeu -lscrip
+
+%CPP cpp
+%FC mpifort -c -cpp
+%FCFLAGS -march=core-avx2 -i4 -r8 -g -O0 -debug all -traceback -fp-model strict -ftrapuv -check all,noarg_temp_created -fpe-all0 -ftz -init=arrays,snan,huge
+%FFLAGS %FCFLAGS
+%LD mpifort
+%LDFLAGS
+%FPPFLAGS -P -traditional
+%AR ar
+%ARFLAGS rs
+%MK gmake
+%USER_INC %XIOS_INC %OASIS_INC %NCDF_INC
+%USER_LIB %XIOS_LIB %OASIS_LIB %NCDF_LIB
+
+%CC mpicc
+%CFLAGS -O0
diff --git a/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI b/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI
new file mode 100644
index 00000000..bdd075c6
--- /dev/null
+++ b/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+
+#SBATCH -J sette
+#SBATCH -o sette.%j.out
+#SBATCH -e sette.%j.err
+#SBATCH -N 1
+#SBATCH --time=0-00:20:00
+#SBATCH --account=frtomerc
+#SBATCH --partition=par
+#SBATCH --qos=np
+##SBATCH --ntasks=NPROCS
+#SBATCH --ntasks=TOTAL_NPROCS
+
+set -x
+
+# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
+# (via sed operating on this template job file).
+#
+ echo " ";
+ OCORES=NPROCS
+ XCORES=NXIOPROCS
+ O_PER_NODE=32
+ X_PER_NODE=8
+ if [ $XCORES -le $X_PER_NODE ]; then X_PER_NODE=$XCORES; fi
+ if [ $OCORES -le $O_PER_NODE ]; then O_PER_NODE=$OCORES; fi
+ export SETTE_DIR=DEF_SETTE_DIR
+
+###############################################################
+#
+export MPIRUN="srun"
+#
+# load sette functions (only post_test_tidyup needed)
+#
+ . ${SETTE_DIR}/all_functions.sh
+###############################################################
+#
+# modules to load
+module purge &> /dev/null
+module use /home/ar5/modules
+module load prgenv/intel intel/2021.4.0 openmpi/4.1.1.1 hdf5-parallel/1.10.6 netcdf4-parallel/4.7.4 xios/trunk/r2320_intel-ompi
+module list
+
+# Don't remove neither change the following line
+# BODY
+#
+# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
+# (via sed operating on this template job file). Note that the number of compute nodes required
+# is also set by the fcm_job.sh on the PBS select header line above.
+#
+# These variables are needed by post_test_tidyup function in all_functions.sh
+#
+ export XIO_HOME=${XIOS_DIR}
+ export INPUT_DIR=DEF_INPUT_DIR
+ export CONFIG_DIR=DEF_CONFIG_DIR
+ export TOOLS_DIR=DEF_TOOLS_DIR
+ export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION
+ export NEW_CONF=DEF_NEW_CONF
+ export CMP_NAM=DEF_CMP_NAM
+ export TEST_NAME=DEF_TEST_NAME
+ export EXE_DIR=DEF_EXE_DIR
+ ulimit -a
+ ulimit -s unlimited
+#
+# end of set up
+###############################################################
+#
+# change to the working directory
+#
+ cd ${EXE_DIR}
+ echo Running on host `hostname`
+ echo Time is `date`
+ echo Directory is `pwd`
+ sed -i "s/one_file/multiple_file/" file_def_nemo-*.xml
+#
+# Run the parallel MPI executable
+#
+ echo "Running time ${MPIRUN} ./nemo"
+#
+ if [ $XCORES -gt 0 ]; then
+#
+# Run MPMD case
+#
+ #XIOS will run on a separate node so will run in parallel queue
+ if [ ! -f ./xios_server.exe ] && [ -f ${XIO_HOME}/bin/xios_server.exe ]; then
+ cp ${XIO_HOME}/bin/xios_server.exe .
+ fi
+ if [ ! -f ./xios_server.exe ]; then
+ echo "./xios_server.exe not found"
+ echo "run aborted"
+ exit
+ fi
+ cat > ./config.file <<-EOF
+0-$((OCORES-1)) ./nemo
+${OCORES}-$((TOTAL_NPROCS-1)) ./xios_server.exe
+EOF
+ $MPIRUN --ntasks=TOTAL_NPROCS --multi-prog config.file
+ else
+#
+# Run SPMD case
+#
+ time ./nemo
+ fi
+#
+
+#
+ post_test_tidyup
+
+# END_BODY
+# Don't remove neither change the previous line
+ exit
diff --git a/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI_DEBUG b/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI_DEBUG
new file mode 120000
index 00000000..11d73e63
--- /dev/null
+++ b/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI_DEBUG
@@ -0,0 +1 @@
+batch-X64_AA_INTEL_OMPI
\ No newline at end of file
diff --git a/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI b/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI
new file mode 120000
index 00000000..11d73e63
--- /dev/null
+++ b/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI
@@ -0,0 +1 @@
+batch-X64_AA_INTEL_OMPI
\ No newline at end of file
diff --git a/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI_DEBUG b/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI_DEBUG
new file mode 120000
index 00000000..178a7f6e
--- /dev/null
+++ b/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI_DEBUG
@@ -0,0 +1 @@
+batch-X64_AA_INTEL_OMPI_DEBUG
\ No newline at end of file
--
GitLab