ARCH + BATCH files for INTEL + OPENMPI ON ECMWF AA

arch/MERCATOR/arch-X64_AA_INTEL_OMPI.fcm

+# compiler options for AA (using INTEL compiler & OpenMPI)
+#
+# module purge
+# module use /home/ar0s/modules
+# module load prgenv/intel intel/2021.4.0 openmpi/4.1.1.1 hdf5-parallel/1.10.6 netcdf4-parallel/4.7.4 xios/trunk/rev2320-nmpi
+#
+#
+# NCDF_INC    netcdf4 include file
+# NCDF_LIB    netcdf4 library
+# XIOS_INC    xios include file    (taken into accound only if key_xios is activated)
+# XIOS_LIB    xios library         (taken into accound only if key_xios is activated)
+# OASIS_INC   oasis include file   (taken into accound only if key_oasis3 is activated)
+# OASIS_LIB   oasis library        (taken into accound only if key_oasis3 is activated)
+#
+# FC          Fortran compiler command
+# FCFLAGS     Fortran compiler flags
+# FFLAGS      Fortran 77 compiler flags
+# LD          linker
+# LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries
+# FPPFLAGS    pre-processing flags
+# AR          assembler
+# ARFLAGS     assembler flags
+# MK          make
+# USER_INC    complete list of include files
+# USER_LIB    complete list of libraries to pass to the linker
+# CC          C compiler used to compile conv for AGRIF
+# CFLAGS      compiler flags used with CC
+#
+# Note that:
+#  - unix variables "$..." are accpeted and will be evaluated before calling fcm.
+#  - fcm variables are starting with a % (and not a $)
+#
+#---------------------------------------------------------------------------------------------
+#---------------------------------------------------------------------------------------------
+# All NETCDF and HDF paths are empty as they are automatically defined through environment
+# variables by the load of modules
+#---------------------------------------------------------------------------------------------
+#---------------------------------------------------------------------------------------------
+#
+#
+%NCDF_INC            ${NETCDF4_INCLUDE}
+%NCDF_LIB            ${NETCDF4_LIB} -L${HDF5_DIR}/lib -Wl,-rpath,${HDF5_DIR}/lib -lhdf5_hl -lhdf5 -lz
+%XIOS_INC            -I${XIOS_INC}
+%XIOS_LIB            -L${XIOS_LIB} -lxios -lstdc++
+%OASIS_INC           -I${OASIS_DIR}/build/lib/mct -I${OASIS_DIR}/build/lib/psmile.MPI1
+%OASIS_LIB           -L${OASIS_DIR}/lib -lpsmile.MPI1 -lmct -lmpeu -lscrip
+
+%CPP                 cpp
+%FC                  mpifort -c -cpp
+%FCFLAGS             -march=core-avx2 -i4 -r8 -O3 -fp-model strict -fno-alias -align array64byte
+%FFLAGS              %FCFLAGS
+%LD                  mpifort
+%LDFLAGS             
+%FPPFLAGS            -P -traditional
+%AR                  ar
+%ARFLAGS             rs
+%MK                  gmake
+%USER_INC            %XIOS_INC %OASIS_INC %NCDF_INC
+%USER_LIB            %XIOS_LIB %OASIS_LIB %NCDF_LIB
+
+%CC                  mpicc
+%CFLAGS              -O0

arch/MERCATOR/arch-X64_AA_INTEL_OMPI_DEBUG.fcm

+# compiler options for AA (using INTEL compiler & OpenMPI)
+#
+# module purge
+# module use /home/ar0s/modules
+# module load prgenv/intel intel/2021.4.0 openmpi/4.1.1.1 hdf5-parallel/1.10.6 netcdf4-parallel/4.7.4 xios/trunk/rev2320-nmpi
+#
+#
+# NCDF_INC    netcdf4 include file
+# NCDF_LIB    netcdf4 library
+# XIOS_INC    xios include file    (taken into accound only if key_xios is activated)
+# XIOS_LIB    xios library         (taken into accound only if key_xios is activated)
+# OASIS_INC   oasis include file   (taken into accound only if key_oasis3 is activated)
+# OASIS_LIB   oasis library        (taken into accound only if key_oasis3 is activated)
+#
+# FC          Fortran compiler command
+# FCFLAGS     Fortran compiler flags
+# FFLAGS      Fortran 77 compiler flags
+# LD          linker
+# LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries
+# FPPFLAGS    pre-processing flags
+# AR          assembler
+# ARFLAGS     assembler flags
+# MK          make
+# USER_INC    complete list of include files
+# USER_LIB    complete list of libraries to pass to the linker
+# CC          C compiler used to compile conv for AGRIF
+# CFLAGS      compiler flags used with CC
+#
+# Note that:
+#  - unix variables "$..." are accpeted and will be evaluated before calling fcm.
+#  - fcm variables are starting with a % (and not a $)
+#
+#---------------------------------------------------------------------------------------------
+#---------------------------------------------------------------------------------------------
+# All NETCDF and HDF paths are empty as they are automatically defined through environment
+# variables by the load of modules
+#---------------------------------------------------------------------------------------------
+#---------------------------------------------------------------------------------------------
+#
+#
+%NCDF_INC            ${NETCDF4_INCLUDE}
+%NCDF_LIB            ${NETCDF4_LIB} -L${HDF5_DIR}/lib -Wl,-rpath,${HDF5_DIR}/lib -lhdf5_hl -lhdf5 -lz
+%XIOS_INC            -I${XIOS_INC}
+%XIOS_LIB            -L${XIOS_LIB} -lxios -lstdc++
+%OASIS_INC           -I${OASIS_DIR}/build/lib/mct -I${OASIS_DIR}/build/lib/psmile.MPI1
+%OASIS_LIB           -L${OASIS_DIR}/lib -lpsmile.MPI1 -lmct -lmpeu -lscrip
+
+%CPP                 cpp
+%FC                  mpifort -c -cpp
+%FCFLAGS             -march=core-avx2 -i4 -r8 -g -O0 -debug all -traceback -fp-model strict -ftrapuv -check all,noarg_temp_created -fpe-all0 -ftz -init=arrays,snan,huge
+%FFLAGS              %FCFLAGS
+%LD                  mpifort
+%LDFLAGS             
+%FPPFLAGS            -P -traditional
+%AR                  ar
+%ARFLAGS             rs
+%MK                  gmake
+%USER_INC            %XIOS_INC %OASIS_INC %NCDF_INC
+%USER_LIB            %XIOS_LIB %OASIS_LIB %NCDF_LIB
+
+%CC                  mpicc
+%CFLAGS              -O0

sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI

+#!/usr/bin/env bash
+
+#SBATCH -J sette
+#SBATCH -o sette.%j.out
+#SBATCH -e sette.%j.err
+#SBATCH -N 1
+#SBATCH --time=0-00:20:00
+#SBATCH --account=frtomerc
+#SBATCH --partition=par
+#SBATCH --qos=np
+##SBATCH --ntasks=NPROCS
+#SBATCH --ntasks=TOTAL_NPROCS
+
+set -x
+
+# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
+# (via sed operating on this template job file). 
+#
+  echo " ";
+  OCORES=NPROCS
+  XCORES=NXIOPROCS
+  O_PER_NODE=32
+  X_PER_NODE=8
+  if [ $XCORES -le $X_PER_NODE ]; then X_PER_NODE=$XCORES; fi
+  if [ $OCORES -le $O_PER_NODE ]; then O_PER_NODE=$OCORES; fi
+  export SETTE_DIR=DEF_SETTE_DIR
+
+###############################################################
+#
+export MPIRUN="srun"
+#
+# load sette functions (only post_test_tidyup needed)
+#
+  . ${SETTE_DIR}/all_functions.sh
+###############################################################
+#
+# modules to load
+module purge &> /dev/null
+module use /home/ar5/modules
+module load prgenv/intel intel/2021.4.0 openmpi/4.1.1.1 hdf5-parallel/1.10.6 netcdf4-parallel/4.7.4 xios/trunk/r2320_intel-ompi
+module list
+
+# Don't remove neither change the following line
+# BODY
+#
+# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
+# (via sed operating on this template job file). Note that the number of compute nodes required
+# is also set by the fcm_job.sh on the PBS select header line above.
+#
+# These variables are needed by post_test_tidyup function in all_functions.sh
+#
+  export XIO_HOME=${XIOS_DIR}
+  export INPUT_DIR=DEF_INPUT_DIR
+  export CONFIG_DIR=DEF_CONFIG_DIR
+  export TOOLS_DIR=DEF_TOOLS_DIR
+  export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION
+  export NEW_CONF=DEF_NEW_CONF
+  export CMP_NAM=DEF_CMP_NAM
+  export TEST_NAME=DEF_TEST_NAME
+  export EXE_DIR=DEF_EXE_DIR
+  ulimit -a
+  ulimit -s unlimited
+#
+# end of set up
+###############################################################
+#
+# change to the working directory 
+#
+  cd ${EXE_DIR}
+  echo Running on host `hostname`
+  echo Time is `date`
+  echo Directory is `pwd`
+  sed -i "s/one_file/multiple_file/" file_def_nemo-*.xml
+# 
+#  Run the parallel MPI executable 
+#
+  echo "Running time ${MPIRUN} ./nemo"
+#
+  if [ $XCORES -gt 0 ]; then
+#
+#  Run MPMD case
+#
+     #XIOS will run on a separate node so will run in parallel queue
+     if [ ! -f ./xios_server.exe ] && [ -f ${XIO_HOME}/bin/xios_server.exe ]; then
+        cp ${XIO_HOME}/bin/xios_server.exe .
+     fi
+     if [ ! -f ./xios_server.exe ]; then
+        echo "./xios_server.exe not found"
+        echo "run aborted"
+        exit
+     fi
+      cat > ./config.file <<-EOF
+0-$((OCORES-1)) ./nemo
+${OCORES}-$((TOTAL_NPROCS-1)) ./xios_server.exe
+EOF
+     $MPIRUN --ntasks=TOTAL_NPROCS --multi-prog config.file
+  else
+#
+# Run SPMD case
+#
+    time ./nemo
+  fi
+#
+
+#
+  post_test_tidyup
+
+# END_BODY
+# Don't remove neither change the previous line
+  exit

sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI_DEBUG

+batch-X64_AA_INTEL_OMPI
\ No newline at end of file

sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI

+batch-X64_AA_INTEL_OMPI
\ No newline at end of file

sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI_DEBUG

+batch-X64_AA_INTEL_OMPI_DEBUG
\ No newline at end of file