From 000aebae8cedbbe754089033a9c67feded3899ea Mon Sep 17 00:00:00 2001
From: Tomas Lovato <tomas.lovato@cmcc.it>
Date: Mon, 11 Dec 2023 19:36:13 +0000
Subject: [PATCH] Resolve "Update arch files for cmcc cluster for v4.2"
---
arch/CMCC/arch-X64_JUNO.env | 17 +++++
arch/CMCC/arch-X64_JUNO.fcm | 26 +++----
arch/CMCC/arch-X64_JUNO_DEBUG.env | 1 +
arch/CMCC/arch-X64_JUNO_DEBUG.fcm | 26 +++----
sette/BATCH_TEMPLATE/batch-X64_JUNO | 105 ++++++++++++++++++++++++++++
5 files changed, 149 insertions(+), 26 deletions(-)
create mode 100644 arch/CMCC/arch-X64_JUNO.env
create mode 120000 arch/CMCC/arch-X64_JUNO_DEBUG.env
create mode 100644 sette/BATCH_TEMPLATE/batch-X64_JUNO
diff --git a/arch/CMCC/arch-X64_JUNO.env b/arch/CMCC/arch-X64_JUNO.env
new file mode 100644
index 00000000..86ee44e8
--- /dev/null
+++ b/arch/CMCC/arch-X64_JUNO.env
@@ -0,0 +1,17 @@
+
+module purge
+
+module load intel-2021.6.0/cmake/3.25.1-7wfsx
+module load oneapi-2022.1.0/compiler-rt/2022.1.0
+module load intel-2021.6.0/2021.6.0
+module load impi-2021.6.0/2021.6.0
+module load intel-2021.6.0/impi-2021.6.0/hdf5-threadsafe/1.13.3-zbgha
+module load intel-2021.6.0/impi-2021.6.0/netcdf-c-threadsafe/4.9.0-wpe4t
+module load intel-2021.6.0/impi-2021.6.0/netcdf-fortran-threadsafe/4.6.0-75oow
+module load intel-2021.6.0/impi-2021.6.0/parallel-netcdf/1.12.3-eshb5
+module load intel-2021.6.0/perl/5.36.0-jj4hw
+module load intel-2021.6.0/perl-uri/1.72-6at2i
+module load intel-2021.6.0/impi-2021.6.0/xios/2.5-36kwn
+
+# set linker path to 64-bit libraries
+export LD_LIBRARY_PATH="/lib64/":$LD_LIBRARY_PATH
diff --git a/arch/CMCC/arch-X64_JUNO.fcm b/arch/CMCC/arch-X64_JUNO.fcm
index 35f0c89a..80c70e0d 100644
--- a/arch/CMCC/arch-X64_JUNO.fcm
+++ b/arch/CMCC/arch-X64_JUNO.fcm
@@ -26,29 +26,29 @@
# - fcm variables are starting with a % (and not a $)
#
-# NETCDF, PNETCDF, HDF5 should be set automatically when loading modules on ZEUS cluster
-# User should set the path of xios compiled code (after r2134)
-#export XIOS=<path-to-xios>/xios
-# and load the same gcc version used to compile xios (e.g., module load gcc_9.1.0/9.1.0)
+# Environment variables are set automatically when loading modules on JUNO cluster (see arch-X64_JUNO.env)
-%NCDF_INC -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include
+%NCDF_INC -I${NETCDF_FORTRAN}/include -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include
%NCDF_LIB -L${NETCDF_FORTRAN}/lib -lnetcdff -L${NETCDF_C}/lib -lnetcdf -L${PARALLEL_NETCDF}/lib -lpnetcdf
%HDF5_INC -I${HDF5}/include
%HDF5_LIB -L${HDF5}/lib -lhdf5_hl -lhdf5
%XIOS_INC -I${XIOS}/inc
%XIOS_LIB -L${XIOS}/lib -lxios
-%CPP cpp
+%USER_INC %XIOS_INC %NCDF_INC %HDF5_INC
+%USER_LIB %XIOS_LIB %NCDF_LIB %HDF5_LIB
+
%FC mpiifort
-%FCFLAGS -r8 -O3 -fp-model source -traceback -mkl=cluster -xSKYLAKE-AVX512 -qopt-zmm-usage=low -no-fma
+%FCFLAGS -r8 -O3 -fp-model source -traceback -qmkl=cluster -march=icelake-client -mtune=icelake-client -qopt-zmm-usage=low -no-fma
%FFLAGS %FCFLAGS
+%CC mpiicc
+%CFLAGS -O0
+
%LD mpiifort
-%FPPFLAGS -P -traditional
-%LDFLAGS -lstdc++ -lz -lgpfs
+%LDFLAGS -lstdc++ -lgpfs
%AR ar
%ARFLAGS -r
+
+%CPP icc
+%FPPFLAGS -E -P -traditional
%MK gmake
-%USER_INC %XIOS_INC %NCDF_INC %HDF5_INC
-%USER_LIB %XIOS_LIB %NCDF_LIB %HDF5_LIB
-%CC icc
-%CFLAGS -O0
diff --git a/arch/CMCC/arch-X64_JUNO_DEBUG.env b/arch/CMCC/arch-X64_JUNO_DEBUG.env
new file mode 120000
index 00000000..94ca7224
--- /dev/null
+++ b/arch/CMCC/arch-X64_JUNO_DEBUG.env
@@ -0,0 +1 @@
+arch-X64_JUNO.env
\ No newline at end of file
diff --git a/arch/CMCC/arch-X64_JUNO_DEBUG.fcm b/arch/CMCC/arch-X64_JUNO_DEBUG.fcm
index 34b4f2a8..50f906e1 100644
--- a/arch/CMCC/arch-X64_JUNO_DEBUG.fcm
+++ b/arch/CMCC/arch-X64_JUNO_DEBUG.fcm
@@ -26,29 +26,29 @@
# - fcm variables are starting with a % (and not a $)
#
-# NETCDF, PNETCDF, HDF5 should be set automatically when loading modules on ZEUS cluster
-# User should set the path of xios compiled code (after r2134)
-#export XIOS=<path-to-xios>/xios
-# and load the same gcc version used to compile xios (e.g., module load gcc_9.1.0/9.1.0)
+# Environment variables are set automatically when loading modules on JUNO cluster (see arch-X64_JUNO.env)
-%NCDF_INC -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include
+%NCDF_INC -I${NETCDF_FORTRAN}/include -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include
%NCDF_LIB -L${NETCDF_FORTRAN}/lib -lnetcdff -L${NETCDF_C}/lib -lnetcdf -L${PARALLEL_NETCDF}/lib -lpnetcdf
%HDF5_INC -I${HDF5}/include
%HDF5_LIB -L${HDF5}/lib -lhdf5_hl -lhdf5
%XIOS_INC -I${XIOS}/inc
%XIOS_LIB -L${XIOS}/lib -lxios
-%CPP cpp
+%USER_INC %XIOS_INC %NCDF_INC %HDF5_INC
+%USER_LIB %XIOS_LIB %NCDF_LIB %HDF5_LIB
+
%FC mpiifort
-%FCFLAGS -r8 -O0 -fp-model source -traceback -mkl=cluster -xSKYLAKE-AVX512 -qopt-zmm-usage=low -no-fma -g -debug all -traceback -check all
+%FCFLAGS -r8 -g -O0 -check all -fp-model source -traceback -qmkl=cluster -march=icelake-client -mtune=icelake-client -qopt-zmm-usage=low -no-fma
%FFLAGS %FCFLAGS
+%CC mpiicc
+%CFLAGS -O0
+
%LD mpiifort
-%FPPFLAGS -P -traditional
-%LDFLAGS -lstdc++ -lz -lgpfs
+%LDFLAGS -lstdc++ -lgpfs
%AR ar
%ARFLAGS -r
+
+%CPP icc
+%FPPFLAGS -E -P -traditional
%MK gmake
-%USER_INC %XIOS_INC %NCDF_INC %HDF5_INC
-%USER_LIB %XIOS_LIB %NCDF_LIB %HDF5_LIB
-%CC icc
-%CFLAGS -O0
diff --git a/sette/BATCH_TEMPLATE/batch-X64_JUNO b/sette/BATCH_TEMPLATE/batch-X64_JUNO
new file mode 100644
index 00000000..3340116e
--- /dev/null
+++ b/sette/BATCH_TEMPLATE/batch-X64_JUNO
@@ -0,0 +1,105 @@
+#!/bin/bash
+#!
+#BSUB -q p_short
+#BSUB -n TOTAL_NPROCS
+#BSUB -J NEMO_SETTE
+#BSUB -o job_sette.out
+#BSUB -e job_sette.out
+#BSUB -P R000
+#BSUB -x
+
+###############################################################
+# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
+# (via sed operating on this template job file).
+#
+ OCEANCORES=NPROCS
+ XIOS_NUMPROCS=NXIOPROCS
+ export SETTE_DIR=DEF_SETTE_DIR
+
+###############################################################
+#
+# load sette functions (only post_test_tidyup needed)
+#
+ . ${SETTE_DIR}/all_functions.sh
+
+
+# Don't remove neither change the following line
+# BODY
+
+#
+# These variables are needed by post_test_tidyup function in all_functions.sh
+#
+ export EXE_DIR=DEF_EXE_DIR
+ export INPUT_DIR=DEF_INPUT_DIR
+ export CONFIG_DIR=DEF_CONFIG_DIR
+ export TOOLS_DIR=DEF_TOOLS_DIR
+ export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION
+ export NEW_CONF=DEF_NEW_CONF
+ export CMP_NAM=DEF_CMP_NAM
+ export TEST_NAME=DEF_TEST_NAME
+#
+# end of set up
+
+# Load environment if exists
+env_file=`find ${TOOLS_DIR}/../arch -name arch-${CMP_NAM}.env`
+if [ -f "${env_file}" ] ; then
+ echo "Load environment file arch-${CMP_NAM}.env"
+ . ${env_file}
+fi
+
+###############################################################
+
+# Local settings for CMCC cluster
+#
+export I_MPI_HYDRA_BRANCH_COUNT=`cat $LSB_DJOB_HOSTFILE | uniq | wc -l`
+export MPIRUN="mpiexec.hydra"
+
+# local xios setting for MPMD
+export LD_LIBRARY_PATH=${XIOS}/lib:${LD_LIBRARY_PATH}
+XIOS_SERVER_PATHNAME="${XIOS}/bin/xios_server.exe"
+
+echo "Start JOBID ${LSB_JOBID}"
+
+###############################################################
+#
+# change to the working directory
+#
+cd ${EXE_DIR}
+
+#
+ echo Running on host `hostname`
+ echo Time is `date`
+ echo Directory is `pwd`
+#
+# Run the parallel MPI executable
+#
+ startTime=$(date +%s)
+ if [ MPI_FLAG == "yes" ]; then
+ if [ ${USING_MPMD} == "yes" ] && [ ${XIOS_NUMPROCS} -gt 0 ]; then
+ # XIOS detached mode
+ xioscmdfile="xioscmdfile"
+ #
+ echo "# Configuration file for mpiexec.hydra" > $xioscmdfile
+ echo "-n ${OCEANCORES} ./nemo" >> $xioscmdfile
+ echo "-n ${XIOS_NUMPROCS} ${XIOS_SERVER_PATHNAME}" >> $xioscmdfile
+
+ time ${MPIRUN} -configfile $xioscmdfile
+
+ else
+ # XIOS attached mode
+ time ${MPIRUN} ./nemo
+ fi
+ else
+# Run the serial executable
+ time ./nemo
+ fi
+ endTime=$(date +%s)
+ totalTime=$(($endTime-$startTime))
+ echo "Model finished after $totalTime seconds for test $TEST_NAME"
+#
+ post_test_tidyup
+
+# END_BODY
+# Don't remove neither change the previous line
+
+ exit
--
GitLab