From 000aebae8cedbbe754089033a9c67feded3899ea Mon Sep 17 00:00:00 2001 From: Tomas Lovato <tomas.lovato@cmcc.it> Date: Mon, 11 Dec 2023 19:36:13 +0000 Subject: [PATCH] Resolve "Update arch files for cmcc cluster for v4.2" --- arch/CMCC/arch-X64_JUNO.env | 17 +++++ arch/CMCC/arch-X64_JUNO.fcm | 26 +++---- arch/CMCC/arch-X64_JUNO_DEBUG.env | 1 + arch/CMCC/arch-X64_JUNO_DEBUG.fcm | 26 +++---- sette/BATCH_TEMPLATE/batch-X64_JUNO | 105 ++++++++++++++++++++++++++++ 5 files changed, 149 insertions(+), 26 deletions(-) create mode 100644 arch/CMCC/arch-X64_JUNO.env create mode 120000 arch/CMCC/arch-X64_JUNO_DEBUG.env create mode 100644 sette/BATCH_TEMPLATE/batch-X64_JUNO diff --git a/arch/CMCC/arch-X64_JUNO.env b/arch/CMCC/arch-X64_JUNO.env new file mode 100644 index 00000000..86ee44e8 --- /dev/null +++ b/arch/CMCC/arch-X64_JUNO.env @@ -0,0 +1,17 @@ + +module purge + +module load intel-2021.6.0/cmake/3.25.1-7wfsx +module load oneapi-2022.1.0/compiler-rt/2022.1.0 +module load intel-2021.6.0/2021.6.0 +module load impi-2021.6.0/2021.6.0 +module load intel-2021.6.0/impi-2021.6.0/hdf5-threadsafe/1.13.3-zbgha +module load intel-2021.6.0/impi-2021.6.0/netcdf-c-threadsafe/4.9.0-wpe4t +module load intel-2021.6.0/impi-2021.6.0/netcdf-fortran-threadsafe/4.6.0-75oow +module load intel-2021.6.0/impi-2021.6.0/parallel-netcdf/1.12.3-eshb5 +module load intel-2021.6.0/perl/5.36.0-jj4hw +module load intel-2021.6.0/perl-uri/1.72-6at2i +module load intel-2021.6.0/impi-2021.6.0/xios/2.5-36kwn + +# set linker path to 64-bit libraries +export LD_LIBRARY_PATH="/lib64/":$LD_LIBRARY_PATH diff --git a/arch/CMCC/arch-X64_JUNO.fcm b/arch/CMCC/arch-X64_JUNO.fcm index 35f0c89a..80c70e0d 100644 --- a/arch/CMCC/arch-X64_JUNO.fcm +++ b/arch/CMCC/arch-X64_JUNO.fcm @@ -26,29 +26,29 @@ # - fcm variables are starting with a % (and not a $) # -# NETCDF, PNETCDF, HDF5 should be set automatically when loading modules on ZEUS cluster -# User should set the path of xios compiled code (after r2134) -#export XIOS=<path-to-xios>/xios -# and load the same gcc version used to compile xios (e.g., module load gcc_9.1.0/9.1.0) +# Environment variables are set automatically when loading modules on JUNO cluster (see arch-X64_JUNO.env) -%NCDF_INC -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include +%NCDF_INC -I${NETCDF_FORTRAN}/include -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include %NCDF_LIB -L${NETCDF_FORTRAN}/lib -lnetcdff -L${NETCDF_C}/lib -lnetcdf -L${PARALLEL_NETCDF}/lib -lpnetcdf %HDF5_INC -I${HDF5}/include %HDF5_LIB -L${HDF5}/lib -lhdf5_hl -lhdf5 %XIOS_INC -I${XIOS}/inc %XIOS_LIB -L${XIOS}/lib -lxios -%CPP cpp +%USER_INC %XIOS_INC %NCDF_INC %HDF5_INC +%USER_LIB %XIOS_LIB %NCDF_LIB %HDF5_LIB + %FC mpiifort -%FCFLAGS -r8 -O3 -fp-model source -traceback -mkl=cluster -xSKYLAKE-AVX512 -qopt-zmm-usage=low -no-fma +%FCFLAGS -r8 -O3 -fp-model source -traceback -qmkl=cluster -march=icelake-client -mtune=icelake-client -qopt-zmm-usage=low -no-fma %FFLAGS %FCFLAGS +%CC mpiicc +%CFLAGS -O0 + %LD mpiifort -%FPPFLAGS -P -traditional -%LDFLAGS -lstdc++ -lz -lgpfs +%LDFLAGS -lstdc++ -lgpfs %AR ar %ARFLAGS -r + +%CPP icc +%FPPFLAGS -E -P -traditional %MK gmake -%USER_INC %XIOS_INC %NCDF_INC %HDF5_INC -%USER_LIB %XIOS_LIB %NCDF_LIB %HDF5_LIB -%CC icc -%CFLAGS -O0 diff --git a/arch/CMCC/arch-X64_JUNO_DEBUG.env b/arch/CMCC/arch-X64_JUNO_DEBUG.env new file mode 120000 index 00000000..94ca7224 --- /dev/null +++ b/arch/CMCC/arch-X64_JUNO_DEBUG.env @@ -0,0 +1 @@ +arch-X64_JUNO.env \ No newline at end of file diff --git a/arch/CMCC/arch-X64_JUNO_DEBUG.fcm b/arch/CMCC/arch-X64_JUNO_DEBUG.fcm index 34b4f2a8..50f906e1 100644 --- a/arch/CMCC/arch-X64_JUNO_DEBUG.fcm +++ b/arch/CMCC/arch-X64_JUNO_DEBUG.fcm @@ -26,29 +26,29 @@ # - fcm variables are starting with a % (and not a $) # -# NETCDF, PNETCDF, HDF5 should be set automatically when loading modules on ZEUS cluster -# User should set the path of xios compiled code (after r2134) -#export XIOS=<path-to-xios>/xios -# and load the same gcc version used to compile xios (e.g., module load gcc_9.1.0/9.1.0) +# Environment variables are set automatically when loading modules on JUNO cluster (see arch-X64_JUNO.env) -%NCDF_INC -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include +%NCDF_INC -I${NETCDF_FORTRAN}/include -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include %NCDF_LIB -L${NETCDF_FORTRAN}/lib -lnetcdff -L${NETCDF_C}/lib -lnetcdf -L${PARALLEL_NETCDF}/lib -lpnetcdf %HDF5_INC -I${HDF5}/include %HDF5_LIB -L${HDF5}/lib -lhdf5_hl -lhdf5 %XIOS_INC -I${XIOS}/inc %XIOS_LIB -L${XIOS}/lib -lxios -%CPP cpp +%USER_INC %XIOS_INC %NCDF_INC %HDF5_INC +%USER_LIB %XIOS_LIB %NCDF_LIB %HDF5_LIB + %FC mpiifort -%FCFLAGS -r8 -O0 -fp-model source -traceback -mkl=cluster -xSKYLAKE-AVX512 -qopt-zmm-usage=low -no-fma -g -debug all -traceback -check all +%FCFLAGS -r8 -g -O0 -check all -fp-model source -traceback -qmkl=cluster -march=icelake-client -mtune=icelake-client -qopt-zmm-usage=low -no-fma %FFLAGS %FCFLAGS +%CC mpiicc +%CFLAGS -O0 + %LD mpiifort -%FPPFLAGS -P -traditional -%LDFLAGS -lstdc++ -lz -lgpfs +%LDFLAGS -lstdc++ -lgpfs %AR ar %ARFLAGS -r + +%CPP icc +%FPPFLAGS -E -P -traditional %MK gmake -%USER_INC %XIOS_INC %NCDF_INC %HDF5_INC -%USER_LIB %XIOS_LIB %NCDF_LIB %HDF5_LIB -%CC icc -%CFLAGS -O0 diff --git a/sette/BATCH_TEMPLATE/batch-X64_JUNO b/sette/BATCH_TEMPLATE/batch-X64_JUNO new file mode 100644 index 00000000..3340116e --- /dev/null +++ b/sette/BATCH_TEMPLATE/batch-X64_JUNO @@ -0,0 +1,105 @@ +#!/bin/bash +#! +#BSUB -q p_short +#BSUB -n TOTAL_NPROCS +#BSUB -J NEMO_SETTE +#BSUB -o job_sette.out +#BSUB -e job_sette.out +#BSUB -P R000 +#BSUB -x + +############################################################### +# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these +# (via sed operating on this template job file). +# + OCEANCORES=NPROCS + XIOS_NUMPROCS=NXIOPROCS + export SETTE_DIR=DEF_SETTE_DIR + +############################################################### +# +# load sette functions (only post_test_tidyup needed) +# + . ${SETTE_DIR}/all_functions.sh + + +# Don't remove neither change the following line +# BODY + +# +# These variables are needed by post_test_tidyup function in all_functions.sh +# + export EXE_DIR=DEF_EXE_DIR + export INPUT_DIR=DEF_INPUT_DIR + export CONFIG_DIR=DEF_CONFIG_DIR + export TOOLS_DIR=DEF_TOOLS_DIR + export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION + export NEW_CONF=DEF_NEW_CONF + export CMP_NAM=DEF_CMP_NAM + export TEST_NAME=DEF_TEST_NAME +# +# end of set up + +# Load environment if exists +env_file=`find ${TOOLS_DIR}/../arch -name arch-${CMP_NAM}.env` +if [ -f "${env_file}" ] ; then + echo "Load environment file arch-${CMP_NAM}.env" + . ${env_file} +fi + +############################################################### + +# Local settings for CMCC cluster +# +export I_MPI_HYDRA_BRANCH_COUNT=`cat $LSB_DJOB_HOSTFILE | uniq | wc -l` +export MPIRUN="mpiexec.hydra" + +# local xios setting for MPMD +export LD_LIBRARY_PATH=${XIOS}/lib:${LD_LIBRARY_PATH} +XIOS_SERVER_PATHNAME="${XIOS}/bin/xios_server.exe" + +echo "Start JOBID ${LSB_JOBID}" + +############################################################### +# +# change to the working directory +# +cd ${EXE_DIR} + +# + echo Running on host `hostname` + echo Time is `date` + echo Directory is `pwd` +# +# Run the parallel MPI executable +# + startTime=$(date +%s) + if [ MPI_FLAG == "yes" ]; then + if [ ${USING_MPMD} == "yes" ] && [ ${XIOS_NUMPROCS} -gt 0 ]; then + # XIOS detached mode + xioscmdfile="xioscmdfile" + # + echo "# Configuration file for mpiexec.hydra" > $xioscmdfile + echo "-n ${OCEANCORES} ./nemo" >> $xioscmdfile + echo "-n ${XIOS_NUMPROCS} ${XIOS_SERVER_PATHNAME}" >> $xioscmdfile + + time ${MPIRUN} -configfile $xioscmdfile + + else + # XIOS attached mode + time ${MPIRUN} ./nemo + fi + else +# Run the serial executable + time ./nemo + fi + endTime=$(date +%s) + totalTime=$(($endTime-$startTime)) + echo "Model finished after $totalTime seconds for test $TEST_NAME" +# + post_test_tidyup + +# END_BODY +# Don't remove neither change the previous line + + exit -- GitLab