Merge branch '347-update-arch-files-for-cmcc-cluster-2' into 'branch_4.2'

Resolve "Update arch files for cmcc cluster for v4.2"

See merge request nemo/nemo!426

arch/CMCC/arch-X64_JUNO.env

+
+module purge
+
+module load intel-2021.6.0/cmake/3.25.1-7wfsx
+module load oneapi-2022.1.0/compiler-rt/2022.1.0
+module load intel-2021.6.0/2021.6.0
+module load impi-2021.6.0/2021.6.0
+module load intel-2021.6.0/impi-2021.6.0/hdf5-threadsafe/1.13.3-zbgha
+module load intel-2021.6.0/impi-2021.6.0/netcdf-c-threadsafe/4.9.0-wpe4t
+module load intel-2021.6.0/impi-2021.6.0/netcdf-fortran-threadsafe/4.6.0-75oow
+module load intel-2021.6.0/impi-2021.6.0/parallel-netcdf/1.12.3-eshb5
+module load intel-2021.6.0/perl/5.36.0-jj4hw
+module load intel-2021.6.0/perl-uri/1.72-6at2i
+module load intel-2021.6.0/impi-2021.6.0/xios/2.5-36kwn
+
+# set linker path to 64-bit libraries
+export LD_LIBRARY_PATH="/lib64/":$LD_LIBRARY_PATH

arch/CMCC/arch-X64_JUNO.fcm

@@ -26,29 +26,29 @@
 #  - fcm variables are starting with a % (and not a $)
 #
 
-# NETCDF, PNETCDF, HDF5  should be set automatically when loading modules on ZEUS cluster
-# User should set the path of xios compiled code (after r2134)
-#export XIOS=<path-to-xios>/xios
-# and load the same gcc version used to compile xios (e.g., module load gcc_9.1.0/9.1.0)
+# Environment variables are set automatically when loading modules on JUNO cluster (see arch-X64_JUNO.env)
 
-%NCDF_INC            -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include
+%NCDF_INC            -I${NETCDF_FORTRAN}/include -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include
 %NCDF_LIB            -L${NETCDF_FORTRAN}/lib -lnetcdff -L${NETCDF_C}/lib -lnetcdf -L${PARALLEL_NETCDF}/lib -lpnetcdf
 %HDF5_INC            -I${HDF5}/include
 %HDF5_LIB            -L${HDF5}/lib -lhdf5_hl -lhdf5
 %XIOS_INC            -I${XIOS}/inc
 %XIOS_LIB            -L${XIOS}/lib -lxios
 
-%CPP                 cpp
+%USER_INC            %XIOS_INC %NCDF_INC %HDF5_INC
+%USER_LIB            %XIOS_LIB %NCDF_LIB %HDF5_LIB
+
 %FC                  mpiifort
-%FCFLAGS             -r8 -O3 -fp-model source -traceback -mkl=cluster -xSKYLAKE-AVX512 -qopt-zmm-usage=low -no-fma
+%FCFLAGS             -r8 -O3 -fp-model source -traceback -qmkl=cluster -march=icelake-client -mtune=icelake-client -qopt-zmm-usage=low -no-fma
 %FFLAGS              %FCFLAGS
+%CC                  mpiicc
+%CFLAGS              -O0
+
 %LD                  mpiifort
-%FPPFLAGS            -P -traditional
-%LDFLAGS             -lstdc++ -lz -lgpfs
+%LDFLAGS             -lstdc++ -lgpfs
 %AR                  ar 
 %ARFLAGS             -r
+
+%CPP                 icc
+%FPPFLAGS            -E -P -traditional
 %MK                  gmake
-%USER_INC            %XIOS_INC %NCDF_INC %HDF5_INC
-%USER_LIB            %XIOS_LIB %NCDF_LIB %HDF5_LIB
-%CC                  icc
-%CFLAGS              -O0

arch/CMCC/arch-X64_JUNO_DEBUG.env

+arch-X64_JUNO.env
\ No newline at end of file

arch/CMCC/arch-X64_JUNO_DEBUG.fcm

@@ -26,29 +26,29 @@
 #  - fcm variables are starting with a % (and not a $)
 #
 
-# NETCDF, PNETCDF, HDF5  should be set automatically when loading modules on ZEUS cluster
-# User should set the path of xios compiled code (after r2134)
-#export XIOS=<path-to-xios>/xios
-# and load the same gcc version used to compile xios (e.g., module load gcc_9.1.0/9.1.0)
+# Environment variables are set automatically when loading modules on JUNO cluster (see arch-X64_JUNO.env)
 
-%NCDF_INC            -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include
+%NCDF_INC            -I${NETCDF_FORTRAN}/include -I${NETCDF_C}/include -I${PARALLEL_NETCDF}/include
 %NCDF_LIB            -L${NETCDF_FORTRAN}/lib -lnetcdff -L${NETCDF_C}/lib -lnetcdf -L${PARALLEL_NETCDF}/lib -lpnetcdf
 %HDF5_INC            -I${HDF5}/include
 %HDF5_LIB            -L${HDF5}/lib -lhdf5_hl -lhdf5
 %XIOS_INC            -I${XIOS}/inc
 %XIOS_LIB            -L${XIOS}/lib -lxios
 
-%CPP                 cpp
+%USER_INC            %XIOS_INC %NCDF_INC %HDF5_INC
+%USER_LIB            %XIOS_LIB %NCDF_LIB %HDF5_LIB
+
 %FC                  mpiifort
-%FCFLAGS             -r8 -O0 -fp-model source -traceback -mkl=cluster -xSKYLAKE-AVX512 -qopt-zmm-usage=low -no-fma -g -debug all -traceback -check all
+%FCFLAGS             -r8 -g -O0 -check all -fp-model source -traceback -qmkl=cluster -march=icelake-client -mtune=icelake-client -qopt-zmm-usage=low -no-fma
 %FFLAGS              %FCFLAGS
+%CC                  mpiicc
+%CFLAGS              -O0
+
 %LD                  mpiifort
-%FPPFLAGS            -P -traditional
-%LDFLAGS             -lstdc++ -lz -lgpfs
+%LDFLAGS             -lstdc++ -lgpfs
 %AR                  ar 
 %ARFLAGS             -r
+
+%CPP                 icc
+%FPPFLAGS            -E -P -traditional
 %MK                  gmake
-%USER_INC            %XIOS_INC %NCDF_INC %HDF5_INC
-%USER_LIB            %XIOS_LIB %NCDF_LIB %HDF5_LIB
-%CC                  icc
-%CFLAGS              -O0

sette/BATCH_TEMPLATE/batch-X64_JUNO

+#!/bin/bash 
+#!
+#BSUB -q p_short
+#BSUB -n TOTAL_NPROCS
+#BSUB -J NEMO_SETTE
+#BSUB -o job_sette.out
+#BSUB -e job_sette.out
+#BSUB -P R000
+#BSUB -x
+
+###############################################################
+# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
+# (via sed operating on this template job file). 
+#
+  OCEANCORES=NPROCS
+  XIOS_NUMPROCS=NXIOPROCS
+  export SETTE_DIR=DEF_SETTE_DIR
+
+###############################################################
+#
+# load sette functions (only post_test_tidyup needed)
+#
+  . ${SETTE_DIR}/all_functions.sh
+
+
+# Don't remove neither change the following line 
+# BODY
+
+#
+# These variables are needed by post_test_tidyup function in all_functions.sh
+#
+  export EXE_DIR=DEF_EXE_DIR
+  export INPUT_DIR=DEF_INPUT_DIR
+  export CONFIG_DIR=DEF_CONFIG_DIR
+  export TOOLS_DIR=DEF_TOOLS_DIR
+  export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION
+  export NEW_CONF=DEF_NEW_CONF
+  export CMP_NAM=DEF_CMP_NAM
+  export TEST_NAME=DEF_TEST_NAME
+#
+# end of set up
+
+# Load environment if exists
+env_file=`find ${TOOLS_DIR}/../arch -name arch-${CMP_NAM}.env`
+if [ -f "${env_file}" ] ; then
+   echo "Load environment file arch-${CMP_NAM}.env"
+   . ${env_file}
+fi
+
+###############################################################
+
+# Local settings for CMCC cluster
+#
+export I_MPI_HYDRA_BRANCH_COUNT=`cat $LSB_DJOB_HOSTFILE | uniq  | wc -l`
+export MPIRUN="mpiexec.hydra"
+
+# local xios setting for MPMD
+export LD_LIBRARY_PATH=${XIOS}/lib:${LD_LIBRARY_PATH}
+XIOS_SERVER_PATHNAME="${XIOS}/bin/xios_server.exe"
+
+echo "Start JOBID ${LSB_JOBID}"
+
+###############################################################
+#
+# change to the working directory 
+#
+cd ${EXE_DIR}
+
+#
+  echo Running on host `hostname`
+  echo Time is `date`
+  echo Directory is `pwd`
+# 
+#  Run the parallel MPI executable 
+#
+  startTime=$(date +%s)
+  if [ MPI_FLAG == "yes" ]; then
+     if [ ${USING_MPMD} == "yes" ] && [ ${XIOS_NUMPROCS} -gt 0 ]; then
+        # XIOS detached mode
+        xioscmdfile="xioscmdfile"
+        #
+        echo "# Configuration file for mpiexec.hydra" > $xioscmdfile
+        echo "-n ${OCEANCORES} ./nemo" >> $xioscmdfile
+        echo "-n ${XIOS_NUMPROCS} ${XIOS_SERVER_PATHNAME}" >> $xioscmdfile
+
+        time ${MPIRUN} -configfile $xioscmdfile
+
+     else
+        # XIOS attached mode
+        time ${MPIRUN} ./nemo          
+     fi
+  else
+#  Run the serial executable
+     time ./nemo
+  fi
+  endTime=$(date +%s)
+  totalTime=$(($endTime-$startTime))
+  echo "Model finished after $totalTime seconds for test $TEST_NAME"
+#
+  post_test_tidyup
+
+# END_BODY
+# Don't remove neither change the previous line 
+
+  exit