diff --git a/arch/MERCATOR/arch-X64_AA_INTEL_OMPI.fcm b/arch/MERCATOR/arch-X64_AA_INTEL_OMPI.fcm new file mode 100644 index 0000000000000000000000000000000000000000..2e8a3139941ccc5df7a6d8812d3df486dfb0acb3 --- /dev/null +++ b/arch/MERCATOR/arch-X64_AA_INTEL_OMPI.fcm @@ -0,0 +1,62 @@ +# compiler options for AA (using INTEL compiler & OpenMPI) +# +# module purge +# module use /home/ar0s/modules +# module load prgenv/intel intel/2021.4.0 openmpi/4.1.1.1 hdf5-parallel/1.10.6 netcdf4-parallel/4.7.4 xios/trunk/rev2320-nmpi +# +# +# NCDF_INC netcdf4 include file +# NCDF_LIB netcdf4 library +# XIOS_INC xios include file (taken into accound only if key_xios is activated) +# XIOS_LIB xios library (taken into accound only if key_xios is activated) +# OASIS_INC oasis include file (taken into accound only if key_oasis3 is activated) +# OASIS_LIB oasis library (taken into accound only if key_oasis3 is activated) +# +# FC Fortran compiler command +# FCFLAGS Fortran compiler flags +# FFLAGS Fortran 77 compiler flags +# LD linker +# LDFLAGS linker flags, e.g. -L<lib dir> if you have libraries +# FPPFLAGS pre-processing flags +# AR assembler +# ARFLAGS assembler flags +# MK make +# USER_INC complete list of include files +# USER_LIB complete list of libraries to pass to the linker +# CC C compiler used to compile conv for AGRIF +# CFLAGS compiler flags used with CC +# +# Note that: +# - unix variables "$..." are accpeted and will be evaluated before calling fcm. +# - fcm variables are starting with a % (and not a $) +# +#--------------------------------------------------------------------------------------------- +#--------------------------------------------------------------------------------------------- +# All NETCDF and HDF paths are empty as they are automatically defined through environment +# variables by the load of modules +#--------------------------------------------------------------------------------------------- +#--------------------------------------------------------------------------------------------- +# +# +%NCDF_INC ${NETCDF4_INCLUDE} +%NCDF_LIB ${NETCDF4_LIB} -L${HDF5_DIR}/lib -Wl,-rpath,${HDF5_DIR}/lib -lhdf5_hl -lhdf5 -lz +%XIOS_INC -I${XIOS_INC} +%XIOS_LIB -L${XIOS_LIB} -lxios -lstdc++ +%OASIS_INC -I${OASIS_DIR}/build/lib/mct -I${OASIS_DIR}/build/lib/psmile.MPI1 +%OASIS_LIB -L${OASIS_DIR}/lib -lpsmile.MPI1 -lmct -lmpeu -lscrip + +%CPP cpp +%FC mpifort -c -cpp +%FCFLAGS -march=core-avx2 -i4 -r8 -O3 -fp-model strict -fno-alias -align array64byte +%FFLAGS %FCFLAGS +%LD mpifort +%LDFLAGS +%FPPFLAGS -P -traditional +%AR ar +%ARFLAGS rs +%MK gmake +%USER_INC %XIOS_INC %OASIS_INC %NCDF_INC +%USER_LIB %XIOS_LIB %OASIS_LIB %NCDF_LIB + +%CC mpicc +%CFLAGS -O0 diff --git a/arch/MERCATOR/arch-X64_AA_INTEL_OMPI_DEBUG.fcm b/arch/MERCATOR/arch-X64_AA_INTEL_OMPI_DEBUG.fcm new file mode 100644 index 0000000000000000000000000000000000000000..dd0994798d826bada2dced97f0a2527dea176e61 --- /dev/null +++ b/arch/MERCATOR/arch-X64_AA_INTEL_OMPI_DEBUG.fcm @@ -0,0 +1,62 @@ +# compiler options for AA (using INTEL compiler & OpenMPI) +# +# module purge +# module use /home/ar0s/modules +# module load prgenv/intel intel/2021.4.0 openmpi/4.1.1.1 hdf5-parallel/1.10.6 netcdf4-parallel/4.7.4 xios/trunk/rev2320-nmpi +# +# +# NCDF_INC netcdf4 include file +# NCDF_LIB netcdf4 library +# XIOS_INC xios include file (taken into accound only if key_xios is activated) +# XIOS_LIB xios library (taken into accound only if key_xios is activated) +# OASIS_INC oasis include file (taken into accound only if key_oasis3 is activated) +# OASIS_LIB oasis library (taken into accound only if key_oasis3 is activated) +# +# FC Fortran compiler command +# FCFLAGS Fortran compiler flags +# FFLAGS Fortran 77 compiler flags +# LD linker +# LDFLAGS linker flags, e.g. -L<lib dir> if you have libraries +# FPPFLAGS pre-processing flags +# AR assembler +# ARFLAGS assembler flags +# MK make +# USER_INC complete list of include files +# USER_LIB complete list of libraries to pass to the linker +# CC C compiler used to compile conv for AGRIF +# CFLAGS compiler flags used with CC +# +# Note that: +# - unix variables "$..." are accpeted and will be evaluated before calling fcm. +# - fcm variables are starting with a % (and not a $) +# +#--------------------------------------------------------------------------------------------- +#--------------------------------------------------------------------------------------------- +# All NETCDF and HDF paths are empty as they are automatically defined through environment +# variables by the load of modules +#--------------------------------------------------------------------------------------------- +#--------------------------------------------------------------------------------------------- +# +# +%NCDF_INC ${NETCDF4_INCLUDE} +%NCDF_LIB ${NETCDF4_LIB} -L${HDF5_DIR}/lib -Wl,-rpath,${HDF5_DIR}/lib -lhdf5_hl -lhdf5 -lz +%XIOS_INC -I${XIOS_INC} +%XIOS_LIB -L${XIOS_LIB} -lxios -lstdc++ +%OASIS_INC -I${OASIS_DIR}/build/lib/mct -I${OASIS_DIR}/build/lib/psmile.MPI1 +%OASIS_LIB -L${OASIS_DIR}/lib -lpsmile.MPI1 -lmct -lmpeu -lscrip + +%CPP cpp +%FC mpifort -c -cpp +%FCFLAGS -march=core-avx2 -i4 -r8 -g -O0 -debug all -traceback -fp-model strict -ftrapuv -check all,noarg_temp_created -fpe-all0 -ftz -init=arrays,snan,huge +%FFLAGS %FCFLAGS +%LD mpifort +%LDFLAGS +%FPPFLAGS -P -traditional +%AR ar +%ARFLAGS rs +%MK gmake +%USER_INC %XIOS_INC %OASIS_INC %NCDF_INC +%USER_LIB %XIOS_LIB %OASIS_LIB %NCDF_LIB + +%CC mpicc +%CFLAGS -O0 diff --git a/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI b/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI new file mode 100644 index 0000000000000000000000000000000000000000..bdd075c6c25cfc14359dcc9eced3b355eed74968 --- /dev/null +++ b/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +#SBATCH -J sette +#SBATCH -o sette.%j.out +#SBATCH -e sette.%j.err +#SBATCH -N 1 +#SBATCH --time=0-00:20:00 +#SBATCH --account=frtomerc +#SBATCH --partition=par +#SBATCH --qos=np +##SBATCH --ntasks=NPROCS +#SBATCH --ntasks=TOTAL_NPROCS + +set -x + +# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these +# (via sed operating on this template job file). +# + echo " "; + OCORES=NPROCS + XCORES=NXIOPROCS + O_PER_NODE=32 + X_PER_NODE=8 + if [ $XCORES -le $X_PER_NODE ]; then X_PER_NODE=$XCORES; fi + if [ $OCORES -le $O_PER_NODE ]; then O_PER_NODE=$OCORES; fi + export SETTE_DIR=DEF_SETTE_DIR + +############################################################### +# +export MPIRUN="srun" +# +# load sette functions (only post_test_tidyup needed) +# + . ${SETTE_DIR}/all_functions.sh +############################################################### +# +# modules to load +module purge &> /dev/null +module use /home/ar5/modules +module load prgenv/intel intel/2021.4.0 openmpi/4.1.1.1 hdf5-parallel/1.10.6 netcdf4-parallel/4.7.4 xios/trunk/r2320_intel-ompi +module list + +# Don't remove neither change the following line +# BODY +# +# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these +# (via sed operating on this template job file). Note that the number of compute nodes required +# is also set by the fcm_job.sh on the PBS select header line above. +# +# These variables are needed by post_test_tidyup function in all_functions.sh +# + export XIO_HOME=${XIOS_DIR} + export INPUT_DIR=DEF_INPUT_DIR + export CONFIG_DIR=DEF_CONFIG_DIR + export TOOLS_DIR=DEF_TOOLS_DIR + export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION + export NEW_CONF=DEF_NEW_CONF + export CMP_NAM=DEF_CMP_NAM + export TEST_NAME=DEF_TEST_NAME + export EXE_DIR=DEF_EXE_DIR + ulimit -a + ulimit -s unlimited +# +# end of set up +############################################################### +# +# change to the working directory +# + cd ${EXE_DIR} + echo Running on host `hostname` + echo Time is `date` + echo Directory is `pwd` + sed -i "s/one_file/multiple_file/" file_def_nemo-*.xml +# +# Run the parallel MPI executable +# + echo "Running time ${MPIRUN} ./nemo" +# + if [ $XCORES -gt 0 ]; then +# +# Run MPMD case +# + #XIOS will run on a separate node so will run in parallel queue + if [ ! -f ./xios_server.exe ] && [ -f ${XIO_HOME}/bin/xios_server.exe ]; then + cp ${XIO_HOME}/bin/xios_server.exe . + fi + if [ ! -f ./xios_server.exe ]; then + echo "./xios_server.exe not found" + echo "run aborted" + exit + fi + cat > ./config.file <<-EOF +0-$((OCORES-1)) ./nemo +${OCORES}-$((TOTAL_NPROCS-1)) ./xios_server.exe +EOF + $MPIRUN --ntasks=TOTAL_NPROCS --multi-prog config.file + else +# +# Run SPMD case +# + time ./nemo + fi +# + +# + post_test_tidyup + +# END_BODY +# Don't remove neither change the previous line + exit diff --git a/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI_DEBUG b/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI_DEBUG new file mode 120000 index 0000000000000000000000000000000000000000..11d73e632c3fff46278fd7367d64fc4e216659e4 --- /dev/null +++ b/sette/BATCH_TEMPLATE/batch-X64_AA_INTEL_OMPI_DEBUG @@ -0,0 +1 @@ +batch-X64_AA_INTEL_OMPI \ No newline at end of file diff --git a/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI b/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI new file mode 120000 index 0000000000000000000000000000000000000000..11d73e632c3fff46278fd7367d64fc4e216659e4 --- /dev/null +++ b/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI @@ -0,0 +1 @@ +batch-X64_AA_INTEL_OMPI \ No newline at end of file diff --git a/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI_DEBUG b/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI_DEBUG new file mode 120000 index 0000000000000000000000000000000000000000..178a7f6eabe69471382e8fb0df6d9c58be6d6fd3 --- /dev/null +++ b/sette/BATCH_TEMPLATE/batch-mpmd-X64_AA_INTEL_OMPI_DEBUG @@ -0,0 +1 @@ +batch-X64_AA_INTEL_OMPI_DEBUG \ No newline at end of file