Skip to content
Snippets Groups Projects
Commit aa6da8e4 authored by Andrew Coward's avatar Andrew Coward
Browse files

Merge branch '127-generic-configuration-coverage-in-sette-test-reports' into 'branch_4.2' 

Resolve "Generic configuration coverage in SETTE test reports"

See merge request nemo/nemo!196
parents 1534fee2 18ab1e03
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
sette/param.default
+ 2
− 1
View file @ aa6da8e4
......@@ -65,8 +65,9 @@ LC_MESSAGES=en_US
#
# LIST OF CONFIGURATIONS
# to be updated if you added an new configuration in sette_test-cases.sh or sette_reference-configuration.sh
TEST_CONFIG_AVAILABLE=(ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 AGRIF_DEMO WED025 GYRE_PISCES ORCA2_SAS_ICE ORCA2_ICE_OBS SWG ICE_AGRIF OVERFLOW LOCK_EXCHANGE VORTEX ISOMIP+)
if [ -z "${TEST_CONFIGS}" ]; then
export TEST_CONFIGS=(${SETTE_TEST_CONFIGS[@]:-"ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 AGRIF WED025 GYRE_PISCES SAS ORCA2_ICE_OBS SWG ICE_AGRIF OVERFLOW LOCK_EXCHANGE VORTEX ISOMIP+"})
export TEST_CONFIGS=(${SETTE_TEST_CONFIGS[@]:-${TEST_CONFIG_AVAILABLE[@]}})
fi
#
# TYPES OF TESTS TO PERFORM
......
sette/sette.sh
+ 2
− 0
View file @ aa6da8e4
......@@ -253,6 +253,8 @@ if [ ! -d $NEMO_VALIDATION_DIR/$SETTE_SUB_VAL ] && [ ${dry_run} -eq 0 ] ; then
fi
export NEMO_VALIDATION_DIR=$NEMO_VALIDATION_DIR/$SETTE_SUB_VAL
TEST_CONFIGS="${TEST_CONFIGS/ORCA2_SAS_ICE/SAS}" # Shortening of 'ORCA2_SAS_ICE' to 'SAS'
TEST_CONFIGS="${TEST_CONFIGS/AGRIF_DEMO/AGRIF}" # Shortening of 'AGRIF_DEMO' to 'AGRIF'
if [ ${#SETTE_TEST_CONFIGS[@]} -eq 0 ]; then
echo "=================================="
echo "Configurations $TEST_CONFIGS will be tested if they are available"
......
sette/sette_eval.sh
+ 4
− 2
View file @ aa6da8e4
......@@ -155,6 +155,9 @@ function runcmpres(){
MAIN_DIR=$(dirname $SETTE_DIR)
quiet=0
. ./param.cfg
TEST_CONFIGS_AVAILABLE=${TEST_CONFIGS_AVAILABLE[@]:-${TEST_CONFIGS[@]}} # workaround for some dated param.cfgs files
TEST_CONFIGS_AVAILABLE=${TEST_CONFIGS_AVAILABLE[@]/ SAS / ORCA2_SAS_ICE } # workaround for some dated param.cfgs files
TEST_CONFIGS_AVAILABLE=${TEST_CONFIGS_AVAILABLE[@]/ AGRIF / AGRIF_DEMO } # workaround for some dated param.cfgs files
USER_INPUT='yes' # Default: yes => request user input on decisions.
mach=${COMPILER}
......@@ -327,8 +330,7 @@ fi
echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF"
echo ''
fi
checklist=(GYRE_PISCES ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 ORCA2_SAS_ICE ORCA2_ICE_OBS AGRIF_DEMO WED025 ISOMIP+ VORTEX ICE_AGRIF OVERFLOW LOCK_EXCHANGE SWG)
for repro_test in ${checklist[@]}
for repro_test in ${TEST_CONFIGS_AVAILABLE[@]}
do
runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $quiet
done
......
sette/sette_rpt.sh
+ 24
− 18
View file @ aa6da8e4
......@@ -267,10 +267,12 @@ function runcmpres(){
#
if [ -d $vdir/$mach/$dorv/$nam ]; then
f1s=$vdir/$mach/$dorv/$nam/LONG/run.stat
f1t=$vdir/$mach/$dorv/$nam/LONG/tracer.stat
f2s=$vdirref/$mach/$dorvref/$nam/LONG/run.stat
f2t=$vdirref/$mach/$dorvref/$nam/LONG/tracer.stat
# Selection of the test run used for the comparison (LONG or one of the reproducibility-test runs)
TESTD=$(ls -1 ${vdir}/${mach}/${dorv}/${nam}/ | grep -m 1 -e '^LONG$' -e '^REPRO_'); TESTD=${TESTD:-LONG}
f1s=$vdir/$mach/$dorv/${nam}/${TESTD}/run.stat
f1t=$vdir/$mach/$dorv/${nam}/${TESTD}/tracer.stat
f2s=$vdirref/$mach/$dorvref/${nam}/${TESTD}/run.stat
f2t=$vdirref/$mach/$dorvref/${nam}/${TESTD}/tracer.stat
if [ ! -f $f1s ] && [ ! -f $f1t ] ; then
printf "%-20s %s\n" $nam " incomplete test";
return;
......@@ -286,11 +288,11 @@ function runcmpres(){
cmp -s $f1s $f2s
if [ $? == 0 ]; then
if [ $pass == 0 ]; then
printf "%-20s %s %s\n" $nam " run.stat files are identical "
printf "%-20s %s (%s)\n" $nam " run.stat files are identical " ${TESTD}
fi
else
get_ktdiff $f1s $f2s
printf "%-20s %s %s %-5s %s\n" $nam " run.stat files are DIFFERENT (results are different after " $ktdiff " time steps)"
printf "%-20s %s %s %-5s (%s)\n" $nam " run.stat files are DIFFERENT (results are different after " $ktdiff " time steps) " ${TESTD}
#
# Offer view of differences on the second pass
#
......@@ -310,11 +312,11 @@ function runcmpres(){
cmp -s $f1t $f2t
if [ $? == 0 ]; then
if [ $pass == 0 ]; then
printf "%-20s %s %s\n" $nam " tracer.stat files are identical "
printf "%-20s %s (%s)\n" $nam " tracer.stat files are identical " ${TESTD}
fi
else
get_ktdiff2 $f1t $f2t
printf "%-20s %s %s %-5s %s\n" $nam " tracer.stat files are DIFFERENT (results are different after " $ktdiff " time steps) "
printf "%-20s %s %s %-5s (%s)\n" $nam " tracer.stat files are DIFFERENT (results are different after " $ktdiff " time steps) " ${TESTD}
#
# Offer view of differences on the second pass
#
......@@ -351,8 +353,10 @@ function runcmptim(){
#
if [ -d $vdir/$mach/$dorv/$nam ]; then
f1a=$vdir/$mach/$dorv/$nam/LONG/timing.output
f2a=$vdirref/$mach/$dorvref/$nam/LONG/timing.output
# Selection of the test run used for the comparison (LONG or one of the reproducibility-test runs)
TESTD=$(ls -1 ${vdir}/${mach}/${dorv}/${nam}/ | grep -m 1 -e '^LONG$' -e '^REPRO_'); TESTD=${TESTD:-LONG}
f1a=$vdir/$mach/$dorv/${nam}/${TESTD}/timing.output
f2a=$vdirref/$mach/$dorvref/${nam}/${TESTD}/timing.output
#
# Report average CPU time differences (if available)
#
......@@ -471,6 +475,9 @@ function identictest(){
SETTE_DIR=$(cd $(dirname "$0"); pwd)
MAIN_DIR=$(dirname $SETTE_DIR)
. ./param.cfg
TEST_CONFIGS_AVAILABLE=${TEST_CONFIGS_AVAILABLE[@]:-${TEST_CONFIGS[@]}} # Workaround for some dated param.cfgs files
TEST_CONFIGS_AVAILABLE=${TEST_CONFIGS_AVAILABLE[@]/ SAS / ORCA2_SAS_ICE } # Workaround for some dated param.cfgs files
TEST_CONFIGS_AVAILABLE=${TEST_CONFIGS_AVAILABLE[@]/ AGRIF / AGRIF_DEMO } # Workaround for some dated param.cfgs files
if [ -z $USER_INPUT ] ; then USER_INPUT='yes' ; fi # Default: yes => request user input on decisions.
# (but may br inherited/imported from sette.sh)
......@@ -634,17 +641,17 @@ do
# Restartability test
echo ""
echo " !----restart----! "
for restart_test in GYRE_PISCES ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 ORCA2_SAS_ICE AGRIF_DEMO WED025 ISOMIP+ OVERFLOW LOCK_EXCHANGE VORTEX ICE_AGRIF SWG
for restart_test in ${TEST_CONFIGS_AVAILABLE[@]}
do
resttest $NEMO_VALID $restart_test $pass
[ "${restart_test}" != "ORCA2_ICE_OBS" ] && resttest $NEMO_VALID $restart_test $pass
done
#
# Reproducibility tests
echo ""
echo " !----repro----! "
for repro_test in GYRE_PISCES ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 ORCA2_SAS_ICE ORCA2_ICE_OBS AGRIF_DEMO WED025 ISOMIP+ VORTEX ICE_AGRIF SWG
for repro_test in ${TEST_CONFIGS_AVAILABLE[@]}
do
reprotest $NEMO_VALID $repro_test $pass
[ "${repro_test}" != "OVERFLOW" -a "${repro_test}" != "LOCK_EXCHANGE" ] && reprotest $NEMO_VALID $repro_test $pass
done
# AGRIF special check to ensure results are unchanged with and without key_agrif
......@@ -669,14 +676,13 @@ do
echo 'and'
echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF"
echo ''
checklist=(GYRE_PISCES ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 ORCA2_SAS_ICE AGRIF_DEMO WED025 ISOMIP+ VORTEX ICE_AGRIF OVERFLOW LOCK_EXCHANGE SWG)
for repro_test in ${checklist[@]}
for runcmp_test in ${TEST_CONFIGS_AVAILABLE[@]}
do
runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass
runcmpres $NEMO_VALID $runcmp_test $NEMO_VALID_REF $NEMO_REV_REF $pass
done
echo ''
echo 'Report timing differences between REFERENCE and VALID (if available) :'
for repro_test in ${checklist[@]}
for repro_test in ${TEST_CONFIGS_AVAILABLE[@]}
do
runcmptim $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass
done
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment