From f4f8fd6a54cddb1c309f2977ac779aa96a282a25 Mon Sep 17 00:00:00 2001
From: Guillaume S <gsamson@mercator-ocean.fr>
Date: Fri, 19 Jan 2024 19:11:58 +0100
Subject: [PATCH] allow to treat only one variable
---
check_if_done.sh | 15 ++++++++++++---
full_prod.sh | 43 ++++++++++++++++++++++++++++++++++---------
job_ablvint.sbatch | 11 +++++++++--
job_cdogrid.sbatch | 1 +
job_cdomiss.sbatch | 1 +
job_mars.sbatch | 1 +
job_ncochunk.sbatch | 1 +
move_to_perm.sh | 15 +++++++++++----
subjob_ablvint.sh | 13 +++++++------
subjob_cdomiss.sh | 12 ++++++++++--
subjob_ncochunk.sh | 13 ++++++++++---
11 files changed, 97 insertions(+), 29 deletions(-)
diff --git a/check_if_done.sh b/check_if_done.sh
index 0b23068..e463dd0 100755
--- a/check_if_done.sh
+++ b/check_if_done.sh
@@ -6,13 +6,22 @@ date_beg=$1
date_end=$2
nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) ))
-var_list=( uwnd vwnd uhpg vhpg tpot humi )
+if [ -n "$3" ]; then
+ var_list=( $3 )
+else
+ var_list=( uwnd vwnd uhpg vhpg tpot humi )
+fi
for d in $(seq -f "%02g" 0 $((nbday-1))); do
date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d")
- fvar="ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc"
- dvar="${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}"
+ if [ ${#var_list[@]} -gt 1 ]; then
+ fvar="ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc"
+ dvar="${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}"
+ else
+ fvar="ERA5_L50Z10_${var_list[*]^^}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc"
+ dvar="${DIR_SAVE}/${var_list[*]^^}"
+ fi
file_nb=$(eval "ls -l ${dvar}/${fvar} 2>/dev/null" | wc -l )
tot_nb=$(( ${tot_nb:-0} + file_nb ))
diff --git a/full_prod.sh b/full_prod.sh
index 25ca21f..1577029 100755
--- a/full_prod.sh
+++ b/full_prod.sh
@@ -2,27 +2,52 @@
set +xv
+
+# args
date_beg=$1
-date_end=$2
+if [[ $2 =~ '^[0-9]+$' ]]; then
+ date_end=$2
+else
+ var_name=$2
+fi
+if [ -n "$3" ]; then
+ var_name=$3
+fi
+
+
+# date range
+if [ -z "${date_end}" ]; then
+ date_end=${date_beg}
+fi
if [[ ${#date_beg} -eq 6 || ${#date_end} -eq 6 ]]; then
date_beg=${date_beg:0:6}01
date_end=$(date -d "${date_beg} + 1 month - 1 day" +"%C%y%m%d" )
fi
-echo "${date_beg}->${date_end}"
+echo "${var_name:+"${var_name}: "}${date_beg}->${date_end}"
+
+
+# directories
export DIR_SRC="/home/ar0s/PROD_ERA5_ABL"
export DIR_SAVE="/ec/res5/project/mercator/ATMO_FORCINGS/ERA5_ABL50_F320_v1/daily/1h"
export SCRATCH_DIR=${SCRATCHDIR}
-./check_if_done.sh ${date_beg} ${date_end}; [ $? -eq 0 ] && exit 0
+
+# scripts
+./check_if_done.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 0 ] && exit 0
./subjob_mars.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
./subjob_cdogrid.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./subjob_ablhpg.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./subjob_ablvint.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./subjob_cdosplit.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./subjob_cdomiss.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./subjob_ncochunk.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./move_to_perm.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
+if [[ -z "${var_name}" || ${var_name} =~ "hpg" ]]; then
+ ./subjob_ablhpg.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
+fi
+./subjob_ablvint.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1
+if [ -z "${var_name}" ]; then
+ ./subjob_cdosplit.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
+fi
+./subjob_cdomiss.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1
+./subjob_ncochunk.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1
+./move_to_perm.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1
+
exit 0
diff --git a/job_ablvint.sbatch b/job_ablvint.sbatch
index 6ad8627..f6ed723 100755
--- a/job_ablvint.sbatch
+++ b/job_ablvint.sbatch
@@ -3,6 +3,7 @@
#SBATCH --output=JOBS/%x-%j.out
#SBATCH --qos=nf
#SBATCH --time=01:00:00
+#SBATCH --gres=ssdtmp:15G
module purge
@@ -12,23 +13,29 @@ module list
set +xv
if [ -n "$1" ]; then date=$1; else exit 1; fi
+if [ -n "$2" ]; then var_name=$2; fi
p2D="${SCRATCH_DIR}/NC_F320/${date:0:6}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc"
p3D="${SCRATCH_DIR}/NC_F320/${date:0:6}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc"
pmsk="/ec/res5/project/mercator/ATMO_FORCINGS/ERA5_ABL50_F320_v1/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc"
fabl=$(basename ${p3D/inst3D/abl}) ; fabl=${fabl/L107-137/L50Z10}
-cp -av ${p2D} ${p3D} ${pmsk} ${p3D/inst3D/hpg} $TMPDIR/.
+cp -av ${p2D} ${p3D} ${pmsk} $TMPDIR/.
+if [[ -n "${var_name}" && ${var_name} =~ "hpg" ]] || [ -z "${var_name}" ]; then cp -av ${p3D/inst3D/hpg} $TMPDIR/.; fi
cp -v /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/main_vinterp.exe $TMPDIR/.
sed -e "s/fsurf2D/$(basename ${p2D})/g; s/finst3D/$(basename ${p3D})/g; s/fmask/$(basename ${pmsk})/g; s/fhpg/$(basename ${p3D/inst3D/hpg})/g; s/fabl/${fabl}/g; s~fdir~$TMPDIR~g" < /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/namelist_abl_tools.base > $TMPDIR/namelist_abl_tools.vint.$date
sed -i "s/ln_read_mask = .false./ln_read_mask = .true./g" $TMPDIR/namelist_abl_tools.vint.$date
+if [ -n "${var_name}" ]; then
+ sed -i "s/ var_name = '',/ var_name = '${var_name}',/g" $TMPDIR/namelist_abl_tools.vint.$date
+ if [[ ! ${var_name} =~ "hpg" ]]; then sed -i "s/ln_hpg_frc = .true. /ln_hpg_frc = .false./g" $TMPDIR/namelist_abl_tools.vint.$date; fi
+fi
ls -l $TMPDIR
cat $TMPDIR/namelist_abl_tools.vint.$date
$TMPDIR/main_vinterp.exe $TMPDIR/namelist_abl_tools.vint.$date
-mv -v $TMPDIR/${fabl} $(dirname ${p3D})/.
+mv -v $TMPDIR/${var_name:+"${var_name}_"}${fabl} $(dirname ${p3D})/.
exit 0
diff --git a/job_cdogrid.sbatch b/job_cdogrid.sbatch
index 57610f1..595d9e7 100755
--- a/job_cdogrid.sbatch
+++ b/job_cdogrid.sbatch
@@ -3,6 +3,7 @@
#SBATCH --output=JOBS/%x-%j.out
#SBATCH --qos=nf
#SBATCH --time=00:30:00
+#SBATCH --gres=ssdtmp:10G
module purge
module load cdo/2.2.0
diff --git a/job_cdomiss.sbatch b/job_cdomiss.sbatch
index c128869..6ab6eb5 100755
--- a/job_cdomiss.sbatch
+++ b/job_cdomiss.sbatch
@@ -4,6 +4,7 @@
#SBATCH --qos=nf
#SBATCH --time=00:30:00
#SBATCH --cpus-per-task=4
+#SBATCH --gres=ssdtmp:10G
module purge
diff --git a/job_mars.sbatch b/job_mars.sbatch
index 50c231b..cbebab5 100755
--- a/job_mars.sbatch
+++ b/job_mars.sbatch
@@ -3,6 +3,7 @@
#SBATCH --output=JOBS/%x-%j.out
#SBATCH --qos=nf
#SBATCH --time=00:30:00
+#SBATCH --gres=ssdtmp:10G
set +xv
diff --git a/job_ncochunk.sbatch b/job_ncochunk.sbatch
index aa6464a..7c47a71 100755
--- a/job_ncochunk.sbatch
+++ b/job_ncochunk.sbatch
@@ -4,6 +4,7 @@
#SBATCH --qos=nf
#SBATCH --time=00:30:00
#SBATCH --cpus-per-task=4
+#SBATCH --gres=ssdtmp:10G
module purge
diff --git a/move_to_perm.sh b/move_to_perm.sh
index 3e8c69f..be47c65 100755
--- a/move_to_perm.sh
+++ b/move_to_perm.sh
@@ -6,8 +6,11 @@ set +xv
date_beg=$1
date_end=$2
nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) ))
-var_list=( uwnd vwnd uhpg vhpg tpot humi )
-var_nb=${#var_list[*]}
+if [ -n "$3" ]; then
+ var_list=( $3 )
+else
+ var_list=( uwnd vwnd uhpg vhpg tpot humi )
+fi
for d in $(seq -f "%02g" 0 $((nbday-1))); do
@@ -36,11 +39,15 @@ for d in $(seq -f "%02g" 0 $((nbday-1))); do
done
if [ ${date} -eq ${date_end} ]; then
- file_nb=$( eval "ls ${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}/ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d{${date_beg:6:2}..${date_end:6:2}}.nc 2>/dev/null" | wc -l )
+ if [ ${#var_list[@]} -gt 1 ]; then
+ file_nb=$( eval "ls ${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}/ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d{${date_beg:6:2}..${date_end:6:2}}.nc 2>/dev/null" | wc -l )
+ else
+ file_nb=$( eval "ls ${DIR_SAVE}/${var_list[*]^^}/ERA5_L50Z10_${var_list[*]^^}_1H_extrap_y${date:0:4}m${date:4:2}d{${date_beg:6:2}..${date_end:6:2}}.nc 2>/dev/null" | wc -l )
+ fi
if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then
echo "MOVE_TO_PERM OK"
eval "rm -v ${SCRATCH_DIR}/NC_F320/${ym}/*{${date_beg}..${date_end}}*.nc"
- eval "rm -v ${SCRATCH_DIR}/GRIB_F320/${ym}/*{${date_beg}..${date_end}}*.grib"
+ eval "rm -v ${SCRATCH_DIR}/GRIB_N320/${ym}/*{${date_beg}..${date_end}}*.grib"
echo "CLEANING OK"
exit 0
else
diff --git a/subjob_ablvint.sh b/subjob_ablvint.sh
index fd4b8b9..9363d42 100755
--- a/subjob_ablvint.sh
+++ b/subjob_ablvint.sh
@@ -7,6 +7,7 @@ set +xv
date_beg=$1
date_end=$2
nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) ))
+if [ -n "$3" ]; then var_name=$3; fi
for d in $(seq -f "%02g" 0 $((nbday-1))); do
@@ -21,24 +22,24 @@ for d in $(seq -f "%02g" 0 $((nbday-1))); do
f2D=$(basename $p2D) ; d2D=$(dirname $p2D)
f3D=$(basename $p3D) ; d3D=$(dirname $p3D)
fmsk=$(basename $pmsk) ; dmsk=$(dirname $pmsk)
- fhpg=${f3D/inst3D/hpg}
fabl=${f3D/inst3D/abl} ; fabl=${fabl/L107-137/L50Z10}
+ fhpg=${f3D/inst3D/hpg}
- if [ ! -s ${d2D}/${fabl} ]; then
+ if [ ! -s ${d2D}/${var_name:+"${var_name}_"}${fabl} ]; then
- if [[ -s ${p2D} && -s ${p3D} && -s ${d2D}/${fhpg} ]]; then
+ if [[ -s ${p2D} && -s ${p3D} ]] && [[ -s ${d2D}/${fhpg} || -n "${var_name}" ]]; then
while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -ge 90 ]; do echo "sleeping..."; sleep 10; done
- sbatch --job-name=ablvint_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_ablvint.sbatch ${date}
+ sbatch --job-name=ablvint${var_name:+"_${var_name}"}_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_ablvint.sbatch ${date} ${var_name}
else
echo "ABL_VINT ERROR: input files not found"; exit 1
fi
fi
- while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep ablvint_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done
+ while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep ablvint${var_name:+"_${var_name}"}_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done
if [ ${date} -eq ${date_end} ]; then
- file_nb=$( eval "ls ${d2D}/${fabl/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l )
+ file_nb=$( eval "ls ${d2D}/${var_name:+"${var_name}_"}${fabl/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l )
if [ ${nbday} -eq ${file_nb} ]; then
echo "ABL_VINT OK"
else
diff --git a/subjob_cdomiss.sh b/subjob_cdomiss.sh
index 0717099..a74506c 100755
--- a/subjob_cdomiss.sh
+++ b/subjob_cdomiss.sh
@@ -7,7 +7,11 @@ set +xv
date_beg=$1
date_end=$2
nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) ))
-var_list=( uwnd vwnd uhpg vhpg tpot humi )
+if [ -n "$3" ]; then
+ var_list=( $3 )
+else
+ var_list=( uwnd vwnd uhpg vhpg tpot humi )
+fi
for d in $(seq -f "%02g" 0 $((nbday-1))); do
@@ -38,7 +42,11 @@ for d in $(seq -f "%02g" 0 $((nbday-1))); do
while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep cdomiss_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done
if [ ${date} -eq ${date_end} ]; then
- file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${f/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis.nc 2>/dev/null" | wc -l )
+ if [ ${#var_list[@]} -gt 1 ]; then
+ file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${f/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis.nc 2>/dev/null" | wc -l )
+ else
+ file_nb=$( eval "ls ${d}/${var_list[*]}_${f/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis.nc 2>/dev/null" | wc -l )
+ fi
if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then
echo "CDO_MISS OK"
else
diff --git a/subjob_ncochunk.sh b/subjob_ncochunk.sh
index 37f5932..b5c199e 100755
--- a/subjob_ncochunk.sh
+++ b/subjob_ncochunk.sh
@@ -7,8 +7,11 @@ set +xv
date_beg=$1
date_end=$2
nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) ))
-var_list=( uwnd vwnd uhpg vhpg tpot humi )
-var_nb=${#var_list[*]}
+if [ -n "$3" ]; then
+ var_list=( $3 )
+else
+ var_list=( uwnd vwnd uhpg vhpg tpot humi )
+fi
for d in $(seq -f "%02g" 0 $((nbday-1))); do
@@ -40,7 +43,11 @@ while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -
if [ ${date} -eq ${date_end} ]; then
fall=${f/${date}_misstodis.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis_cnk64.nc
- file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fall} 2>/dev/null" | wc -l )
+ if [ ${#var_list[@]} -gt 1 ]; then
+ file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fall} 2>/dev/null" | wc -l )
+ else
+ file_nb=$( eval "ls ${d}/${var_list[*]}_${fall} 2>/dev/null" | wc -l )
+ fi
if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then
echo "NCO_CHUNK OK"
else
--
GitLab