From f4f8fd6a54cddb1c309f2977ac779aa96a282a25 Mon Sep 17 00:00:00 2001
From: Guillaume S <gsamson@mercator-ocean.fr>
Date: Fri, 19 Jan 2024 19:11:58 +0100
Subject: [PATCH] allow to treat only one variable

---
 check_if_done.sh    | 15 ++++++++++++---
 full_prod.sh        | 43 ++++++++++++++++++++++++++++++++++---------
 job_ablvint.sbatch  | 11 +++++++++--
 job_cdogrid.sbatch  |  1 +
 job_cdomiss.sbatch  |  1 +
 job_mars.sbatch     |  1 +
 job_ncochunk.sbatch |  1 +
 move_to_perm.sh     | 15 +++++++++++----
 subjob_ablvint.sh   | 13 +++++++------
 subjob_cdomiss.sh   | 12 ++++++++++--
 subjob_ncochunk.sh  | 13 ++++++++++---
 11 files changed, 97 insertions(+), 29 deletions(-)

diff --git a/check_if_done.sh b/check_if_done.sh
index 0b23068..e463dd0 100755
--- a/check_if_done.sh
+++ b/check_if_done.sh
@@ -6,13 +6,22 @@ date_beg=$1
 date_end=$2
 nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) ))
 
-var_list=( uwnd vwnd uhpg vhpg tpot humi )
+if [ -n "$3" ]; then
+  var_list=( $3 )
+else
+  var_list=( uwnd vwnd uhpg vhpg tpot humi )
+fi
 
 for d in $(seq -f "%02g" 0 $((nbday-1))); do
 
   date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d")
-  fvar="ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc"
-  dvar="${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}"
+  if [ ${#var_list[@]} -gt 1 ]; then
+    fvar="ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc"
+    dvar="${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}"
+  else
+    fvar="ERA5_L50Z10_${var_list[*]^^}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc"
+    dvar="${DIR_SAVE}/${var_list[*]^^}"
+  fi
   file_nb=$(eval "ls -l ${dvar}/${fvar} 2>/dev/null" | wc -l )
   tot_nb=$(( ${tot_nb:-0} + file_nb ))
 
diff --git a/full_prod.sh b/full_prod.sh
index 25ca21f..1577029 100755
--- a/full_prod.sh
+++ b/full_prod.sh
@@ -2,27 +2,52 @@
   
 set +xv
 
+
+# args
 date_beg=$1
-date_end=$2
+if [[ $2 =~ '^[0-9]+$' ]]; then
+  date_end=$2
+else
+  var_name=$2
+fi
+if [ -n "$3" ]; then
+  var_name=$3
+fi
+
+
+# date range
+if [ -z "${date_end}" ]; then
+  date_end=${date_beg}
+fi
 
 if [[ ${#date_beg} -eq 6 || ${#date_end} -eq 6 ]]; then
   date_beg=${date_beg:0:6}01
   date_end=$(date -d "${date_beg} + 1 month - 1 day" +"%C%y%m%d" )
 fi
-echo "${date_beg}->${date_end}"
 
+echo "${var_name:+"${var_name}: "}${date_beg}->${date_end}"
+
+
+# directories
 export DIR_SRC="/home/ar0s/PROD_ERA5_ABL"
 export DIR_SAVE="/ec/res5/project/mercator/ATMO_FORCINGS/ERA5_ABL50_F320_v1/daily/1h"
 export SCRATCH_DIR=${SCRATCHDIR}
 
-./check_if_done.sh   ${date_beg} ${date_end}; [ $? -eq 0 ] && exit 0
+
+# scripts
+./check_if_done.sh   ${date_beg} ${date_end} ${var_name}; [ $? -eq 0 ] && exit 0
 ./subjob_mars.sh     ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
 ./subjob_cdogrid.sh  ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./subjob_ablhpg.sh   ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./subjob_ablvint.sh  ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./subjob_cdosplit.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./subjob_cdomiss.sh  ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./subjob_ncochunk.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
-./move_to_perm.sh    ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
+if [[ -z "${var_name}" || ${var_name} =~ "hpg" ]]; then
+  ./subjob_ablhpg.sh   ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
+fi
+./subjob_ablvint.sh  ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1
+if [ -z "${var_name}" ]; then
+  ./subjob_cdosplit.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1
+fi
+./subjob_cdomiss.sh  ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1
+./subjob_ncochunk.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1
+./move_to_perm.sh    ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1
+
 
 exit 0
diff --git a/job_ablvint.sbatch b/job_ablvint.sbatch
index 6ad8627..f6ed723 100755
--- a/job_ablvint.sbatch
+++ b/job_ablvint.sbatch
@@ -3,6 +3,7 @@
 #SBATCH --output=JOBS/%x-%j.out 
 #SBATCH --qos=nf
 #SBATCH --time=01:00:00
+#SBATCH --gres=ssdtmp:15G
 
 
 module purge
@@ -12,23 +13,29 @@ module list
 set +xv
 
 if [ -n "$1" ]; then date=$1; else exit 1; fi
+if [ -n "$2" ]; then var_name=$2; fi
 
 p2D="${SCRATCH_DIR}/NC_F320/${date:0:6}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc"
 p3D="${SCRATCH_DIR}/NC_F320/${date:0:6}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc"
 pmsk="/ec/res5/project/mercator/ATMO_FORCINGS/ERA5_ABL50_F320_v1/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc"
 fabl=$(basename ${p3D/inst3D/abl}) ; fabl=${fabl/L107-137/L50Z10}
 
-cp -av ${p2D} ${p3D} ${pmsk} ${p3D/inst3D/hpg} $TMPDIR/.
+cp -av ${p2D} ${p3D} ${pmsk} $TMPDIR/.
+if [[ -n "${var_name}" && ${var_name} =~ "hpg" ]] || [ -z "${var_name}" ]; then cp -av ${p3D/inst3D/hpg} $TMPDIR/.; fi
 cp -v /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/main_vinterp.exe $TMPDIR/.
 
 sed -e "s/fsurf2D/$(basename ${p2D})/g; s/finst3D/$(basename ${p3D})/g; s/fmask/$(basename ${pmsk})/g; s/fhpg/$(basename ${p3D/inst3D/hpg})/g; s/fabl/${fabl}/g; s~fdir~$TMPDIR~g" < /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/namelist_abl_tools.base > $TMPDIR/namelist_abl_tools.vint.$date
 sed -i "s/ln_read_mask    =   .false./ln_read_mask    =    .true./g" $TMPDIR/namelist_abl_tools.vint.$date
+if [ -n "${var_name}" ]; then
+  sed -i "s/    var_name  = '',/    var_name  = '${var_name}',/g" $TMPDIR/namelist_abl_tools.vint.$date
+  if [[ ! ${var_name} =~ "hpg" ]]; then sed -i "s/ln_hpg_frc      =   .true. /ln_hpg_frc      =   .false./g" $TMPDIR/namelist_abl_tools.vint.$date; fi
+fi
 
 ls -l $TMPDIR
 cat $TMPDIR/namelist_abl_tools.vint.$date
 
 $TMPDIR/main_vinterp.exe $TMPDIR/namelist_abl_tools.vint.$date
 
-mv -v $TMPDIR/${fabl} $(dirname ${p3D})/.
+mv -v $TMPDIR/${var_name:+"${var_name}_"}${fabl} $(dirname ${p3D})/.
 
 exit 0
diff --git a/job_cdogrid.sbatch b/job_cdogrid.sbatch
index 57610f1..595d9e7 100755
--- a/job_cdogrid.sbatch
+++ b/job_cdogrid.sbatch
@@ -3,6 +3,7 @@
 #SBATCH --output=JOBS/%x-%j.out 
 #SBATCH --qos=nf
 #SBATCH --time=00:30:00
+#SBATCH --gres=ssdtmp:10G
 
 module purge
 module load cdo/2.2.0
diff --git a/job_cdomiss.sbatch b/job_cdomiss.sbatch
index c128869..6ab6eb5 100755
--- a/job_cdomiss.sbatch
+++ b/job_cdomiss.sbatch
@@ -4,6 +4,7 @@
 #SBATCH --qos=nf
 #SBATCH --time=00:30:00
 #SBATCH --cpus-per-task=4
+#SBATCH --gres=ssdtmp:10G
 
 
 module purge
diff --git a/job_mars.sbatch b/job_mars.sbatch
index 50c231b..cbebab5 100755
--- a/job_mars.sbatch
+++ b/job_mars.sbatch
@@ -3,6 +3,7 @@
 #SBATCH --output=JOBS/%x-%j.out 
 #SBATCH --qos=nf
 #SBATCH --time=00:30:00
+#SBATCH --gres=ssdtmp:10G
 
 set +xv
 
diff --git a/job_ncochunk.sbatch b/job_ncochunk.sbatch
index aa6464a..7c47a71 100755
--- a/job_ncochunk.sbatch
+++ b/job_ncochunk.sbatch
@@ -4,6 +4,7 @@
 #SBATCH --qos=nf
 #SBATCH --time=00:30:00
 #SBATCH --cpus-per-task=4
+#SBATCH --gres=ssdtmp:10G
 
 
 module purge
diff --git a/move_to_perm.sh b/move_to_perm.sh
index 3e8c69f..be47c65 100755
--- a/move_to_perm.sh
+++ b/move_to_perm.sh
@@ -6,8 +6,11 @@ set +xv
 date_beg=$1
 date_end=$2
 nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) ))
-var_list=( uwnd vwnd uhpg vhpg tpot humi )
-var_nb=${#var_list[*]}
+if [ -n "$3" ]; then
+  var_list=( $3 )
+else
+  var_list=( uwnd vwnd uhpg vhpg tpot humi )
+fi
 
 for d in $(seq -f "%02g" 0 $((nbday-1))); do
 
@@ -36,11 +39,15 @@ for d in $(seq -f "%02g" 0 $((nbday-1))); do
   done
   
   if [ ${date} -eq ${date_end} ]; then
-    file_nb=$( eval "ls ${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}/ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d{${date_beg:6:2}..${date_end:6:2}}.nc 2>/dev/null" | wc -l )
+    if [ ${#var_list[@]} -gt 1 ]; then
+      file_nb=$( eval "ls ${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}/ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d{${date_beg:6:2}..${date_end:6:2}}.nc 2>/dev/null" | wc -l )
+    else
+      file_nb=$( eval "ls ${DIR_SAVE}/${var_list[*]^^}/ERA5_L50Z10_${var_list[*]^^}_1H_extrap_y${date:0:4}m${date:4:2}d{${date_beg:6:2}..${date_end:6:2}}.nc 2>/dev/null" | wc -l )
+    fi
     if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then
       echo "MOVE_TO_PERM OK"
       eval "rm -v ${SCRATCH_DIR}/NC_F320/${ym}/*{${date_beg}..${date_end}}*.nc"
-      eval "rm -v ${SCRATCH_DIR}/GRIB_F320/${ym}/*{${date_beg}..${date_end}}*.grib"
+      eval "rm -v ${SCRATCH_DIR}/GRIB_N320/${ym}/*{${date_beg}..${date_end}}*.grib"
       echo "CLEANING OK"
       exit 0
     else
diff --git a/subjob_ablvint.sh b/subjob_ablvint.sh
index fd4b8b9..9363d42 100755
--- a/subjob_ablvint.sh
+++ b/subjob_ablvint.sh
@@ -7,6 +7,7 @@ set +xv
 date_beg=$1
 date_end=$2
 nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) ))
+if [ -n "$3" ]; then var_name=$3; fi
 
 for d in $(seq -f "%02g" 0 $((nbday-1))); do
 
@@ -21,24 +22,24 @@ for d in $(seq -f "%02g" 0 $((nbday-1))); do
   f2D=$(basename $p2D)   ; d2D=$(dirname $p2D)
   f3D=$(basename $p3D)   ; d3D=$(dirname $p3D)
   fmsk=$(basename $pmsk) ; dmsk=$(dirname $pmsk)
-  fhpg=${f3D/inst3D/hpg}
   fabl=${f3D/inst3D/abl} ; fabl=${fabl/L107-137/L50Z10}
+  fhpg=${f3D/inst3D/hpg}
 
-  if [ ! -s ${d2D}/${fabl} ]; then
+  if [ ! -s ${d2D}/${var_name:+"${var_name}_"}${fabl} ]; then
   
-    if [[ -s ${p2D} && -s ${p3D} && -s ${d2D}/${fhpg} ]]; then
+    if [[ -s ${p2D} && -s ${p3D} ]] && [[ -s ${d2D}/${fhpg} || -n "${var_name}" ]]; then
       while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -ge 90 ]; do echo "sleeping..."; sleep 10; done
-      sbatch --job-name=ablvint_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_ablvint.sbatch ${date}
+      sbatch --job-name=ablvint${var_name:+"_${var_name}"}_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_ablvint.sbatch ${date} ${var_name}
     else
       echo "ABL_VINT ERROR: input files not found"; exit 1
     fi
   
   fi
   
-  while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep ablvint_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done
+  while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep ablvint${var_name:+"_${var_name}"}_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done
   
   if [ ${date} -eq ${date_end} ]; then
-    file_nb=$( eval "ls ${d2D}/${fabl/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l )
+    file_nb=$( eval "ls ${d2D}/${var_name:+"${var_name}_"}${fabl/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l )
     if [ ${nbday} -eq ${file_nb} ]; then
       echo "ABL_VINT OK"
     else 
diff --git a/subjob_cdomiss.sh b/subjob_cdomiss.sh
index 0717099..a74506c 100755
--- a/subjob_cdomiss.sh
+++ b/subjob_cdomiss.sh
@@ -7,7 +7,11 @@ set +xv
 date_beg=$1
 date_end=$2
 nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) ))
-var_list=( uwnd vwnd uhpg vhpg tpot humi )
+if [ -n "$3" ]; then
+  var_list=( $3 )
+else
+  var_list=( uwnd vwnd uhpg vhpg tpot humi )
+fi
 
 for d in $(seq -f "%02g" 0 $((nbday-1))); do
 
@@ -38,7 +42,11 @@ for d in $(seq -f "%02g" 0 $((nbday-1))); do
   while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep cdomiss_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done
   
   if [ ${date} -eq ${date_end} ]; then
-    file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${f/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis.nc 2>/dev/null" | wc -l )
+    if [ ${#var_list[@]} -gt 1 ]; then
+      file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${f/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis.nc 2>/dev/null" | wc -l )
+    else
+      file_nb=$( eval "ls ${d}/${var_list[*]}_${f/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis.nc 2>/dev/null" | wc -l )
+    fi
     if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then
       echo "CDO_MISS OK"
     else
diff --git a/subjob_ncochunk.sh b/subjob_ncochunk.sh
index 37f5932..b5c199e 100755
--- a/subjob_ncochunk.sh
+++ b/subjob_ncochunk.sh
@@ -7,8 +7,11 @@ set +xv
 date_beg=$1
 date_end=$2
 nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) ))
-var_list=( uwnd vwnd uhpg vhpg tpot humi )
-var_nb=${#var_list[*]}
+if [ -n "$3" ]; then
+  var_list=( $3 )
+else
+  var_list=( uwnd vwnd uhpg vhpg tpot humi )
+fi
 
 for d in $(seq -f "%02g" 0 $((nbday-1))); do
 
@@ -40,7 +43,11 @@ while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -
 
 if [ ${date} -eq ${date_end} ]; then
   fall=${f/${date}_misstodis.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis_cnk64.nc
-  file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fall} 2>/dev/null" | wc -l )
+  if [ ${#var_list[@]} -gt 1 ]; then
+    file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fall} 2>/dev/null" | wc -l )
+  else
+    file_nb=$( eval "ls ${d}/${var_list[*]}_${fall} 2>/dev/null" | wc -l )
+  fi
   if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then
     echo "NCO_CHUNK OK"
   else
-- 
GitLab