diff --git a/check_if_done.sh b/check_if_done.sh index 0b23068c034d235c215fff546bef6649595f1dd8..e463dd09100ffd7d154b4a2a9820908757b26bb5 100755 --- a/check_if_done.sh +++ b/check_if_done.sh @@ -6,13 +6,22 @@ date_beg=$1 date_end=$2 nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) -var_list=( uwnd vwnd uhpg vhpg tpot humi ) +if [ -n "$3" ]; then + var_list=( $3 ) +else + var_list=( uwnd vwnd uhpg vhpg tpot humi ) +fi for d in $(seq -f "%02g" 0 $((nbday-1))); do date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d") - fvar="ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc" - dvar="${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}" + if [ ${#var_list[@]} -gt 1 ]; then + fvar="ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc" + dvar="${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}" + else + fvar="ERA5_L50Z10_${var_list[*]^^}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc" + dvar="${DIR_SAVE}/${var_list[*]^^}" + fi file_nb=$(eval "ls -l ${dvar}/${fvar} 2>/dev/null" | wc -l ) tot_nb=$(( ${tot_nb:-0} + file_nb )) diff --git a/full_prod.sh b/full_prod.sh index 25ca21f4f779fd50d3f58bcbf2a2c0e6f0db2abb..1577029b66fd8ad2fa6ad5c632036ad890e13259 100755 --- a/full_prod.sh +++ b/full_prod.sh @@ -2,27 +2,52 @@ set +xv + +# args date_beg=$1 -date_end=$2 +if [[ $2 =~ '^[0-9]+$' ]]; then + date_end=$2 +else + var_name=$2 +fi +if [ -n "$3" ]; then + var_name=$3 +fi + + +# date range +if [ -z "${date_end}" ]; then + date_end=${date_beg} +fi if [[ ${#date_beg} -eq 6 || ${#date_end} -eq 6 ]]; then date_beg=${date_beg:0:6}01 date_end=$(date -d "${date_beg} + 1 month - 1 day" +"%C%y%m%d" ) fi -echo "${date_beg}->${date_end}" +echo "${var_name:+"${var_name}: "}${date_beg}->${date_end}" + + +# directories export DIR_SRC="/home/ar0s/PROD_ERA5_ABL" export DIR_SAVE="/ec/res5/project/mercator/ATMO_FORCINGS/ERA5_ABL50_F320_v1/daily/1h" export SCRATCH_DIR=${SCRATCHDIR} -./check_if_done.sh ${date_beg} ${date_end}; [ $? -eq 0 ] && exit 0 + +# scripts +./check_if_done.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 0 ] && exit 0 ./subjob_mars.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 ./subjob_cdogrid.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 -./subjob_ablhpg.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 -./subjob_ablvint.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 -./subjob_cdosplit.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 -./subjob_cdomiss.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 -./subjob_ncochunk.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 -./move_to_perm.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 +if [[ -z "${var_name}" || ${var_name} =~ "hpg" ]]; then + ./subjob_ablhpg.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 +fi +./subjob_ablvint.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1 +if [ -z "${var_name}" ]; then + ./subjob_cdosplit.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 +fi +./subjob_cdomiss.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1 +./subjob_ncochunk.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1 +./move_to_perm.sh ${date_beg} ${date_end} ${var_name}; [ $? -eq 1 ] && exit 1 + exit 0 diff --git a/job_ablvint.sbatch b/job_ablvint.sbatch index 6ad8627827248bac3461239b032ce135bb838686..f6ed723e074a35aed2f3000c0ffbb66b0c97caf3 100755 --- a/job_ablvint.sbatch +++ b/job_ablvint.sbatch @@ -3,6 +3,7 @@ #SBATCH --output=JOBS/%x-%j.out #SBATCH --qos=nf #SBATCH --time=01:00:00 +#SBATCH --gres=ssdtmp:15G module purge @@ -12,23 +13,29 @@ module list set +xv if [ -n "$1" ]; then date=$1; else exit 1; fi +if [ -n "$2" ]; then var_name=$2; fi p2D="${SCRATCH_DIR}/NC_F320/${date:0:6}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" p3D="${SCRATCH_DIR}/NC_F320/${date:0:6}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" pmsk="/ec/res5/project/mercator/ATMO_FORCINGS/ERA5_ABL50_F320_v1/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc" fabl=$(basename ${p3D/inst3D/abl}) ; fabl=${fabl/L107-137/L50Z10} -cp -av ${p2D} ${p3D} ${pmsk} ${p3D/inst3D/hpg} $TMPDIR/. +cp -av ${p2D} ${p3D} ${pmsk} $TMPDIR/. +if [[ -n "${var_name}" && ${var_name} =~ "hpg" ]] || [ -z "${var_name}" ]; then cp -av ${p3D/inst3D/hpg} $TMPDIR/.; fi cp -v /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/main_vinterp.exe $TMPDIR/. sed -e "s/fsurf2D/$(basename ${p2D})/g; s/finst3D/$(basename ${p3D})/g; s/fmask/$(basename ${pmsk})/g; s/fhpg/$(basename ${p3D/inst3D/hpg})/g; s/fabl/${fabl}/g; s~fdir~$TMPDIR~g" < /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/namelist_abl_tools.base > $TMPDIR/namelist_abl_tools.vint.$date sed -i "s/ln_read_mask = .false./ln_read_mask = .true./g" $TMPDIR/namelist_abl_tools.vint.$date +if [ -n "${var_name}" ]; then + sed -i "s/ var_name = '',/ var_name = '${var_name}',/g" $TMPDIR/namelist_abl_tools.vint.$date + if [[ ! ${var_name} =~ "hpg" ]]; then sed -i "s/ln_hpg_frc = .true. /ln_hpg_frc = .false./g" $TMPDIR/namelist_abl_tools.vint.$date; fi +fi ls -l $TMPDIR cat $TMPDIR/namelist_abl_tools.vint.$date $TMPDIR/main_vinterp.exe $TMPDIR/namelist_abl_tools.vint.$date -mv -v $TMPDIR/${fabl} $(dirname ${p3D})/. +mv -v $TMPDIR/${var_name:+"${var_name}_"}${fabl} $(dirname ${p3D})/. exit 0 diff --git a/job_cdogrid.sbatch b/job_cdogrid.sbatch index 57610f1d1115206600100cddcfbeeef6bbeeb284..595d9e74dbe3a1002845593763c3b65244c74ec0 100755 --- a/job_cdogrid.sbatch +++ b/job_cdogrid.sbatch @@ -3,6 +3,7 @@ #SBATCH --output=JOBS/%x-%j.out #SBATCH --qos=nf #SBATCH --time=00:30:00 +#SBATCH --gres=ssdtmp:10G module purge module load cdo/2.2.0 diff --git a/job_cdomiss.sbatch b/job_cdomiss.sbatch index c1288696adccf5ddaef63516c17668a1ab915d66..6ab6eb533c7099e26fa36f64d152743f0d0912a4 100755 --- a/job_cdomiss.sbatch +++ b/job_cdomiss.sbatch @@ -4,6 +4,7 @@ #SBATCH --qos=nf #SBATCH --time=00:30:00 #SBATCH --cpus-per-task=4 +#SBATCH --gres=ssdtmp:10G module purge diff --git a/job_mars.sbatch b/job_mars.sbatch index 50c231ba8644314cf538945cae8a9af4e0d50528..cbebab557c23e85e7406208ae6584a3e38147214 100755 --- a/job_mars.sbatch +++ b/job_mars.sbatch @@ -3,6 +3,7 @@ #SBATCH --output=JOBS/%x-%j.out #SBATCH --qos=nf #SBATCH --time=00:30:00 +#SBATCH --gres=ssdtmp:10G set +xv diff --git a/job_ncochunk.sbatch b/job_ncochunk.sbatch index aa6464aba9f0a64918ac9be8a33ce624ff9b6aa5..7c47a714b1d52d267ffa097786c07c0620f5b02a 100755 --- a/job_ncochunk.sbatch +++ b/job_ncochunk.sbatch @@ -4,6 +4,7 @@ #SBATCH --qos=nf #SBATCH --time=00:30:00 #SBATCH --cpus-per-task=4 +#SBATCH --gres=ssdtmp:10G module purge diff --git a/move_to_perm.sh b/move_to_perm.sh index 3e8c69fa45f8775abaf79a7719c2ae458ece685a..be47c65f8369832cae1563668fbb19f965fda5a5 100755 --- a/move_to_perm.sh +++ b/move_to_perm.sh @@ -6,8 +6,11 @@ set +xv date_beg=$1 date_end=$2 nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) -var_list=( uwnd vwnd uhpg vhpg tpot humi ) -var_nb=${#var_list[*]} +if [ -n "$3" ]; then + var_list=( $3 ) +else + var_list=( uwnd vwnd uhpg vhpg tpot humi ) +fi for d in $(seq -f "%02g" 0 $((nbday-1))); do @@ -36,11 +39,15 @@ for d in $(seq -f "%02g" 0 $((nbday-1))); do done if [ ${date} -eq ${date_end} ]; then - file_nb=$( eval "ls ${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}/ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d{${date_beg:6:2}..${date_end:6:2}}.nc 2>/dev/null" | wc -l ) + if [ ${#var_list[@]} -gt 1 ]; then + file_nb=$( eval "ls ${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}/ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d{${date_beg:6:2}..${date_end:6:2}}.nc 2>/dev/null" | wc -l ) + else + file_nb=$( eval "ls ${DIR_SAVE}/${var_list[*]^^}/ERA5_L50Z10_${var_list[*]^^}_1H_extrap_y${date:0:4}m${date:4:2}d{${date_beg:6:2}..${date_end:6:2}}.nc 2>/dev/null" | wc -l ) + fi if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then echo "MOVE_TO_PERM OK" eval "rm -v ${SCRATCH_DIR}/NC_F320/${ym}/*{${date_beg}..${date_end}}*.nc" - eval "rm -v ${SCRATCH_DIR}/GRIB_F320/${ym}/*{${date_beg}..${date_end}}*.grib" + eval "rm -v ${SCRATCH_DIR}/GRIB_N320/${ym}/*{${date_beg}..${date_end}}*.grib" echo "CLEANING OK" exit 0 else diff --git a/subjob_ablvint.sh b/subjob_ablvint.sh index fd4b8b9447d7f20e74dfce68d4877232837279f6..9363d428782ef9750bf7b90fe4950bdfa46f6f05 100755 --- a/subjob_ablvint.sh +++ b/subjob_ablvint.sh @@ -7,6 +7,7 @@ set +xv date_beg=$1 date_end=$2 nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) +if [ -n "$3" ]; then var_name=$3; fi for d in $(seq -f "%02g" 0 $((nbday-1))); do @@ -21,24 +22,24 @@ for d in $(seq -f "%02g" 0 $((nbday-1))); do f2D=$(basename $p2D) ; d2D=$(dirname $p2D) f3D=$(basename $p3D) ; d3D=$(dirname $p3D) fmsk=$(basename $pmsk) ; dmsk=$(dirname $pmsk) - fhpg=${f3D/inst3D/hpg} fabl=${f3D/inst3D/abl} ; fabl=${fabl/L107-137/L50Z10} + fhpg=${f3D/inst3D/hpg} - if [ ! -s ${d2D}/${fabl} ]; then + if [ ! -s ${d2D}/${var_name:+"${var_name}_"}${fabl} ]; then - if [[ -s ${p2D} && -s ${p3D} && -s ${d2D}/${fhpg} ]]; then + if [[ -s ${p2D} && -s ${p3D} ]] && [[ -s ${d2D}/${fhpg} || -n "${var_name}" ]]; then while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -ge 90 ]; do echo "sleeping..."; sleep 10; done - sbatch --job-name=ablvint_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_ablvint.sbatch ${date} + sbatch --job-name=ablvint${var_name:+"_${var_name}"}_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_ablvint.sbatch ${date} ${var_name} else echo "ABL_VINT ERROR: input files not found"; exit 1 fi fi - while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep ablvint_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done + while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep ablvint${var_name:+"_${var_name}"}_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done if [ ${date} -eq ${date_end} ]; then - file_nb=$( eval "ls ${d2D}/${fabl/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l ) + file_nb=$( eval "ls ${d2D}/${var_name:+"${var_name}_"}${fabl/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l ) if [ ${nbday} -eq ${file_nb} ]; then echo "ABL_VINT OK" else diff --git a/subjob_cdomiss.sh b/subjob_cdomiss.sh index 07170993c432e44daba8f0ff38ea230afa673828..a74506c6f7dbc0a2db7a6ef63bd9a2cd493e50c5 100755 --- a/subjob_cdomiss.sh +++ b/subjob_cdomiss.sh @@ -7,7 +7,11 @@ set +xv date_beg=$1 date_end=$2 nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) -var_list=( uwnd vwnd uhpg vhpg tpot humi ) +if [ -n "$3" ]; then + var_list=( $3 ) +else + var_list=( uwnd vwnd uhpg vhpg tpot humi ) +fi for d in $(seq -f "%02g" 0 $((nbday-1))); do @@ -38,7 +42,11 @@ for d in $(seq -f "%02g" 0 $((nbday-1))); do while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep cdomiss_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done if [ ${date} -eq ${date_end} ]; then - file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${f/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis.nc 2>/dev/null" | wc -l ) + if [ ${#var_list[@]} -gt 1 ]; then + file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${f/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis.nc 2>/dev/null" | wc -l ) + else + file_nb=$( eval "ls ${d}/${var_list[*]}_${f/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis.nc 2>/dev/null" | wc -l ) + fi if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then echo "CDO_MISS OK" else diff --git a/subjob_ncochunk.sh b/subjob_ncochunk.sh index 37f59326f0463d3ce5a89a9ca6425f39d060325c..b5c199e2b29145c1467772f6815554b82be3c121 100755 --- a/subjob_ncochunk.sh +++ b/subjob_ncochunk.sh @@ -7,8 +7,11 @@ set +xv date_beg=$1 date_end=$2 nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) -var_list=( uwnd vwnd uhpg vhpg tpot humi ) -var_nb=${#var_list[*]} +if [ -n "$3" ]; then + var_list=( $3 ) +else + var_list=( uwnd vwnd uhpg vhpg tpot humi ) +fi for d in $(seq -f "%02g" 0 $((nbday-1))); do @@ -40,7 +43,11 @@ while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r - if [ ${date} -eq ${date_end} ]; then fall=${f/${date}_misstodis.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis_cnk64.nc - file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fall} 2>/dev/null" | wc -l ) + if [ ${#var_list[@]} -gt 1 ]; then + file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fall} 2>/dev/null" | wc -l ) + else + file_nb=$( eval "ls ${d}/${var_list[*]}_${fall} 2>/dev/null" | wc -l ) + fi if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then echo "NCO_CHUNK OK" else