diff --git a/check_if_done.sh b/check_if_done.sh new file mode 100755 index 0000000000000000000000000000000000000000..0b23068c034d235c215fff546bef6649595f1dd8 --- /dev/null +++ b/check_if_done.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set +xv + +date_beg=$1 +date_end=$2 +nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) + +var_list=( uwnd vwnd uhpg vhpg tpot humi ) + +for d in $(seq -f "%02g" 0 $((nbday-1))); do + + date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d") + fvar="ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc" + dvar="${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}" + file_nb=$(eval "ls -l ${dvar}/${fvar} 2>/dev/null" | wc -l ) + tot_nb=$(( ${tot_nb:-0} + file_nb )) + +done + +if [ $((nbday*${#var_list[@]})) -eq ${tot_nb} ]; then echo "CHECK_IF_DONE OK"; exit 0; else echo "CHECK_IF_DONE ERROR"; exit 1; fi diff --git a/do_by_day.sh b/do_by_day.sh deleted file mode 100755 index 5e805dba283f226b7cbfb06783bbb8b70da5eee0..0000000000000000000000000000000000000000 --- a/do_by_day.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set +xv - -# ARGUMENTS -date_beg=$1 -date_end=$2 -script=$3 -script_arg=$4 - - -# TIME -year_beg=${1:0:4} -year_end=${2:0:4} -mnth_beg=${1:4:2} -mnth_end=${2:4:2} -day_beg=${1:6:2} -day_end=${2:6:2} - -if [ -z ${day_beg} ]; then - day_beg=01 - date_beg=${date_beg}${day_beg} -fi - -if [ -z ${day_end} ]; then - day_end=$( date -d "${mnth_end}/1 + 1 month - 1 day" "+%d" ) - date_end=${date_end}${day_end} -fi - -nb_day=$(( ($(date --date="${date_end}H00 + 1 day" +%s) - $(date --date="${date_beg}H00" +%s) ) / (60*60*24) )) -echo $date_beg $date_end $nb_day - -for (( d=0; d<$nb_day; d++ )); do - - date_day=$(date -d "$date_beg + $d days" +%Y%m%d) - echo $script $date_day $script_arg - ${script} ${date_day} ${script_arg} - -done - - -exit 0 diff --git a/full_prod.sh b/full_prod.sh index 222c1dab66b7e2cc95fd87141748f225d3bd834c..25ca21f4f779fd50d3f58bcbf2a2c0e6f0db2abb 100755 --- a/full_prod.sh +++ b/full_prod.sh @@ -5,12 +5,24 @@ set +xv date_beg=$1 date_end=$2 -time ./do_by_day.sh ${date_beg} ${date_end} ./subjob_mars.sh mars_inst3D_ana.req.base -time ./do_by_day.sh ${date_beg} ${date_end} ./subjob_mars.sh mars_surf2D_ana.req.base -time ./do_by_day.sh ${date_beg} ${date_end} ./subjob_cdogrid.sh -time ./do_by_day.sh ${date_beg} ${date_end} ./subjob_ablhpg.sh -time ./do_by_day.sh ${date_beg} ${date_end} ./subjob_ablvint.sh -time ./do_by_day.sh ${date_beg} ${date_end} ./subjob_cdosplit.sh -time ./do_by_day.sh ${date_beg} ${date_end} ./subjob_cdomiss.sh +if [[ ${#date_beg} -eq 6 || ${#date_end} -eq 6 ]]; then + date_beg=${date_beg:0:6}01 + date_end=$(date -d "${date_beg} + 1 month - 1 day" +"%C%y%m%d" ) +fi +echo "${date_beg}->${date_end}" + +export DIR_SRC="/home/ar0s/PROD_ERA5_ABL" +export DIR_SAVE="/ec/res5/project/mercator/ATMO_FORCINGS/ERA5_ABL50_F320_v1/daily/1h" +export SCRATCH_DIR=${SCRATCHDIR} + +./check_if_done.sh ${date_beg} ${date_end}; [ $? -eq 0 ] && exit 0 +./subjob_mars.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 +./subjob_cdogrid.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 +./subjob_ablhpg.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 +./subjob_ablvint.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 +./subjob_cdosplit.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 +./subjob_cdomiss.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 +./subjob_ncochunk.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 +./move_to_perm.sh ${date_beg} ${date_end}; [ $? -eq 1 ] && exit 1 exit 0 diff --git a/job_ablhpg.sbatch b/job_ablhpg.sbatch index e81334ee5028c11f857c22c82f807f3463ca039f..b2f345c4ce7af757475e287e4560a0f507fe7a10 100755 --- a/job_ablhpg.sbatch +++ b/job_ablhpg.sbatch @@ -9,10 +9,24 @@ module purge module load prgenv/gnu gcc/12.2.0 hpcx-openmpi/may23 hdf5-parallel/may23 netcdf4-parallel/may23 xios/trunk/r2515_gcc-xmpi.may23 module list -set -xv +set +xv -if [ -n "$1" ]; then pnml=$1; else exit 1; fi +if [ -n "$1" ]; then date=$1; else exit 1; fi -/home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/main_uvg_hpg.exe ${pnml} +p2D="${SCRATCH_DIR}/NC_F320/${date:0:6}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" +p3D="${SCRATCH_DIR}/NC_F320/${date:0:6}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" +pmsk="/ec/res5/project/mercator/ATMO_FORCINGS/ERA5_ABL50_F320_v1/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc" + +cp -av ${p2D} ${p3D} ${pmsk} $TMPDIR/. +cp -v /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/main_uvg_hpg.exe $TMPDIR/. + +sed -e "s/fsurf2D/$(basename ${p2D})/g; s/finst3D/$(basename ${p3D})/g; s/fmask/$(basename ${pmsk})/g; s/fhpg/$(basename ${p3D/inst3D/hpg})/g; s~fdir~$TMPDIR~g" < /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/namelist_abl_tools.base > $TMPDIR/namelist_abl_tools.hpg.$date + +ls -l $TMPDIR +cat $TMPDIR/namelist_abl_tools.hpg.$date + +$TMPDIR/main_uvg_hpg.exe $TMPDIR/namelist_abl_tools.hpg.$date + +mv -v $TMPDIR/$(basename ${p3D/inst3D/hpg}) $(dirname ${p3D})/. exit 0 diff --git a/job_ablvint.sbatch b/job_ablvint.sbatch index d5e168a95c9e40359a37d1f59e9f75c2c15173cf..2acae7f1b89c093be6d2783cb0569c382aea4715 100755 --- a/job_ablvint.sbatch +++ b/job_ablvint.sbatch @@ -11,8 +11,24 @@ module list set +xv -if [ -n "$1" ]; then pnml=$1; else exit 1; fi +if [ -n "$1" ]; then date=$1; else exit 1; fi -/home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/main_vinterp.exe ${pnml} +p2D="${SCRATCH_DIR}/NC_F320/${date:0:6}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" +p3D="${SCRATCH_DIR}/NC_F320/${date:0:6}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" +pmsk="/ec/res5/project/mercator/ATMO_FORCINGS/ERA5_ABL50_F320_v1/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc" +fabl=$(basename ${p3D/inst3D/abl}) ; fabl=${fabl/L107-137/L50Z10} + +cp -av ${p2D} ${p3D} ${pmsk} ${p3D/inst3D/hpg} $TMPDIR/. +cp -v /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/main_vinterp.exe $TMPDIR/. + +sed -e "s/fsurf2D/$(basename ${p2D})/g; s/finst3D/$(basename ${p3D})/g; s/fmask/$(basename ${pmsk})/g; s/fhpg/$(basename ${p3D/inst3D/hpg})/g; s/fabl/${fabl}/g; s~fdir~$TMPDIR~g" < /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/namelist_abl_tools.base > $TMPDIR/namelist_abl_tools.vint.$date +sed -i "s/ln_read_mask = .false./ln_read_mask = .true./g" $TMPDIR/namelist_abl_tools.vint.$date + +ls -l $TMPDIR +cat $TMPDIR/namelist_abl_tools.vint.$date + +$TMPDIR/main_vinterp.exe $TMPDIR/namelist_abl_tools.vint.$date + +mv -v $TMPDIR/${fabl} $(dirname ${p3D})/. exit 0 diff --git a/job_cdogrid.sbatch b/job_cdogrid.sbatch index f716b032822f622c1d51df050877fcc58ef34a87..57610f1d1115206600100cddcfbeeef6bbeeb284 100755 --- a/job_cdogrid.sbatch +++ b/job_cdogrid.sbatch @@ -13,6 +13,12 @@ set +xv if [ -n "$1" ]; then pinp=$1; else exit 1; fi if [ -n "$2" ]; then pout=$2; else exit 1; fi -cdo -v -t ecmwf -z zip -k grid -f nc4 -setgridtype,regular ${pinp} ${pout} + +cp -av ${pinp} $TMPDIR/. + +cdo -v -t ecmwf -z zip -k grid -f nc4 -setgridtype,regular $TMPDIR/$(basename ${pinp}) $TMPDIR/$(basename ${pout}) + +mkdir -pv $(dirname ${pout}) +mv -v $TMPDIR/$(basename ${pout}) ${pout} exit 0 diff --git a/job_cdomiss.sbatch b/job_cdomiss.sbatch index b13aa20c6a709975b09c03f3ad9794b89fe0593c..c1288696adccf5ddaef63516c17668a1ab915d66 100755 --- a/job_cdomiss.sbatch +++ b/job_cdomiss.sbatch @@ -12,9 +12,16 @@ module list set +xv -if [ -n "$1" ]; then pinp=$1; else exit 1; fi -if [ -n "$2" ]; then pout=$2; else exit 1; fi +if [ -n "$1" ]; then date=$1; else exit 1; fi +if [ -n "$2" ]; then var=$2; else exit 1; fi -cdo -v -P 4 -t ecmwf -z zip -k grid -f nc4 -pack -setmisstodis -setctomiss,9.9692e+36 ${pinp} ${pout} +d="${SCRATCH_DIR}/NC_F320/${date:0:6}" +f="abl_ERA5_L50Z10_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" + +cp -av ${d}/${var}_${f} $TMPDIR/. + +cdo -v -P 4 -t ecmwf -z zip -k grid -f nc4 -pack -setmisstodis -setctomiss,9.9692e+36 $TMPDIR/${var}_${f} $TMPDIR/${var}_${f%.nc}_misstodis.nc + +mv -v $TMPDIR/${var}_${f%.nc}_misstodis.nc ${d}/. exit 0 diff --git a/job_cdosplit.sbatch b/job_cdosplit.sbatch index c51dce02a49094ec2f569c3f125925f9b56e557d..c3489ee94e3c77edc3d893857d8512a67ff8354b 100755 --- a/job_cdosplit.sbatch +++ b/job_cdosplit.sbatch @@ -10,10 +10,18 @@ module list set +xv -if [ -n "$1" ]; then f=$(basename $1); d=$(dirname $1); else exit 1; fi +if [ -n "$1" ]; then date=$1; else exit 1; fi -cd ${d}; pwd +pabl="${SCRATCH_DIR}/NC_F320/${date:0:6}/abl_ERA5_L50Z10_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" +var_list=( uwnd vwnd uhpg vhpg tpot humi ) -cdo -v splitname,swap ${f} _${f%.nc} +cp -a ${pabl} ${TMPDIR}/. + +cd $TMPDIR; pwd +cdo -v splitname,swap $(basename ${pabl}) _$(basename ${pabl%.nc}) + +ls -l $TMPDIR +#mv -v $TMPDIR/{$(tr ' ' ',' <<< ${var_list[*]})}_$(basename ${pabl}) $(dirname ${pabl})/. +mv -v $TMPDIR/????_$(basename ${pabl}) $(dirname ${pabl})/. exit 0 diff --git a/job_mars.sbatch b/job_mars.sbatch new file mode 100755 index 0000000000000000000000000000000000000000..50c231ba8644314cf538945cae8a9af4e0d50528 --- /dev/null +++ b/job_mars.sbatch @@ -0,0 +1,19 @@ +#!/bin/bash +#SBATCH --job-name=marsreq +#SBATCH --output=JOBS/%x-%j.out +#SBATCH --qos=nf +#SBATCH --time=00:30:00 + +set +xv + +file_req=$1 +date=$2 + +sed -e "s~date_req~${date}~g; s~dir_out~$TMPDIR~g" < ${file_req} > $TMPDIR/$(basename ${file_req%.req.base}_${date}.req) + +mars $TMPDIR/$(basename ${file_req%.req.base}_${date}.req) + +mkdir -pv ${SCRATCH_DIR}/GRIB_N320/${date:0:6} +mv -v $TMPDIR/*_${date}.grib ${SCRATCH_DIR}/GRIB_N320/${date:0:6} + +exit 0 diff --git a/job_ncochunk.sbatch b/job_ncochunk.sbatch new file mode 100755 index 0000000000000000000000000000000000000000..aa6464aba9f0a64918ac9be8a33ce624ff9b6aa5 --- /dev/null +++ b/job_ncochunk.sbatch @@ -0,0 +1,27 @@ +#!/bin/bash +#SBATCH --job-name=ncochunk +#SBATCH --output=JOBS/%x-%j.out +#SBATCH --qos=nf +#SBATCH --time=00:30:00 +#SBATCH --cpus-per-task=4 + + +module purge +module load nco/4.9.7 +module list + +set +xv + +if [ -n "$1" ]; then date=$1; else exit 1; fi +if [ -n "$2" ]; then var=$2; else exit 1; fi + +d="${SCRATCH_DIR}/NC_F320/${date:0:6}" +f="${var}_abl_ERA5_L50Z10_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}_misstodis.nc" + +cp -av ${d}/${f} $TMPDIR/. + +ncks --4 --dfl_lvl 1 --no_abc -v ${var} --cnk_plc=xpl --cnk_dmn lon,64 --cnk_dmn lat,64 --cnk_dmn time,1 --cnk_dmn jpka,1 $TMPDIR/${f} $TMPDIR/${f%.nc}_cnk64.nc + +mv -v $TMPDIR/${f%.nc}_cnk64.nc ${d}/. + +exit 0 diff --git a/move_to_perm.sh b/move_to_perm.sh new file mode 100755 index 0000000000000000000000000000000000000000..3e8c69fa45f8775abaf79a7719c2ae458ece685a --- /dev/null +++ b/move_to_perm.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set +xv + +# ARGUMENTS +date_beg=$1 +date_end=$2 +nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) +var_list=( uwnd vwnd uhpg vhpg tpot humi ) +var_nb=${#var_list[*]} + +for d in $(seq -f "%02g" 0 $((nbday-1))); do + + # TIME + date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d") + ym=${date:0:6} + + # DIRECTORIES & FILES + d="${SCRATCH_DIR}/NC_F320/${ym}" + f="abl_ERA5_L50Z10_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}_misstodis_cnk64.nc" + + for v in ${var_list[@]}; do + + finp=${v}_${f} + fout="ERA5_L50Z10_${v^^}_1H_extrap_y${date:0:4}m${date:4:2}d${date:6:2}.nc" + + if [ ! -s ${DIR_SAVE}/${v^^}/${fout} ]; then + if [ -s ${d}/${finp} ]; then + mkdir -pv ${DIR_SAVE}/${v^^} + mv -v ${d}/${finp} ${DIR_SAVE}/${v^^}/${fout} + else + echo "MOVE_TO_PERM ERROR: input files not found"; exit 1 + fi + fi + + done + + if [ ${date} -eq ${date_end} ]; then + file_nb=$( eval "ls ${DIR_SAVE}/{$(tr ' ' ',' <<< ${var_list[*]^^})}/ERA5_L50Z10_{$(tr ' ' ',' <<< ${var_list[*]^^})}_1H_extrap_y${date:0:4}m${date:4:2}d{${date_beg:6:2}..${date_end:6:2}}.nc 2>/dev/null" | wc -l ) + if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then + echo "MOVE_TO_PERM OK" + eval "rm -v ${SCRATCH_DIR}/NC_F320/${ym}/*{${date_beg}..${date_end}}*.nc" + eval "rm -v ${SCRATCH_DIR}/GRIB_F320/${ym}/*{${date_beg}..${date_end}}*.grib" + echo "CLEANING OK" + exit 0 + else + echo "MOVE_TO_PERM ERROR" + exit 1 + fi + fi + +done + +exit 0 diff --git a/subjob_ablhpg.sh b/subjob_ablhpg.sh index 1345b8403ce1bc4dbd5925f52dae762350d3ff6c..4266b73834b708bb7fb97297fe3d3b3ff2a60a36 100755 --- a/subjob_ablhpg.sh +++ b/subjob_ablhpg.sh @@ -2,37 +2,49 @@ set +xv -date=$1 -day_end=$( date -d "${date:4:2}/1 + 1 month - 1 day" +"%d" ) -date_end=${date:0:6}${day_end} -ym=${date:0:6} -day_nb=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${ym}01H00" +"%s") )/(60*60*24) )) - -p2D="/ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" -p3D="/ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" -pmsk="/ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc" -ln -sf /ec/res4/scratch/ar0s/ERA5/NC_F320/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc /ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}/. - -f2D=$(basename $p2D) ; d2D=$(dirname $p2D) -f3D=$(basename $p3D) ; d3D=$(dirname $p3D) -fmsk=$(basename $pmsk) ; dmsk=$(dirname $pmsk) -fhpg=${f3D/inst3D/hpg} - -if [ ! -s ${d2D}/${fhpg} ]; then - - if [[ -s ${p2D} && -s ${p3D} ]]; then - sed -e "s/fsurf2D/${f2D}/g; s/finst3D/${f3D}/g; s/fmask/${fmsk}/g; s/fhpg/${fhpg}/g; s~fdir~${d2D}~g" < /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/namelist_abl_tools.base > /home/ar0s/PROD_ERA5_ABL/JOBS/namelist_abl_tools.hpg.$date - while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done - sbatch --job-name=ablhpg_${date} /home/ar0s/PROD_ERA5_ABL/job_ablhpg.sbatch /home/ar0s/PROD_ERA5_ABL/JOBS/namelist_abl_tools.hpg.$date - else - echo "ERROR: input files not found"; exit 1 - fi -fi +# ARGUMENTS +date_beg=$1 +date_end=$2 +nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) + +for d in $(seq -f "%02g" 0 $((nbday-1))); do + + # TIME + date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d") + ym=${date:0:6} + + # DIRECTORIES & FILES + p2D="${SCRATCH_DIR}/NC_F320/${ym}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" + p3D="${SCRATCH_DIR}/NC_F320/${ym}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" + pmsk="${SCRATCH_DIR}/NC_F320/${ym}/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc" + f2D=$(basename $p2D) ; d2D=$(dirname $p2D) + f3D=$(basename $p3D) ; d3D=$(dirname $p3D) + fmsk=$(basename $pmsk) ; dmsk=$(dirname $pmsk) + fhpg=${f3D/inst3D/hpg} + + if [ ! -s ${d2D}/${fhpg} ]; then + + if [[ -s ${p2D} && -s ${p3D} ]]; then + while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done + sbatch --job-name=ablhpg_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_ablhpg.sbatch ${date} + else + echo "ABL_HPG ERROR: input files not found"; exit 1 + fi + + fi + + while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep ablhpg_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done + + if [ ${date} -eq ${date_end} ]; then + file_nb=$( eval "ls ${d2D}/${fhpg/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l ) + if [ ${nbday} -eq ${file_nb} ]; then + echo "ABL_HPG OK" + else + echo "ABL_HPG ERROR"; exit 1 + fi + fi -while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep ablhpg_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done +done -if [ ${date} -eq ${date_end} ]; then - file_nb=$( eval "ls ${d2D}/${fhpg/${date}/${ym}??} 2>/dev/null" | wc -l ) - if [ ${day_nb} -eq ${file_nb} ]; then exit 0; else echo "ERROR"; exit 1; fi -fi +exit 0 diff --git a/subjob_ablvint.sh b/subjob_ablvint.sh index 332eac84c64f8aead1d8157fc8ecc147ffcfc116..5939ead906e5ff86ec3daf22f90e6ac4ded5750e 100755 --- a/subjob_ablvint.sh +++ b/subjob_ablvint.sh @@ -2,39 +2,51 @@ set +xv -date=$1 -day_end=$( date -d "${date:4:2}/1 + 1 month - 1 day" +"%d" ) -date_end=${date:0:6}${day_end} -ym=${date:0:6} -day_nb=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${ym}01H00" +"%s") )/(60*60*24) )) - -p2D="/ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" -p3D="/ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" -pmsk="/ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc" -ln -sf /ec/res4/scratch/ar0s/ERA5/NC_F320/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc /ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}/. - -f2D=$(basename $p2D) ; d2D=$(dirname $p2D) -f3D=$(basename $p3D) ; d3D=$(dirname $p3D) -fmsk=$(basename $pmsk) ; dmsk=$(dirname $pmsk) -fhpg=${f3D/inst3D/hpg} -fabl=${f3D/inst3D/abl} ; fabl=${fabl/L107-137/L50Z10} - -if [ ! -s ${d2D}/${fabl} ]; then - - if [[ -s ${p2D} && -s ${p3D} && -s ${d2D}/${fhpg} ]]; then - sed -e "s/fsurf2D/${f2D}/g; s/finst3D/${f3D}/g; s/fmask/${fmsk}/g; s/fhpg/${fhpg}/g; s/fabl/${fabl}/g; s~fdir~${d2D}~g" < /home/ar0s/SRC/NEMO/main/tools/ABL_TOOLS/namelist_abl_tools.base > /home/ar0s/PROD_ERA5_ABL/JOBS/namelist_abl_tools.vint.$date - sed -i "s/ln_read_mask = .false./ln_read_mask = .true./g" /home/ar0s/PROD_ERA5_ABL/JOBS/namelist_abl_tools.vint.$date - while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done - sbatch --job-name=ablvint_${date} /home/ar0s/PROD_ERA5_ABL/job_ablvint.sbatch /home/ar0s/PROD_ERA5_ABL/JOBS/namelist_abl_tools.vint.$date - else - echo "ERROR: input files not found"; exit 1 - fi -fi +# ARGUMENTS +date_beg=$1 +date_end=$2 +nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) + +for d in $(seq -f "%02g" 0 $((nbday-1))); do + + # TIME + date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d") + ym=${date:0:6} + + # DIRECTORIES & FILES + p2D="${SCRATCH_DIR}/NC_F320/${ym}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" + p3D="${SCRATCH_DIR}/NC_F320/${ym}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" + pmsk="${SCRATCH_DIR}/NC_F320/${ym}/mask0_ERA5_GLO_TLAUTO_IGNONE_GF320_NNint_20120101.nc" + f2D=$(basename $p2D) ; d2D=$(dirname $p2D) + f3D=$(basename $p3D) ; d3D=$(dirname $p3D) + fmsk=$(basename $pmsk) ; dmsk=$(dirname $pmsk) + fhpg=${f3D/inst3D/hpg} + fabl=${f3D/inst3D/abl} ; fabl=${fabl/L107-137/L50Z10} + + if [ ! -s ${d2D}/${fabl} ]; then + + if [[ -s ${p2D} && -s ${p3D} && -s ${d2D}/${fhpg} ]]; then + while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done + sbatch --job-name=ablvint_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_ablvint.sbatch ${date} + else + echo "ABL_VINT ERROR: input files not found"; exit 1 + fi + + fi + + while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep ablvint_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done + + if [ ${date} -eq ${date_end} ]; then + file_nb=$( eval "ls ${d2D}/${fabl/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l ) + if [ ${nbday} -eq ${file_nb} ]; then + echo "ABL_VINT OK" + else + echo "ABL_VINT ERROR" + exit 1 + fi + fi -while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep ablvint_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done +done -if [ ${date} -eq ${date_end} ]; then - file_nb=$( eval "ls ${d2D}/${fabl/${date}/${ym}??} 2>/dev/null" | wc -l ) - if [ ${day_nb} -eq ${file_nb} ]; then exit 0; else echo "ERROR"; exit 1; fi -fi +exit 0 diff --git a/subjob_cdogrid.sh b/subjob_cdogrid.sh index db938b9f52b7bd07ba1873d1cc3bb037554c7e4c..0e60598571db407e01b0d9c6c436a2a4e9d348f6 100755 --- a/subjob_cdogrid.sh +++ b/subjob_cdogrid.sh @@ -2,47 +2,69 @@ set +xv -date=$1 -day_end=$( date -d "${date:4:2}/1 + 1 month - 1 day" +"%d" ) -date_end=${date:0:6}${day_end} -ym=${date:0:6} -day_nb=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${ym}01H00" +"%s") )/(60*60*24) )) - -p2D="/ec/res4/scratch/ar0s/ERA5/GRIB_N320/${ym}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GN320_ana01h_${date}.grib" -p3D="/ec/res4/scratch/ar0s/ERA5/GRIB_N320/${ym}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GN320_ana01h_${date}.grib" -dout="/ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}" ; mkdir -p ${dout} - -f2D=$(basename ${p2D/GN320/GF320reg}) ; f2D=${f2D%.grib}.nc -f3D=$(basename ${p3D/GN320/GF320reg}) ; f3D=${f3D%.grib}.nc -d2D=$(dirname $p2D) ; d3D=$(dirname $p3D) - -if [ ! -s ${dout}/${f2D} ]; then - - if [ -s ${p2D} ]; then - while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done - sbatch --job-name=cdogrid2D_${date} /home/ar0s/PROD_ERA5_ABL/job_cdogrid.sbatch ${p2D} ${dout}/${f2D} - else - echo "ERROR: input files not found"; exit 1 - fi -fi +# ARGUMENTS +date_beg=$1 +date_end=$2 +nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) -if [ ! -s ${dout}/${f3D} ]; then +for d in $(seq -f "%02g" 0 $((nbday-1))); do - if [ -s ${p3D} ]; then - while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done - sbatch --job-name=cdogrid3D_${date} /home/ar0s/PROD_ERA5_ABL/job_cdogrid.sbatch ${p3D} ${dout}/${f3D} - else - echo "ERROR: input files not found"; exit 1 - fi + # TIME + date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d") + ym=${date:0:6} + + # DIRECTORIES & FILES + p2D="${SCRATCH_DIR}/GRIB_N320/${ym}/surf2D_ERA5_L1_GLO_TLAUTO_IGNONE_GN320_ana01h_${date}.grib" + p3D="${SCRATCH_DIR}/GRIB_N320/${ym}/inst3D_ERA5_L107-137_GLO_TLAUTO_IGNONE_GN320_ana01h_${date}.grib" + dout="${SCRATCH_DIR}/NC_F320/${ym}" ; mkdir -p ${dout} + f2D=$(basename ${p2D/GN320/GF320reg}) ; f2D=${f2D%.grib}.nc + f3D=$(basename ${p3D/GN320/GF320reg}) ; f3D=${f3D%.grib}.nc + d2D=$(dirname $p2D) ; d3D=$(dirname $p3D) -fi + if [ ! -s ${dout}/${f2D} ]; then + + mkdir -pv ${dout} + + if [ -s ${p2D} ]; then + while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done + sbatch --job-name=cdogrid2D_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_cdogrid.sbatch ${p2D} ${dout}/${f2D} + else + echo "CDO_GRID2D ERROR: input files not found"; exit 1 + fi + + fi + + if [ ! -s ${dout}/${f3D} ]; then + + mkdir -pv ${dout} + + if [ -s ${p3D} ]; then + while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done + sbatch --job-name=cdogrid3D_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_cdogrid.sbatch ${p3D} ${dout}/${f3D} + else + echo "CDO_GRID3D ERROR: input files not found"; exit 1 + fi + + fi + + while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep cdogrid.D_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done + + if [ ${date} -eq ${date_end} ]; then + file_nb=$( eval "ls ${dout}/${f2D/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l ) + if [ ${nbday} -eq ${file_nb} ]; then + echo "CDO_GRID 2D OK" + else + echo "ERROR CDO_GRID 2D"; exit 1 + fi + file_nb=$( eval "ls ${dout}/${f3D/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l ) + if [ ${nbday} -eq ${file_nb} ]; then + echo "CDO_GRID 3D OK" + else + echo "ERROR CDO_GRID 3D"; exit 1 + fi + fi -while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep cdogrid.D_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done +done -if [ ${date} -eq ${date_end} ]; then - file_nb=$( eval "ls ${dout}/${f2D/${date}/${ym}??} 2>/dev/null" | wc -l ) - if [ ${day_nb} -eq ${file_nb} ]; then echo "2D OK"; else echo "ERROR 2D"; exit 1; fi - file_nb=$( eval "ls ${dout}/${f3D/${date}/${ym}??} 2>/dev/null" | wc -l ) - if [ ${day_nb} -eq ${file_nb} ]; then echo "3D OK"; exit 0; else echo "ERROR 3D"; exit 1; fi -fi +exit 0 diff --git a/subjob_cdomiss.sh b/subjob_cdomiss.sh index 65e5a6f32d75a82842f6c73b9912f7fa38eb7b24..45bcd696893eeddbbe6d645a425eebb620bf44ec 100755 --- a/subjob_cdomiss.sh +++ b/subjob_cdomiss.sh @@ -2,35 +2,51 @@ set +xv -date=$1 -day_end=$( date -d "${date:4:2}/1 + 1 month - 1 day" +"%d" ) -date_end=${date:0:6}${day_end} -ym=${date:0:6} -day_nb=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${ym}01H00" +"%s") )/(60*60*24) )) +# ARGUMENTS +date_beg=$1 +date_end=$2 +nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) var_list=( uwnd vwnd uhpg vhpg tpot humi ) -d="/ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}" -fabl="abl_ERA5_L50Z10_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" -for v in ${var_list[@]}; do - - finp=${v}_${fabl} - fout=${finp%.nc}_misstodis.nc - - if [ ! -s ${d}/${fout} ]; then - if [ -s ${d}/${finp} ]; then - while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done - sbatch --job-name=cdomiss_${date} /home/ar0s/PROD_ERA5_ABL/job_cdomiss.sbatch ${d}/${finp} ${d}/${fout} +for d in $(seq -f "%02g" 0 $((nbday-1))); do + + # TIME + date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d") + ym=${date:0:6} + + # DIRECTORIES & FILES + d="${SCRATCH_DIR}/NC_F320/${ym}" + f="abl_ERA5_L50Z10_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" + + for v in ${var_list[@]}; do + + finp=${v}_${f} + fout=${finp%.nc}_misstodis.nc + + if [ ! -s ${d}/${fout} ]; then + if [ -s ${d}/${finp} ]; then + while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done + sbatch --job-name=cdomiss_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} ${DIR_SRC}/job_cdomiss.sbatch ${date} ${v} + else + echo "CDO_MISS ERROR: input files not found"; exit 1 + fi + fi + + done + + while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep cdomiss_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done + + if [ ${date} -eq ${date_end} ]; then + file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${f/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis.nc 2>/dev/null" | wc -l ) + if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then + echo "CDO_MISS OK" else - echo "ERROR: input files not found"; exit 1 + echo "CDO_MISS ERROR" + exit 1 fi fi done -while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep cdomiss_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done - -if [ ${date} -eq ${date_end} ]; then - file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fabl/${date}/${ym}??} 2>/dev/null" | wc -l ) - if [ $((day_nb*${#var_list[@]})) -eq ${file_nb} ]; then echo "OK"; exit 0; else echo "ERROR"; exit 1; fi -fi +exit 0 diff --git a/subjob_cdosplit.sh b/subjob_cdosplit.sh index 10667ddde4898ca284b25d59a2ec80233e01d5f8..14cc16d4e31d0b86fec166841941bc7cb854e970 100755 --- a/subjob_cdosplit.sh +++ b/subjob_cdosplit.sh @@ -6,36 +6,52 @@ set +xv -date=$1 -day_end=$( date -d "${date:4:2}/1 + 1 month - 1 day" +"%d" ) -date_end=${date:0:6}${day_end} -ym=${date:0:6} -day_nb=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${ym}01H00" +"%s") )/(60*60*24) )) - -pabl="/ec/res4/scratch/ar0s/ERA5/NC_F320/${ym}/abl_ERA5_L50Z10_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" -fabl=$(basename $pabl) -dabl=$(dirname $pabl) +# ARGUMENTS +date_beg=$1 +date_end=$2 +nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) #var_list=( $(cdo showname ${fabl}) ) #var_list=( ${var_list[*]/"lsm"} ) var_list=( uwnd vwnd uhpg vhpg tpot humi ) var_nb=${#var_list[*]} -file_nb=$( eval "ls ${dabl}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fabl} 2>/dev/null" | wc -l ) - -if [ ${file_nb} -lt ${var_nb} ]; then - if [ -s ${pabl} ]; then - while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done - sbatch --job-name=cdosplit_${date} /home/ar0s/PROD_ERA5_ABL/job_cdosplit.sbatch $pabl - else - echo "ERROR: input files not found"; exit 1 +for d in $(seq -f "%02g" 0 $((nbday-1))); do + + # TIME + date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d") + ym=${date:0:6} + + # DIRECTORIES & FILES + pabl="${SCRATCH_DIR}/NC_F320/${ym}/abl_ERA5_L50Z10_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" + fabl=$(basename $pabl) + dabl=$(dirname $pabl) + + file_nb=$( eval "ls ${dabl}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fabl} 2>/dev/null" | wc -l ) + + if [ ${file_nb} -lt ${var_nb} ]; then + + if [ -s ${pabl} ]; then + while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done + sbatch --job-name=cdosplit_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} ${DIR_SRC}/job_cdosplit.sbatch ${date} + else + echo "CDO_SPLIT ERROR: input files not found"; exit 1 + fi + + fi + + while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep cdosplit_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done + + if [ ${date} -eq ${date_end} ]; then + file_nb=$( eval "ls ${dabl}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fabl/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l ) + if [ $((nbday*var_nb)) -eq ${file_nb} ]; then + echo "CDO_SPLIT OK" + else + echo "CDO_SPLIT ERROR" + exit 1 + fi fi -fi - -while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep cdosplit_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done +done -if [ ${date} -eq ${date_end} ]; then - file_nb=$( eval "ls ${dabl}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fabl/${date}/${ym}??} 2>/dev/null" | wc -l ) - if [ $((day_nb*var_nb)) -eq ${file_nb} ]; then echo "OK"; exit 0; else echo "ERROR"; exit 1; fi -fi +exit 0 diff --git a/subjob_mars.sh b/subjob_mars.sh index c9125855a74934521b5c2edd0dc1b863d8ae7f0b..781dbf68420962eb65764771fd0cc85c2fb50d10 100755 --- a/subjob_mars.sh +++ b/subjob_mars.sh @@ -3,51 +3,52 @@ set +xv -date=${1} -day_end=$( date -d "${date:4:2}/1 + 1 month - 1 day" +"%d" ) -date_end=${date:0:6}${day_end} -ym=${date:0:6} -day_nb=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${ym}01H00" +"%s") )/(60*60*24) )) - -dir_out="/ec/res4/scratch/ar0s/ERA5/GRIB_N320/${ym}"; mkdir -p ${dir_out} -dir_req=$(dirname ${2}:-".") -file_req=$(basename ${2%*.req*}) -req_type=${file_req/mars_} ; req_type=${req_type%%_*} -file_out="${req_type}_ERA5_L*_GLO_TLAUTO_IGNONE_GN320_ana01h_${date}.grib" - -if [ ! -s ${dir_out}/${file_out} ]; then - - # EDIT MARS REQ - if [ -f ${dir_req}/${file_req}.req.base ]; then - sed -e "s~date_req~${date}~g; s~dir_out~${dir_out}~g" < ${dir_req}/${file_req}.req.base > ${dir_req}/JOBS/${file_req}_${date}.req - else - echo "MARS REQUEST NOT FOUND !" - exit 1 +# ARGUMENTS +date_beg=$1 +date_end=$2 +nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) + +for d in $(seq -f "%02g" 0 $((nbday-1))); do + + # TIME + date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d") + ym=${date:0:6} + + # DIRECTORIES & FILES + dout="${SCRATCH_DIR}/GRIB_N320/${ym}" + dreg="${SCRATCH_DIR}/NC_F320/${ym}" + f3D="inst3D_ERA5_L*_GLO_TLAUTO_IGNONE_GN320_ana01h_${date}.grib" + f2D="surf2D_ERA5_L*_GLO_TLAUTO_IGNONE_GN320_ana01h_${date}.grib" + n3D="inst3D_ERA5_L*_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" + n2D="surf2D_ERA5_L*_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}.nc" + + # SUBMIT JOB + if [ ! -s ${dout}/${f3D} ] && [ ! -s ${dreg}/${n3D} ]; then + while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done + sbatch --job-name=mars_inst3D_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} ${DIR_SRC}/job_mars.sbatch ${DIR_SRC}/mars_inst3D_ana.req.base ${date} fi - # SUBMIT MARS REQUEST - while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done - - sbatch --job-name=mars_${req_type}_${date} <<EOF -#!/bin/bash -#SBATCH --job-name=marsreq -#SBATCH --output=JOBS/%x-%j.out -#SBATCH --qos=nf -#SBATCH --time=00:30:00 - -set +xv + if [ ! -s ${dout}/${f2D} ] && [ ! -s ${dreg}/${n2D} ]; then + while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done + sbatch --job-name=mars_surf2D_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} ${DIR_SRC}/job_mars.sbatch ${DIR_SRC}/mars_surf2D_ana.req.base ${date} + fi -mars ${dir_req}/JOBS/${file_req}_${date}.req + # WAIT FOR THE LAST DATE TO FINISH + while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep mars_.*_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done -exit 0 + # TEST ALL FILES ARE PRODUCED + if [ ${date} -eq ${date_end} ]; then + + fnb=$( eval "ls ${dout}/${f3D/${date}.grib/${ym}{${date_beg:6:2}..${date_end:6:2}}}.grib 2>/dev/null" | wc -l ) + nnb=$( eval "ls ${dreg}/${n3D/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l ) + if [[ ${nbday} -eq ${fnb} || ${nbday} -eq ${nnb} ]]; then echo "MARS INST3D OK"; else echo "MARS INST3D ERROR"; exit 1; fi -EOF + fnb=$( eval "ls ${dout}/${f2D/${date}.grib/${ym}{${date_beg:6:2}..${date_end:6:2}}}.grib 2>/dev/null" | wc -l ) + nnb=$( eval "ls ${dreg}/${n2D/${date}.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}.nc 2>/dev/null" | wc -l ) + if [[ ${nbday} -eq ${fnb} || ${nbday} -eq ${nnb} ]]; then echo "MARS SURF2D OK"; else echo "MARS SURF2D ERROR"; exit 1; fi -fi + fi -while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep mars_${req_type}_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done +done -if [ ${date} -eq ${date_end} ]; then - file_nb=$( eval "ls ${dir_out}/${file_out/${date}/${ym}??} 2>/dev/null" | wc -l ) - if [ ${day_nb} -eq ${file_nb} ]; then echo "OK"; exit 0; else echo "ERROR"; exit 1; fi -fi +exit 0 diff --git a/subjob_ncochunk.sh b/subjob_ncochunk.sh new file mode 100755 index 0000000000000000000000000000000000000000..ff0495f52bf70fa2467942e7f09386d13a543316 --- /dev/null +++ b/subjob_ncochunk.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set +xv + + +# ARGUMENTS +date_beg=$1 +date_end=$2 +nbday=$(( ($(date -d "${date_end}H00 + 1 day" +%s) - $(date -d "${date_beg}H00" +"%s") )/(60*60*24) )) +var_list=( uwnd vwnd uhpg vhpg tpot humi ) +var_nb=${#var_list[*]} + +for d in $(seq -f "%02g" 0 $((nbday-1))); do + + # TIME + date=$(date -d "${date_beg}H00 + ${d} days" +"%C%y%m%d") + ym=${date:0:6} + + # DIRECTORIES & FILES + d="${SCRATCH_DIR}/NC_F320/${ym}" + f="abl_ERA5_L50Z10_GLO_TLAUTO_IGNONE_GF320reg_ana01h_${date}_misstodis.nc" + +for v in ${var_list[@]}; do + + finp=${v}_${f} + fout=${finp%.nc}_cnk64.nc + + if [ ! -s ${d}/${fout} ]; then + if [ -s ${d}/${finp} ]; then + while [ $(squeue -u $USER -h -t pending,running -r | wc -l) -eq 90 ]; do echo "sleeping..."; sleep 10; done + sbatch --job-name=ncochunk_${date} --export=SCRATCH_DIR=${SCRATCH_DIR} /home/ar0s/PROD_ERA5_ABL/job_ncochunk.sbatch ${date} ${v} + else + echo "NCO_CHUNK ERROR: input files not found"; exit 1 + fi + fi + +done + +while [[ ${date} -eq ${date_end} && $(squeue -u $USER -h -t pending,running -r -o '%20j' | grep cdomiss_${ym} | wc -l) -gt 0 ]]; do echo "sleeping..."; sleep 10; done + +if [ ${date} -eq ${date_end} ]; then + fall=${f/${date}_misstodis.nc/${ym}{${date_beg:6:2}..${date_end:6:2}}}_misstodis_cnk64.nc + file_nb=$( eval "ls ${d}/{$(tr ' ' ',' <<< ${var_list[*]})}_${fall} 2>/dev/null" | wc -l ) + if [ $((nbday*${#var_list[@]})) -eq ${file_nb} ]; then + echo "NCO_CHUNK OK" + else + echo "NCO_CHUNK ERROR" + exit 1 + fi +fi + +done + +exit 0