From 0bd4e53e5437546793b525208063a13b32a58a8d Mon Sep 17 00:00:00 2001 From: Alexander Mahura Date: Tue, 10 Oct 2023 20:24:58 +0300 Subject: [PATCH] add updated scripts for SYNOP, STATS, request.yml, run_obsall.py --- .../produce_rank_histograms_all_stations.sh | 6 +- .../produce_standard_plots_all_stations.sh | 2 +- SYNOP/gsv_mod_data.sh | 2 +- SYNOP/main_synop.sh | 161 +++++++++++------- SYNOP/synop_mod.sh | 2 +- SYNOP/synop_obs.sh | 2 + SYNOP/synop_stats.sh | 1 + request_synop.yml | 39 +++++ run_obsall.py | 27 +++ 9 files changed, 172 insertions(+), 70 deletions(-) create mode 100644 request_synop.yml create mode 100644 run_obsall.py diff --git a/SYNOP/STATS/produce_rank_histograms_all_stations.sh b/SYNOP/STATS/produce_rank_histograms_all_stations.sh index 29ea15fff..48a2d8c39 100755 --- a/SYNOP/STATS/produce_rank_histograms_all_stations.sh +++ b/SYNOP/STATS/produce_rank_histograms_all_stations.sh @@ -73,11 +73,11 @@ month2="${5:-12}" # number of years. Otherwise, the number of months is recorded. # let nmonths=12*nyears+month2-month1-11 -echo 'Number of months' $nmonths +echo ' Number of months' $nmonths # n=$( echo "$nmonths / 12" | bc -l ) nyears_round=$( echo "($n + 0.5) / 1" | bc ) -echo 'Number of years rounded' ${nyears_round} +echo ' Number of years rounded' ${nyears_round} # if [ $nmonths -ge 12 ] then @@ -199,7 +199,7 @@ echo " Synop station ID, longitude, latitude: ${station} ${longitude} ${latitu ################################################################ # Select the simulation data for the station (mimicked this far by observations!). # When simulation data are available, a means to retrieve the correct data points from it must be designed -echo " Selecting the simulation data (mimicked his far by observations) for synop station: ${station}" +echo " Selecting the simulation data (mimicked this far by observations) for synop station: ${station}" # odb_command="odb sql 'select year,month,day,hour,value where station=${station} and (year>=${year1} and year<=${year2}) and (hour=0 or hour=6 or hour=12 or hour=18) and variable=${variable}' -i ${sim_file} -o sim_data" eval ${odb_command} diff --git a/SYNOP/STATS/produce_standard_plots_all_stations.sh b/SYNOP/STATS/produce_standard_plots_all_stations.sh index b45052afc..59546a8e6 100755 --- a/SYNOP/STATS/produce_standard_plots_all_stations.sh +++ b/SYNOP/STATS/produce_standard_plots_all_stations.sh @@ -205,7 +205,7 @@ echo " Synop station ID, longitude, latitude: ${station} ${longitude} ${latitu ################################################################ # Select the simulation data for the station (mimicked this far by observations!) # When simulation data are available, a means to retrieve the correct data points from it must be designed. -echo " Selecting the simulation data (mimicked his far by observations) for synop station: ${station}" +echo " Selecting the simulation data (mimicked this far by observations) for synop station: ${station}" # odb_command="odb sql 'select year,month,day,hour,value where station=${station} and (year>=${year1} and year<=${year2}) and (hour=0 or hour=6 or hour=12 or hour=18) and variable=${variable}' -i ${sim_file} -o sim_data" eval ${odb_command} diff --git a/SYNOP/gsv_mod_data.sh b/SYNOP/gsv_mod_data.sh index a819e5e63..e711b932d 100755 --- a/SYNOP/gsv_mod_data.sh +++ b/SYNOP/gsv_mod_data.sh @@ -69,7 +69,7 @@ done rm $tmp_file_list_timestamp ###############tmp test -rm -rf 20200120* +### rm -rf 20200120* exit diff --git a/SYNOP/main_synop.sh b/SYNOP/main_synop.sh index 522e12ebb..03005c1d4 100755 --- a/SYNOP/main_synop.sh +++ b/SYNOP/main_synop.sh @@ -1,30 +1,47 @@ #!/bin/bash echo "Bash version ${BASH_VERSION}..." ######################################################################## -# Author: Alexander Mahura : 2023-08-03 +# Author: Alexander Mahura - 2023-08-03 +# UPDATED: +# 2023-08; 2023-09; ######################################################################## - +# +# NEEDED MODULES LOADED ... +# module load LUMI/23.03 +# module load partition/C +# module load PrgEnv-gnu +# module load odb_api/0.18.1-cpeCray-23.03.lua +# module load python-climatedt/3.11.3-cpeCray-23.03.lua +# NOTE: +# these are part of /appl/AS/AUTOSUBMIT_DATA/expid/proj/workflow/lib/LUMI/config.sh +# +######################################################################### # PRE-SETUP (TESTING FOR LIMITED ODB DATASET & LIMITED GEOGRAPHICAL AREA) # FOR AIR TEMPERATURE AT 2 METRE # ANY OTHER METEOROLOGICAL PARAMETER (FROM SYNOP LISTED) CAN BE ADDED -echo "PRE-SETUP :" -echo "TESTING FOR LIMITED ODB DATASET & LIMITED GEOGRAPHICAL AREA" -echo "FOR AIR TEMPERATURE AT 2 METRE" +echo "====================================================================" +echo " DestinE Digital Twin --- OBSALL Apps for SYNOP OBS DATA:" +echo " TESTING/ RUNING FOR: " +echo " -- LIMITED SYNOP OBS DATA ODB DATASET" +echo " -- LIMITED GEOGRAPHICAL AREA" +echo " -- FOR AIR TEMPERATURE AT 2 METRE" +echo "====================================================================" +echo " " varMETnum="39" varMETstr="2t" -lon_val_min="21" #20 -lon_val_max="23" #25 -lat_val_min="61" #60 -lat_val_max="63" #65 +lon_val_min="20" #20 +lon_val_max="25" #25 +lat_val_min="60" #60 +lat_val_max="65" #65 echo "varMETnum varMETstr lon_val_min lon_val_max lat_val_min lat_val_max:" echo "$varMETnum $varMETstr $lon_val_min $lon_val_max $lat_val_min $lat_val_max" -echo "=================================================================" -echo "STEP 0 - PRE-CHEKING EXISTANCE OF NECESSARY DIRECTORIES ..." -echo "=================================================================" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo " STEP #0 : PRE-CHEKING EXISTANCE OF NECESSARY DIRECTORIES ..." +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" pwd echo " Directory for pre-processing mod data extracted with gsv ..." inp_dir_gsvmoddata="GSVMODDATA" @@ -70,19 +87,25 @@ else mkdir $inp_dir_graphs fi +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo " ..... STEP 0 - COMPLETED ....." +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo " " -echo "=================================================================" -echo "STEP 1 - PRE-PROCESSING MOD DATA EXTRACTED WITH GSV_INTERFACE ..." -echo "=================================================================" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo " STEP #1 : PRE-PROCESSING MOD DATA EXTRACTED WITH GSV_INTERFACE ..." +echo " SCRIPT --- gsv_mod_data.sh" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" ./gsv_mod_data.sh -echo "..... STEP 1 - COMPLETED ....." +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo " ..... STEP 1 - COMPLETED ....." +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo "=================================================================" +echo "====================================================================" echo " SYNOP : START LOOP OVER ALL AVAILABLE TIME-SLICES " -echo "=================================================================" +echo "====================================================================" # List all available splitted files (by time-slices) in dir GSVMODDATA/ # Define YYYY MM DD HH to start calculations for time-slices @@ -103,29 +126,31 @@ do #echo "b_yy, b_mm, b_dd, b_hh : $b_yy $b_mm $b_dd $b_hh" #tail -n +2 "$tmp_file_tstamps" > "$tmp_file_tstamps.tmp" && mv "$tmp_file_tstamps.tmp" "$tmp_file_tstamps" # see line above (it is added at the bottom of the script) - echo "=================================================================" - echo " START CALCULATIONS FOR TIME-SLICE: $b_yy $b_mm $b_dd $b_hh " - echo "=================================================================" + echo "====================================================================" + echo " START CALCULATIONS FOR TIME-SLICE: $b_yy $b_mm $b_dd $b_hh " + echo "====================================================================" echo " " -echo "=================================================================" -echo "STEP 2 - EXTRACTING SYNOP OBS DATA FROM ODB ..." -echo "=================================================================" -#./synop_obs.sh +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo " STEP #2 : EXTRACTING SYNOP OBS DATA FROM ODB ..." +echo " SCRIPT --- synop_obs.sh" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" #./synop_obs.sh 2020 01 20 00 00 00 39 21 24 61 64 #./synop_obs.sh 2020 01 20 15 00 00 $varMETnum $lon_val_min $lon_val_max $lat_val_min $lat_val_max b_min="00" b_sec="00" ./synop_obs.sh $b_yy $b_mm $b_dd $b_hh $b_min $b_sec $varMETnum $lon_val_min $lon_val_max $lat_val_min $lat_val_max -echo "..... STEP 2 - COMPLETED ....." +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo " ..... STEP 2 - COMPLETED ....." +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo "=================================================================" -echo "STEP 3 - EXTRACTING AND INTERPOLATING MOD DATA TO SYNOP STATIONS " -echo " GEOGRAPHICAL COORDINATES/POSITIONS" -echo " AND ADDING SUCH INTERPOLATED MOD DATA TO ODB ..." -echo "=================================================================" -#synop_mod.sh +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo " STEP #3 : EXTRACTING AND INTERPOLATING MOD DATA TO SYNOP STATIONS " +echo " GEOGRAPHICAL COORDINATES/POSITIONS" +echo " AND ADDING SUCH INTERPOLATED MOD DATA TO ODB ..." +echo " SCRIPT --- synop_mod.sh" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" #./synop_mod.sh 2020012000_2t_r360x180.nc 2020012000 2t #./synop_mod.sh 2020012015_2t_r360x180.nc 2020012015 $varMETstr b_2t="_2t_r360x180.nc" @@ -135,57 +160,65 @@ b_fname_yyyymmddhh_2t=$b_yyyymmddhh$b_2t echo "b_fname_yyyymmddhh_2t = $b_fname_yyyymmddhh_2t" ./synop_mod.sh $b_fname_yyyymmddhh_2t $b_yyyymmddhh $varMETstr -echo "..... STEP 3 - COMPLETED ....." +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo " ..... STEP 3 - COMPLETED ....." +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo "=================================================================" -echo "STEP 4 - SELF-CONTROL : PLOTTING DIFFEENCE OBS VS MOD ..." -echo "=================================================================" -#python graph_mod_obs.py +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo " STEP #4 : SELF-CONTROL BY PLOTTING DIFFEENCE OBS VS MOD ..." +echo " SCRIPT --- graph_mod_obs.py" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" #python graph_mod_obs.py 2020012000 2t #python graph_mod_obs.py 2020012015 $varMETstr python graph_mod_obs.py $b_yyyymmddhh $varMETstr mv *.png GRAPHS/ -echo "..... STEP 4 - COMPLETED ....." +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo " ..... STEP 4 - COMPLETED ....." +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo "=================================================================" -echo " ALL STEPS COMPLETED" -echo " END CALCULATIONS FOR TIME-SLICE: $b_yy $b_mm $b_dd $b_hh " -echo "=================================================================" +echo "====================================================================" +echo " END CALCULATIONS FOR TIME-SLICE: $b_yy $b_mm $b_dd $b_hh " +echo "====================================================================" # Removing record with completed timestamp from file ... # in order to start next time-slice calculations tail -n +2 "$tmp_file_tstamps" > "$tmp_file_tstamps.tmp" && mv "$tmp_file_tstamps.tmp" "$tmp_file_tstamps" echo " " -echo " Calculating/producing quantile rank histogram statistics and plots ..." -echo " BUT ONLY (!!!)" -echo " on a monthly basis - done on 2nd day of a month" -echo " NOW --- $b_yy $b_mm $b_dd $b_hh" -echo " " - -#echo "=================================================================" -#echo " STATS - PRODUCING QUANTILE RANK HISTOGRAM STATISTICS AND PLOTS" -#echo " ON EXAMPLE OF METEOROLOGICAL VARIABLE: $varMETnum" -#echo " AT SYNOP STATIONS" -#echo "=================================================================" +echo "====================================================================" +echo " Checking if STATS should be run ..." +echo " STATS - PRODUCING QUANTILE RANK HISTOGRAM STATISTICS AND PLOTS" +echo " SCRIPT --- synop_stats.sh" +echo " STATS is run on a monthly basis - i.e., done on 2nd day of a month" +echo " ... BUT NOW : $b_yy $b_mm $b_dd $b_hh" ##if [ $b_dd == "02" ] && [ $b_hh == "00" ]; then if [ $b_dd == "21" ] && [ $b_hh == "01" ]; then -b_mm_start="1" -b_mm_end="12" -echo " CALCULATING NOW --- $b_yy $b_mm $b_dd $b_hh" -echo " b_yy, b_mm, b_dd, b_hh, b_mm_start, b_mm_end : $b_yy $b_mm $b_dd $b_hh $b_mm_start $b_mm_end" -echo " varMETnum, b_yy, b_yy, b_mm_start, b_mm_end : $varMETnum, $b_yy, $b_yy, $b_mm_start, $b_mm_end" + b_mm_start="1" + b_mm_end="12" + echo " ... CALCULATING NOW : $b_yy $b_mm $b_dd $b_hh" + echo "====================================================================" + echo " b_yy, b_mm, b_dd, b_hh, b_mm_start, b_mm_end : $b_yy $b_mm $b_dd $b_hh $b_mm_start $b_mm_end" + echo " varMETnum, b_yy, b_yy, b_mm_start, b_mm_end : $varMETnum, $b_yy, $b_yy, $b_mm_start, $b_mm_end" ./synop_stats.sh $varMETnum $b_yy $b_yy $b_mm_start $b_mm_end fi -done +echo " Checking size of file with time stamps: $tmp_file_tstamps ..." -echo "=================================================================" -echo " SYNOP : END LOOP OVER ALL AVAILABLE TIME-SLICES " -echo "=================================================================" +file_actualsize=$(wc -c <"$tmp_file_tstamps") +if [ $file_actualsize == "0" ]; then + echo size is equal to zero bytes + echo "====================================================================" + echo " SYNOP : END LOOP OVER ALL AVAILABLE TIME-SLICES " + echo "====================================================================" + rm $tmp_file_tstamps + exit 1 +else + echo size is over zero bytes +fi + +done -# Remove unneccessary temporary files -rm file_with_tstamps.txt +exit 1 diff --git a/SYNOP/synop_mod.sh b/SYNOP/synop_mod.sh index 9ea416c3f..f5a0cd903 100755 --- a/SYNOP/synop_mod.sh +++ b/SYNOP/synop_mod.sh @@ -214,4 +214,4 @@ mv $outfile $out_path_mod # Deleting header line as 1st line in mod_satellite_YYYYMMDDHH.dat file ... - +exit diff --git a/SYNOP/synop_obs.sh b/SYNOP/synop_obs.sh index 67692dc38..10d59e7a2 100755 --- a/SYNOP/synop_obs.sh +++ b/SYNOP/synop_obs.sh @@ -176,3 +176,5 @@ mv ${out_file}.dat DATAOBS/ else echo $in_file does not exist fi + +exit diff --git a/SYNOP/synop_stats.sh b/SYNOP/synop_stats.sh index 31542e89e..34ea1a55d 100755 --- a/SYNOP/synop_stats.sh +++ b/SYNOP/synop_stats.sh @@ -55,3 +55,4 @@ echo "*****************************************************************" echo " ALL STATS-# STEP COMPLETED" echo "*****************************************************************" +exit diff --git a/request_synop.yml b/request_synop.yml new file mode 100644 index 000000000..fc27bb7df --- /dev/null +++ b/request_synop.yml @@ -0,0 +1,39 @@ +EXPERIMENT: + DATELIST: 20200201 #Startdate + MEMBERS: fc0 + CHUNKSIZEUNIT: day #unit length of the chunk (hour, day, month, year) + CHUNKSIZE: 1 #number of unit lengths per chunk + NUMCHUNKS: 1 #chunkc to be run + CALENDAR: standard #including/ excluding leap years + +GSVREQUEST: #raw data + domain: g + class: rd + type: fc + expver: hz9n + stream: lwda + anoffset: 9 + levtype: "sfc" + date: "%CHUNK_START_DATE%/to/%CHUNK_SECOND_TO_LAST_DATE%" + time: 0000/to/2300/by/0100 # every hour + step: ["0"] # Irrelevant. step will be recalculated in workflow + param: #["2t"] + grid: 1/1 + method: nn + +APP: + other: "empty" + +OPAREQUEST: + NSTATS: "1" # number of variables to request + 1: + variable: "2t" + stat: "hourly" + stat_freq: "hourly" + output_freq: "daily" + time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "/scratch/project_465000454/tmp/%DEFAULT.EXPID%/" + out_filepath: "/scratch/project_465000454/tmp/%DEFAULT.EXPID%/" + diff --git a/run_obsall.py b/run_obsall.py new file mode 100644 index 000000000..a5b9ae02f --- /dev/null +++ b/run_obsall.py @@ -0,0 +1,27 @@ +#!/scratch/project_465000454/devaraju/SW/LUMI-23.03/C/python-climatedt/bin/python +# OBSALL Apps (3 parts: SYNOP, TEMP, AMSU-A observations) + +# Import required libraries +import sys +import subprocess + +# IN IMPLEMENTATION +# --- Processing ground-based observations (SYNOP) +print('**********************************************************') +print('DestinE Climate Digital Twin - OBSALL Apps') +print('--- Processing ground-based observations (SYNOP)') +print('**********************************************************') +command_synop_run = "cd SYNOP; pwd; ./main_synop.sh; exit 0" +subprocess.run(command_synop_run, shell=True, check=True, executable="/bin/bash") + +# IN DEVELOPMENT +# --- Processing radiosounding observations (TEMP) +#command_radsound_run = "cd RADSOUND; pwd; ./main_radsound.sh; exit 0" +#subprocess.run(command_radsound_run, shell=True, executable="/bin/bash") + +# IN DEVELOPMENT +# Processing satellite observations (AMSU-A) +#command_satellite_run = "cd SATELLITE; pwd; ./main_satellite.sh; exit 0" +#subprocess.run(command_satellite_run, shell=True, executable="/bin/bash") + +sys.exit(0) -- GitLab