diff --git a/SYNOP/STATS/fortran-programs/plots_for_one_station b/SYNOP/STATS/fortran-programs/plots_for_one_station new file mode 100755 index 0000000000000000000000000000000000000000..ed3054838289d9c49461f9cd550f6b8849089333 Binary files /dev/null and b/SYNOP/STATS/fortran-programs/plots_for_one_station differ diff --git a/SYNOP/STATS/fortran-programs/plots_for_one_station.f95 b/SYNOP/STATS/fortran-programs/plots_for_one_station.f95 index b74cb22e5ba26d8a8466fe831b7f9dda84a6cc8d..b8212593ec65829b2dea2bb99a57aac33cecb85f 100644 --- a/SYNOP/STATS/fortran-programs/plots_for_one_station.f95 +++ b/SYNOP/STATS/fortran-programs/plots_for_one_station.f95 @@ -167,6 +167,7 @@ program plots_for_one_station character*160 :: txt_data_time_series ! see above character*160 :: txt_data_quantiles ! see above character*160 :: txt_data_rank_histogram ! see above + character*160 :: out_dir integer :: year1,year2 ! first and last year of simulation integer :: month1,month2 ! first and last month of simulation integer :: nyear ! number of years = year2-year1+1 @@ -227,7 +228,7 @@ program plots_for_one_station grads_data_time_series,grads_data_rank_histogram,& year1,year2,month1,month2,& txt_data_time_series,txt_data_rank_histogram,txt_data_quantiles,& - include_rank_histograms,l_code_in_char,l_round_6h,miss + include_rank_histograms,l_code_in_char,l_round_6h,out_dir,miss data color/9,14,4,11,5,13,3,10,7,12,8,2,6/ ! @@ -345,7 +346,7 @@ program plots_for_one_station ! ! Write the commands to the auxiliary script files for Grads ! - open(unit=1,form='formatted',file='vrange_commands') + open(unit=1,form='formatted',file=trim(out_dir)//'vrange_commands') write(1,'(A11,2F10.3)')'set vrange ',fmin_down(1),fmax_up(1) write(1,'(A11,2F10.3)')'set vrange ',fmin_down(2),fmax_up(2) write(1,'(A11,2F10.3)')'set vrange ',fmin_down(3),fmax_up(3) @@ -357,7 +358,7 @@ program plots_for_one_station write(1,'(A13,F7.3)')'set vrange 0 ',1.1*(nquant+1)*freqmax(5) ! conversion to relative frequencies close(1) - open(unit=1,form='formatted',file='time_series_commands') + open(unit=1,form='formatted',file=trim(out_dir)//'time_series_commands') do i=1,nyear write(1,'(A12)')'set cthick 4' write(1,'(A12)')'set cstyle 0' @@ -376,37 +377,37 @@ program plots_for_one_station enddo close(1) - open(unit=1,form='formatted',file='coordinates') + open(unit=1,form='formatted',file=trim(out_dir)//'coordinates') write(1,'(F8.3)')lon write(1,'(F8.3)')lat close(1) if(include_rank_histograms)then - open(unit=1,form='formatted',file='msd_and_p-value_00') + open(unit=1,form='formatted',file=trim(out_dir)//'msd_and_p-value_00') write(1,'(A15)')'set strsiz 0.17' write(1,'(A18)')'set string 1 l 4 0' write(1,'(A25,F5.3)')'draw string 1.3 &0 MSD=',msd(1) write(1,'(A29,F5.3)')'draw string 3.0 &0 p-value=',p_value(1) close(1) - open(unit=1,form='formatted',file='msd_and_p-value_06') + open(unit=1,form='formatted',file=trim(out_dir)//'msd_and_p-value_06') write(1,'(A15)')'set strsiz 0.17' write(1,'(A18)')'set string 1 l 4 0' write(1,'(A25,F5.3)')'draw string 1.3 &0 MSD=',msd(2) write(1,'(A29,F5.3)')'draw string 3.0 &0 p-value=',p_value(2) close(1) - open(unit=1,form='formatted',file='msd_and_p-value_12') + open(unit=1,form='formatted',file=trim(out_dir)//'msd_and_p-value_12') write(1,'(A15)')'set strsiz 0.17' write(1,'(A18)')'set string 1 l 4 0' write(1,'(A25,F5.3)')'draw string 1.3 &0 MSD=',msd(3) write(1,'(A29,F5.3)')'draw string 3.0 &0 p-value=',p_value(3) close(1) - open(unit=1,form='formatted',file='msd_and_p-value_18') + open(unit=1,form='formatted',file=trim(out_dir)//'msd_and_p-value_18') write(1,'(A15)')'set strsiz 0.17' write(1,'(A18)')'set string 1 l 4 0' write(1,'(A25,F5.3)')'draw string 1.3 &0 MSD=',msd(4) write(1,'(A29,F5.3)')'draw string 3.0 &0 p-value=',p_value(4) close(1) - open(unit=1,form='formatted',file='msd_and_p-value') + open(unit=1,form='formatted',file=trim(out_dir)//'msd_and_p-value') write(1,'(A15)')'set strsiz 0.17' write(1,'(A18)')'set string 1 l 4 0' write(1,'(A25,F5.3)')'draw string 1.3 &0 MSD=',msd(5) diff --git a/SYNOP/STATS/fortran-programs/rank_histogram_summary_statistics b/SYNOP/STATS/fortran-programs/rank_histogram_summary_statistics new file mode 100755 index 0000000000000000000000000000000000000000..8e727b3be9456b0ccd97e6a2a6822c2669e91214 Binary files /dev/null and b/SYNOP/STATS/fortran-programs/rank_histogram_summary_statistics differ diff --git a/SYNOP/STATS/fortran-programs/rank_histograms_one_station b/SYNOP/STATS/fortran-programs/rank_histograms_one_station new file mode 100755 index 0000000000000000000000000000000000000000..671c1c429f02e3e0ddbc96c7ab2ad3b081332169 Binary files /dev/null and b/SYNOP/STATS/fortran-programs/rank_histograms_one_station differ diff --git a/SYNOP/STATS/produce_rank_histograms_all_stations.sh b/SYNOP/STATS/produce_rank_histograms_all_stations.sh index f82de60f57c934924d47fc9c400d0f6256e5b488..24032a9d8e14775b531e2f25cdb88d0d90ffa94d 100755 --- a/SYNOP/STATS/produce_rank_histograms_all_stations.sh +++ b/SYNOP/STATS/produce_rank_histograms_all_stations.sh @@ -124,27 +124,30 @@ fi # # 1) Raw data from model simulation (this far, mimicked by observations) echo " Raw data from model simulation (this far, mimicked by observations) ..." -sim_dir="/scratch/project_465000454/ama/open_data_ODB" -sim_file=${sim_dir}/FMI_open_data_2010-2021.odb +sim_dir=$rootdir/SYNOP/output/ODB_data/$experiment #="/scratch/project_465000454/ama/open_data_ODB" +sim_file=${sim_dir}/simulated_synop_data.odb echo " $sim_file" # 2) Pre-computed quantiles echo " Directory with pre-computed quantiles ..." #quant_dir=quantiles -quant_dir=/scratch/project_465000454/ama/STATDATASYNOP/quantiles +#quant_dir=/scratch/project_465000454/ama/STATDATASYNOP/quantiles +quant_dir=$rootdir/SYNOP/input/STATDATASYNOP/quantiles echo " $quant_dir" quant_file=${quant_dir}/quant_all_stations_${variable}_4.odb # 3) Pre-computed bootstrap MSD statistics echo " Directory with pre-computed bootstrap MSD statistics ..." #bootstrap_dir=bootstrap_statistics/${variable}_${period} -bootstrap_dir=/scratch/project_465000454/ama/STATDATASYNOP/bootstrap_statistics/${variable}_${period} +#bootstrap_dir=/scratch/project_465000454/ama/STATDATASYNOP/bootstrap_statistics/${variable}_${period} +bootstrap_dir=$rootdir/SYNOP/input/STATDATASYNOP/bootstrap_statistics/${variable}_${period} echo " $bootstrap_dir" msd_bootstrap_file=${bootstrap_dir}/MSD_bootstrap_${variable}_${period}_all_stations.odb # 4) Directory for results echo " Directory for results ..." -outdir=rank_histograms/${variable}_${year1}${month1}-${year2}${month2} +#outdir=rank_histograms/${variable}_${year1}${month1}-${year2}${month2} +outdir=$rootdir/SYNOP/output/scratch/$experiment/rank_histograms/${variable}_${year1}${month1}-${year2}${month2} echo " $outdir" if test -d "$outdir"; then echo $outdir exists @@ -175,7 +178,7 @@ echo " List of synop stations (with geographical coordinates): $station_list" echo " Compiling fortran-program to calculate rank histograms ..." echo " fortran-programs/rank_histograms_one_station.f95" -gfortran fortran-programs/rank_histograms_one_station.f95 -o rank_histograms_one_station +#gfortran fortran-programs/rank_histograms_one_station.f95 -o rank_histograms_one_station ################################################################# # @@ -191,8 +194,8 @@ line_number=2 # while [ ${line_number} -le `expr $n_lines` ] do - head -`expr ${line_number}` ${station_list} | tail -1 > input.txt - read station longitude latitude < input.txt + head -`expr ${line_number}` ${station_list} | tail -1 > $rootdir/SYNOP/output/scratch/$experiment/input.txt + read station longitude latitude < $rootdir/SYNOP/output/scratch/$experiment/input.txt echo " **********" echo " Synop station ID, longitude, latitude: ${station} ${longitude} ${latitude}" @@ -205,7 +208,7 @@ echo " Synop station ID, longitude, latitude: ${station} ${longitude} ${latitu # When simulation data are available, a means to retrieve the correct data points from it must be designed echo " Selecting the simulation data (mimicked this far by observations) for synop station: ${station}" # -odb_command="odb sql 'select year,month,day,hour,value where station=${station} and (year>=${year1} and year<=${year2}) and (hour=0 or hour=6 or hour=12 or hour=18) and variable=${variable}' -i ${sim_file} -o sim_data" +odb_command="odb sql 'select year,month,day,hour,value where station=${station} and (year>=${year1} and year<=${year2}) and (hour=0 or hour=6 or hour=12 or hour=18) and variable=${variable}' -i ${sim_file} -o $rootdir/SYNOP/output/scratch/$experiment/sim_data" eval ${odb_command} # ############################################################### @@ -214,12 +217,12 @@ echo " Selecting the quantiles for synop station: ${station}" # 00, 06, 12 and 18 UTC are picked separately, since # select \* does not allow for parentheses (very strange) # -rm quantile_selection_* -odb sql select \* where hour=0 and station=${station} -i ${quant_file} -o quantile_selection_00 -odb sql select \* where hour=6 and station=${station} -i ${quant_file} -o quantile_selection_06 -odb sql select \* where hour=12 and station=${station} -i ${quant_file} -o quantile_selection_12 -odb sql select \* where hour=18 and station=${station} -i ${quant_file} -o quantile_selection_18 -cat quantile_selection_* > quantile_selection +#rm quantile_selection_* +odb sql select \* where hour=0 and station=${station} -i ${quant_file} -o $rootdir/SYNOP/output/scratch/$experiment/quantile_selection_00 +odb sql select \* where hour=6 and station=${station} -i ${quant_file} -o $rootdir/SYNOP/output/scratch/$experiment/quantile_selection_06 +odb sql select \* where hour=12 and station=${station} -i ${quant_file} -o $rootdir/SYNOP/output/scratch/$experiment/quantile_selection_12 +odb sql select \* where hour=18 and station=${station} -i ${quant_file} -o $rootdir/SYNOP/output/scratch/$experiment/quantile_selection_18 +cat $rootdir/SYNOP/output/scratch/$experiment/quantile_selection_* > $rootdir/SYNOP/output/scratch/$experiment/quantile_selection # ################################################################ # @@ -228,7 +231,7 @@ cat quantile_selection_* > quantile_selection # of the time period (included in the name of msd_bootstrap_file) # and (to a smaller extent) the selected station. # -odb sql select \* where station=${station} -i ${msd_bootstrap_file} -o msd_bootstrap_selection +odb sql select \* where station=${station} -i ${msd_bootstrap_file} -o $rootdir/SYNOP/output/scratch/$experiment/msd_bootstrap_selection # ############################################################### # Produce the rank histogram for one station. @@ -251,16 +254,16 @@ fi # -./rank_histograms_one_station < input.txt - read station longitude latitude < input.txt + head -`expr ${line_number}` ${station_list} | tail -1 > $rootdir/SYNOP/output/scratch/$experiment/input.txt + read station longitude latitude < $rootdir/SYNOP/output/scratch/$experiment/input.txt echo " **********" echo " Synop station ID, longitude, latitude: ${station} ${longitude} ${latitude}" # @@ -207,14 +211,14 @@ echo " Synop station ID, longitude, latitude: ${station} ${longitude} ${latitu # When simulation data are available, a means to retrieve the correct data points from it must be designed. echo " Selecting the simulation data (mimicked this far by observations) for synop station: ${station}" # -odb_command="odb sql 'select year,month,day,hour,value where station=${station} and (year>=${year1} and year<=${year2}) and (hour=0 or hour=6 or hour=12 or hour=18) and variable=${variable}' -i ${sim_file} -o sim_data" +odb_command="odb sql 'select year,month,day,hour,value where station=${station} and (year>=${year1} and year<=${year2}) and (hour=0 or hour=6 or hour=12 or hour=18) and variable=${variable}' -i ${sim_file} -o $rootdir/SYNOP/output/scratch/$experiment/sim_data" eval ${odb_command} ################################################################ # Select the rank histogram data for the station echo " Selecting rank histogram data for synop station: ${station}" # -odb sql select \* where station=${station} -i ${rh_file} -o rank_histograms_${variable}_${station}_${year1}${month1}-${year2}${month2} +odb sql select \* where station=${station} -i ${rh_file} -o $rootdir/SYNOP/output/scratch/$experiment/rank_histograms_${variable}_${station}_${year1}${month1}-${year2}${month2} # ############################################################### # Select the quantiles for the station @@ -223,12 +227,12 @@ echo " Selecting the quantiles for synop station: ${station}" # 00, 06, 12 and 18 UTC are picked separately, since # select \* does not allow for parentheses (very strange) # -rm quantile_selection_* -odb sql select \* where hour=0 and station=${station} -i ${quant_file} -o quantile_selection_00 -odb sql select \* where hour=6 and station=${station} -i ${quant_file} -o quantile_selection_06 -odb sql select \* where hour=12 and station=${station} -i ${quant_file} -o quantile_selection_12 -odb sql select \* where hour=18 and station=${station} -i ${quant_file} -o quantile_selection_18 -cat quantile_selection_* > quantile_selection +#rm quantile_selection_* +odb sql select \* where hour=0 and station=${station} -i ${quant_file} -o $rootdir/SYNOP/output/scratch/$experiment/quantile_selection_00 +odb sql select \* where hour=6 and station=${station} -i ${quant_file} -o $rootdir/SYNOP/output/scratch/$experiment/quantile_selection_06 +odb sql select \* where hour=12 and station=${station} -i ${quant_file} -o $rootdir/SYNOP/output/scratch/$experiment/quantile_selection_12 +odb sql select \* where hour=18 and station=${station} -i ${quant_file} -o $rootdir/SYNOP/output/scratch/$experiment/quantile_selection_18 +cat quantile_selection_* > $rootdir/SYNOP/output/scratch/$experiment/quantile_selection # ############################################################## # @@ -240,11 +244,13 @@ cat quantile_selection_* > quantile_selection # in GrADS, but plotting in GrADS is omitted in this script. # ############################################################### -./plots_for_one_station < quantile_selection l_code_in_char=.false.,l_round_6h=.false. &END EOF - +fi +if [ 1 -eq 1 ]; then + $SRC_OBSALL_DIR/SYNOP/STATS/fortran-programs/plots_for_one_station < standard_plot.cfg +cat < $rootdir/SYNOP/output/scratch/$experiment/standard_plot.cfg [data] variable=${variable} station_id=${station} @@ -270,9 +293,9 @@ month_beg=${month1} month_end=${month2} [input] -sim_data=time_series_${variable}_${station}_${year1}${month1}-${year2}${month2}.txt -quantile_sel=quantiles_${variable}_${station}.txt -rank_hists=rank_histogram_${variable}_${station}_${year1}${month1}-${year2}${month2}.txt +sim_data=$rootdir/SYNOP/output/scratch/$experiment/time_series_${variable}_${station}_${year1}${month1}-${year2}${month2}.txt +quantile_sel=$rootdir/SYNOP/output/scratch/$experiment/quantiles_${variable}_${station}.txt +rank_hists=$rootdir/SYNOP/output/scratch/$experiment/rank_histogram_${variable}_${station}_${year1}${month1}-${year2}${month2}.txt [output] fig_name=${figure_dir}/standard_plot_${variable}_${station}_${year1}${month1}-${year2}${month2}_python.png @@ -281,27 +304,27 @@ EOF ################################################################ # echo " Calling python to plot quantiles rank histogram for synop station: ${station}" -python3 python/plot_quantiles_rankhist_synop.py standard_plot.cfg +python3 python/plot_quantiles_rankhist_synop.py $rootdir/SYNOP/output/scratch/$experiment/standard_plot.cfg ################################################################ # Remove unnecessary files echo " Removing unnecessary temporary files ..." -rm input.txt -rm vrange_* -rm msd_and_p-value -rm quantile_selection -rm rank_histograms_${variable}_${station}_${year1}${month1}-${year2}${month2} -rm sim_data +#rm input.txt +#rm vrange_* +#rm msd_and_p-value +#rm quantile_selection +#rm rank_histograms_${variable}_${station}_${year1}${month1}-${year2}${month2} +#rm sim_data ###rm msd_bootstrap_selection ###rm standard_plot_one_station -rm time_series_commands -rm time_series_${variable}_${station}_${year1}${month1}-${year2}${month2}.grads -rm rank_histogram_${variable}_${station}_${year1}${month1}-${year2}${month2}.grads -rm standard_plot.cfg -rm time_series_${variable}_${station}_${year1}${month1}-${year2}${month2}.txt -rm quantiles_${variable}_${station}.txt -rm rank_histogram_${variable}_${station}_${year1}${month1}-${year2}${month2}.txt +#rm time_series_commands +#rm time_series_${variable}_${station}_${year1}${month1}-${year2}${month2}.grads +#rm rank_histogram_${variable}_${station}_${year1}${month1}-${year2}${month2}.grads +#rm standard_plot.cfg +#rm time_series_${variable}_${station}_${year1}${month1}-${year2}${month2}.txt +#rm quantiles_${variable}_${station}.txt +#rm rank_histogram_${variable}_${station}_${year1}${month1}-${year2}${month2}.txt # ((line_number++)) done @@ -309,7 +332,7 @@ done ################################################################ # The Fortran executable is not needed any more: # -rm plots_for_one_station -rm quantile_selection_* -rm msd_and_p-value_* -rm coordinates +#rm plots_for_one_station +#rm quantile_selection_* +#rm msd_and_p-value_* +#rm coordinates diff --git a/SYNOP/STATS/python/plot_p_values_map_synop.py b/SYNOP/STATS/python/plot_p_values_map_synop.py index 4c17da7b5b1e6d0e9ad8e494230425161e861f2d..bc243153490de92db3b5a0366505f46f203bcd1d 100644 --- a/SYNOP/STATS/python/plot_p_values_map_synop.py +++ b/SYNOP/STATS/python/plot_p_values_map_synop.py @@ -34,7 +34,7 @@ def read_data(p_values_file): return p_values_data -def plot_p_values(p_values): +def plot_p_values(p_values,figname): ''' Description: plot p values on a map restricted to an area limited by lons (19, 32) and lats (59, 71) Input: numpy array @@ -122,21 +122,24 @@ def plot_p_values(p_values): fig.suptitle('SYNOP stations: p-values for ' + variable_shortnames[variable] +'\nquantiles in ' + timestring, fontsize=fs1) # save figure - plt.savefig('p_values.png', dpi=300) + #plt.savefig('p_values.png', dpi=300) + plt.savefig(figname, dpi=300) -def main(p_values_file): +def main(p_values_file,figurename): ''' Main file: (1) reads in p values from file, (2) plot p values on a map ''' p_values_data = read_data(p_values_file) - plot_p_values(p_values_data) + plot_p_values(p_values_data,figurename) if __name__=='__main__': p_values_file = sys.argv[1] - timestring = sys.argv[2] + timestring = sys.argv[2] # Not used????????? + figurename = sys.argv[3] - main(p_values_file) + #main(p_values_file) # original + main(p_values_file,figurename) diff --git a/SYNOP/STATS/python/plot_rankhist_sum_all_stations_synop.py b/SYNOP/STATS/python/plot_rankhist_sum_all_stations_synop.py index 83d945b539259d42bde2c47455225c811f69b2d0..4fbcc46a983e0deb38218391f91180a3f703556b 100644 --- a/SYNOP/STATS/python/plot_rankhist_sum_all_stations_synop.py +++ b/SYNOP/STATS/python/plot_rankhist_sum_all_stations_synop.py @@ -17,7 +17,7 @@ import matplotlib.pyplot as plt from matplotlib.ticker import (MultipleLocator, AutoMinorLocator) import sys -def plot_rank_hist(p_freq, q_freq, number_of_stations, p01, p1, p5, max_freq_p, max_freq_q): +def plot_rank_hist(p_freq, q_freq, number_of_stations, p01, p1, p5, max_freq_p, max_freq_q,figname): ''' Input: p_freq, q_freq, number of stations, p01, p1, p5, max freq p, max freq q Description: Plots p and q frequencies and saves the figure with figname @@ -103,7 +103,7 @@ def plot_rank_hist(p_freq, q_freq, number_of_stations, p01, p1, p5, max_freq_p, fig.suptitle(title, fontsize=fs1) # save figure - figname='rank_hist_sumstats.png' + #figname='rank_hist_sumstats.png' plt.savefig(figname, dpi=300) def read_data(rh_summary_file): @@ -131,7 +131,8 @@ if __name__=='__main__': p5 = sys.argv[5] max_freq_p = float(sys.argv[6]) max_freq_q = float(sys.argv[7]) + figname = sys.argv[8] p_freq, q_freq = read_data(rh_summary_file) - plot_rank_hist(p_freq, q_freq, number_of_stations, p01, p1, p5, max_freq_p, max_freq_q) + plot_rank_hist(p_freq, q_freq, number_of_stations, p01, p1, p5, max_freq_p, max_freq_q,figname) diff --git a/SYNOP/STATS/summary_rank_histograms_all_stations.sh b/SYNOP/STATS/summary_rank_histograms_all_stations.sh index e5b7e385f01ab31be2a96949fdda6b6c067a5c95..267b0e217dc0abfaf890598502873e5187ddf0cb 100755 --- a/SYNOP/STATS/summary_rank_histograms_all_stations.sh +++ b/SYNOP/STATS/summary_rank_histograms_all_stations.sh @@ -103,7 +103,7 @@ fi echo " Compiling the Fortran program that produces the rank histogram summary statistics ..." echo " fortran-programs/rank_histogram_summary_statistics.f95" -gfortran fortran-programs/rank_histogram_summary_statistics.f95 -o rank_histogram_summary_statistics +#gfortran fortran-programs/rank_histogram_summary_statistics.f95 -o rank_histogram_summary_statistics ################################################################## # @@ -111,7 +111,8 @@ gfortran fortran-programs/rank_histogram_summary_statistics.f95 -o rank_histogra # # 1) Rank histogram directory and input name file name without extension echo " Directory for rank histogram ..." -rh_dir=rank_histograms/${variable}_${year1}${month1}-${year2}${month2} +#rh_dir=rank_histograms/${variable}_${year1}${month1}-${year2}${month2} +rh_dir=$rootdir/SYNOP/output/scratch/$experiment/rank_histograms/${variable}_${year1}${month1}-${year2}${month2} echo " $rh_dir" rh_file=${rh_dir}/rank_histograms_${variable}_${year1}${month1}-${year2}${month2}_all_stations rh_summary_file=${rh_dir}/rank_histograms_${variable}_${year1}${month1}-${year2}${month2}_summary @@ -121,7 +122,8 @@ out_file=${rh_dir}/rh_summary_${variable}_${year1}${month1}-${year2}${month2} # 3) Directory for figures echo " Directory for figures ..." -figure_dir=figures +#figure_dir=figures +figure_dir=$rootdir/SYNOP/output/figures/$experiment echo " $figure_dir" ################################################################## @@ -135,7 +137,7 @@ odb sql select \* -i ${rh_file}.odb -o ${rh_file}.txt # echo " Calculating the rank histogram summary statistics ..." # -./rank_histogram_summary_statistics < $tmp_file_tstamps ls $inp_dir_gsvmoddata > $tmp_file_tstamps tail -n 1 "$tmp_file_tstamps" | tee >(wc -c | xargs -I {} truncate "$tmp_file_tstamps" -s -{}) @@ -142,6 +154,7 @@ echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" #./synop_obs.sh 2020 01 20 15 00 00 $varMETnum $lon_val_min $lon_val_max $lat_val_min $lat_val_max b_min="00" b_sec="00" +pwd ./synop_obs.sh $b_yy $b_mm $b_dd $b_hh $b_min $b_sec $varMETnum $lon_val_min $lon_val_max $lat_val_min $lat_val_max echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" @@ -173,9 +186,9 @@ echo " SCRIPT --- graph_mod_obs.py" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" #python graph_mod_obs.py 2020012000 2t #python graph_mod_obs.py 2020012015 $varMETstr -python graph_mod_obs.py $b_yyyymmddhh $varMETstr +python graph_mod_obs.py $inp_dir_dataobs/ $inp_dir_datamod/ $inp_dir_graphs/ $b_yyyymmddhh $varMETstr -mv *.png GRAPHS/ +#mv *.png $inp_dir_graphs # GRAPHS/ echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo " ..... STEP 4 - COMPLETED ....." diff --git a/SYNOP/synop_mod.sh b/SYNOP/synop_mod.sh index f5a0cd903186bf02572959a2704a0ef55c40ee06..c1bd15d8906027f5e32267d852e09b7923e64bdc 100755 --- a/SYNOP/synop_mod.sh +++ b/SYNOP/synop_mod.sh @@ -29,17 +29,17 @@ echo "Bash version ${BASH_VERSION}..." echo " Input path to location of files with extracted (using gsv_interface) modelled data:" #inp_path_mod="/projappl/project_465000454/ama/DEOBS/SYNOP/GSVMODDATA/" -inp_path_mod="GSVMODDATA/" +inp_path_mod=$rootdir/SYNOP/output/scratch/$experiment/GSVMODDATA/ #"GSVMODDATA/" echo " $inp_path_mod" echo " Input path to location of files with extracted (using sql) observed data:" #inp_path_obs="/projappl/project_465000454/ama/DEOBS/SYNOP/DATAOBS/" -inp_path_obs="DATAOBS/" +inp_path_obs=$rootdir/SYNOP/output/scratch/$experiment/DATAOBS/ #"DATAOBS/" echo " $inp_path_obs" echo " Output path to location of modelled data interpolated to coordinates of synop stations:" #out_path_mod="/projappl/project_465000454/ama/DEOBS/SYNOP/DATAMOD/" -out_path_mod="DATAMOD/" +out_path_mod=$rootdir/SYNOP/output/scratch/$experiment/DATAMOD/ #"DATAMOD/" echo " $out_path_mod" # Input name of nc-file with modelled data extracted with gsv_interface @@ -60,7 +60,7 @@ infile=$inp_path_obs$infile # Name of output file with interpolated modelled data to positions of synop stations vardef="_" -outfile="mod_synop$vardef$timefilename$vardef$varMETstr.dat" +outfile=$rootdir/SYNOP/output/scratch/$experiment/mod_synop$vardef$timefilename$vardef$varMETstr.dat #"mod_synop$vardef$timefilename$vardef$varMETstr.dat" echo " Name of output file for interpolated mod data: $outfile" @@ -87,7 +87,7 @@ echo " Number of records (numrec) in obs data file: $numrec" echo " Creating unstructured grid for 2D geographical locations of synop stations ..." -unstr_grid_file="grid_unstructured_synop.txt" +unstr_grid_file=$rootdir/SYNOP/output/scratch/$experiment/grid_unstructured_synop.txt #"grid_unstructured_synop.txt" echo " Name of file with unstructured grid: $unstr_grid_file" echo "gridtype = unstructured" > $unstr_grid_file # adding grid type @@ -108,8 +108,8 @@ done < $infile echo " CDO calculating weights for unstructured grid ... " #cdo genbil,$unstr_grid_file -selname,2t $ncinfile weights_synop.nc -cdo genbil,$unstr_grid_file -selname,$varMETstr $ncinfile weights_synop.nc -cdo remap,$unstr_grid_file,weights_synop.nc $ncinfile tmpfile_synop_new.nc +cdo genbil,$unstr_grid_file -selname,$varMETstr $ncinfile $rootdir/SYNOP/output/scratch/$experiment/weights_synop.nc +cdo remap,$unstr_grid_file,$rootdir/SYNOP/output/scratch/$experiment/weights_synop.nc $ncinfile $rootdir/SYNOP/output/scratch/$experiment/tmpfile_synop_new.nc while IFS=' ' read -r f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 do @@ -135,12 +135,12 @@ do # without weights - original ... #cdo -s -outputtab,date,time,lon,lat,value -remapnn,"lon=${longitude}_lat=${latitude}" -selvar,${varMETstr} $ncinfile > tmpout_${station}_${variable}.txt # with weight - modified - cdo -s -outputtab,date,time,lon,lat,value -remapnn,"lon=${longitude}_lat=${latitude}" -selvar,${varMETstr} tmpfile_synop_new.nc > tmpout_${station}_${variable}.txt - cat tmpout_${station}_${variable}.txt | sed '1,1d' > tmpoutrev_${station}_${variable}.txt # deliting header line - sed -i s/$/" "${variable}/ tmpoutrev_${station}_${variable}.txt # adding variable ID - sed -i s/$/" "${station}/ tmpoutrev_${station}_${variable}.txt # adding station ID - cat tmpoutrev_${station}_${variable}.txt >> $outfile # adding record to summary file - rm -rf tmpout_*.txt tmpoutrev_*.txt # cleaning tmp-files + cdo -s -outputtab,date,time,lon,lat,value -remapnn,"lon=${longitude}_lat=${latitude}" -selvar,${varMETstr} $rootdir/SYNOP/output/scratch/$experiment/tmpfile_synop_new.nc > $rootdir/SYNOP/output/scratch/$experiment/tmpout_${station}_${variable}.txt + cat $rootdir/SYNOP/output/scratch/$experiment/tmpout_${station}_${variable}.txt | sed '1,1d' > $rootdir/SYNOP/output/scratch/$experiment/tmpoutrev_${station}_${variable}.txt # deliting header line + sed -i s/$/" "${variable}/ $rootdir/SYNOP/output/scratch/$experiment/tmpoutrev_${station}_${variable}.txt # adding variable ID + sed -i s/$/" "${station}/ $rootdir/SYNOP/output/scratch/$experiment/tmpoutrev_${station}_${variable}.txt # adding station ID + cat $rootdir/SYNOP/output/scratch/$experiment/tmpoutrev_${station}_${variable}.txt >> $outfile # adding record to summary file + rm -f $rootdir/SYNOP/output/scratch/$experiment/tmpout_*.txt #$rootdir/SYNOP/output/scratch/$experiment/tmpoutrev_*.txt # cleaning tmp-files done < $infile @@ -151,8 +151,8 @@ sed -i 's/:/ /' $outfile sed -i 's/:/ /' $outfile # Removing extra temporary files ... -rm -rf tmpfile_synop_new.nc weights_synop.nc -rm -rf grid_unstructured_synop.txt +rm -r $rootdir/SYNOP/output/scratch/$experiment/tmpfile_synop_new.nc $rootdir/SYNOP/output/scratch/$experiment/weights_synop.nc +rm -r $rootdir/SYNOP/output/scratch/$experiment/grid_unstructured_synop.txt ############################################################################### @@ -175,14 +175,15 @@ header_mod_synop_data="year@descr:integer month@descr:integer day@descr:integer sed -i "1i ${header_mod_synop_data}" $outfile # Adding data to odb file ... -mod_synop_odb="/scratch/project_465000454/ama/open_data_ODB/synop_mod_data_2010-2021.odb" # Opt1 - new +#mod_synop_odb="/scratch/project_465000454/ama/open_data_ODB/synop_mod_data_2010-2021.odb" # Opt1 - new ##mod_synop_odb="/scratch/project_465000454/ama/open_data_ODB/synop_open_data_2010-2021.odb" # Opt2 - original +mod_synop_odb=$rootdir/SYNOP/output/ODB_data/$experiment/simulated_synop_data.odb #synop_open_data_2010-2021.odb echo " Path and Name of odb-file with saved records:" echo " $mod_synop_odb" -odb import -d ' ' $outfile tmp_mod_synop_odb.odb -cat tmp_mod_synop_odb.odb >> $mod_synop_odb -rm -rf tmp_mod_synop_odb.odb +odb import -d ' ' $outfile $rootdir/SYNOP/output/scratch/$experiment/tmp_mod_synop_odb.odb +cat $rootdir/SYNOP/output/scratch/$experiment/tmp_mod_synop_odb.odb >> $mod_synop_odb +rm -f $rootdir/SYNOP/output/scratch/$experiment/tmp_mod_synop_odb.odb # Deleting header line as 1st line in mod_synop_YYYYMMDDHH.dat file ... sed -i -e "1d" $outfile diff --git a/SYNOP/synop_obs.sh b/SYNOP/synop_obs.sh index d0c9fdfff384c3130b371468d0b148f86173d275..b3b0037b195a837535fff3291dc31821167bcadc 100755 --- a/SYNOP/synop_obs.sh +++ b/SYNOP/synop_obs.sh @@ -1,7 +1,7 @@ #!/bin/bash -echo "Bash version ${BASH_VERSION}..." +#echo "Bash version ${BASH_VERSION}..." #module list -pwd + ######################################################################## # DestinE Climate DT Project # @@ -93,7 +93,8 @@ fi # On Lumi (ODB as a single file) echo " Path to dir where ODB file is placed:" -in_dir=/scratch/project_465000454/ama/open_data_ODB +#in_dir=/scratch/project_465000454/ama/open_data_ODB +in_dir=$rootdir/SYNOP/input/ODB_data echo " $in_dir" echo " Name of ODB file containing all synop obs data:" @@ -101,7 +102,8 @@ in_file=synop_open_data_2010-2021.odb echo " $in_file" echo " Name of file for extracted synop obs data at geographical coordinates of stations:" -out_file=obs_synop_${yyyymmdd}${hour} +#out_file=obs_synop_${yyyymmdd}${hour} +out_file=$rootdir/SYNOP/output/scratch/$experiment/DATAOBS/obs_synop_${yyyymmdd}${hour} echo " $out_file" rm -f ${out_file} @@ -166,10 +168,17 @@ echo " year, month, day, hour, minute, second : $year $month $day $hour $mi # 2020 1 20 0 0 0 24.956800 60.326698 39 3.500000 100968 echo " Cleaning and moving extracted from odb synop obs data into separate folder DATAOBS ..." -cat ${out_file} | sed '1,1d' > ${out_file}.dat -rm -rf ${out_file} -pwd -mv ${out_file}.dat DATAOBS/ +if [ $test_run == 'true' ]; then + echo "### Note: running synop_obs.sh in test mode ###" + echo "2020 1 20 0 0 0 23.576000 68.602997 39 1.100000 102019" > ${out_file}.dat + echo "2020 1 20 0 0 0 20.850901 69.042801 39 1.200000 102017" >> ${out_file}.dat + echo "2020 1 20 0 0 0 25.423300 64.142601 39 1.300000 101695" >> ${out_file}.dat +else + cat ${out_file} | sed '1,1d' > ${out_file}.dat +fi +rm -f ${out_file} +#pwd +#mv ${out_file}.dat DATAOBS/ # Some more 'odb sql' examples: # diff --git a/compile_programs.sh b/compile_programs.sh new file mode 100644 index 0000000000000000000000000000000000000000..c2613347e489aea8f02c37af38bc586b2c2b3fb4 --- /dev/null +++ b/compile_programs.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Use this to compile all Fortran programs before containerization if they are not compiled yet. +# List of programs that are used during the run: + +# SYNOP: +# SYNOP/STATS/fortran-programs/rank_histograms_one_station.f95 +# SYNOP/STATS/fortran-programs/plots_for_one_station.f95 +# SYNOP/STATS/fortran-programs/rank_histogram_summary_statistics.f95 + +# TEMP: +# RADSOUND/STATRS/fortran-programs/rank_histograms_one_station.f95 +# RADSOUND/STATRS/fortran-programs/plots_for_one_station.f95 +# RADSOUND/STATRS/fortran-programs/rank_histogram_summary_statistics.f95 + +gfortran SYNOP/STATS/fortran-programs/rank_histograms_one_station.f95 -o SYNOP/STATS/fortran-programs/rank_histograms_one_station +gfortran SYNOP/STATS/fortran-programs/plots_for_one_station.f95 -o SYNOP/STATS/fortran-programs/plots_for_one_station +gfortran SYNOP/STATS/fortran-programs/rank_histogram_summary_statistics.f95 -o SYNOP/STATS/fortran-programs/rank_histogram_summary_statistics + +gfortran RADSOUND/STATRS/fortran-programs/rank_histograms_one_station.f95 -o RADSOUND/STATRS/fortran-programs/rank_histograms_one_station +gfortran RADSOUND/STATRS/fortran-programs/plots_for_one_station.f95 -o RADSOUND/STATRS/fortran-programs/plots_for_one_station +gfortran RADSOUND/STATRS/fortran-programs/rank_histogram_summary_statistics.f95 -o RADSOUND/STATRS/fortran-programs/rank_histogram_summary_statistics + + diff --git a/load_modules.sh b/load_modules.sh index 3f378da57384714c64409134603e0121233ddae6..7dad629c05c978fb20d09ad28db5d1558a4711a8 100755 --- a/load_modules.sh +++ b/load_modules.sh @@ -1,7 +1,9 @@ #!/bin/bash -# In the case of testing the software, export the following variable +# In the case of testing the software, export the following variables export test_run=true +export rootdir=/projappl/project_465000454/data/OBSALL +export experiment='a001' echo "#######################################################" echo "### OBSALL Apps ###" @@ -31,6 +33,6 @@ module load metkit/1.11.0-cpeCray-23.03.lua ## later upgrade OBSALL Apps for Polytope tool ## (required for SATELLITE observations) in Phase-2 -module list +#module list diff --git a/quick_test_as_batchjob.bash b/quick_test_as_batchjob.bash index 7a321e03c701b36593205e016fdc77c783e5a550..46a7c132d6ba1e8740bdf05b4fed2afb8c7d03af 100644 --- a/quick_test_as_batchjob.bash +++ b/quick_test_as_batchjob.bash @@ -1,7 +1,7 @@ #!/bin/bash -l #SBATCH --job-name=test_obsall # Job name #SBATCH --output=test_obsall.eo # Name of stdout output file -#SBATCH --error=test_obsall_2.eo# Name of stderr error file +#SBATCH --error=test_obsall.eo# Name of stderr error file #SBATCH --partition=small # Partition (queue) name small, debug #SBATCH --ntasks=1 # One task (process) #SBATCH --cpus-per-task=1 # Number of cores (threads) @@ -9,9 +9,12 @@ #SBATCH --time=12:00:00 # Run time (hh:mm:ss) #SBATCH --account=project_465000454 # Project for billing +module purge source load_modules.sh # In the case of testing the software, export the following variable export test_run=true +export experiment='a002' -python run_obsall.py +#python run_obsall.py +./run_obsall.sh diff --git a/run_obsall.sh b/run_obsall.sh index 9b7e6060866c581843da3e4c558c8ac110d90703..5954461246f278eeb414e6ec89a454d1ca075348 100755 --- a/run_obsall.sh +++ b/run_obsall.sh @@ -4,26 +4,32 @@ echo "Bash version ${BASH_VERSION}..." ## OBSALL Apps (3 parts: SYNOP, TEMP, AMSU-A observations) ## Loading necessary modules ... -module use /project/project_465000454/devaraju/modules/LUMI/23.03/C -module load LUMI/23.03 -module load partition/C -module load PrgEnv-gnu -module load ecCodes/2.32.0-cpeCray-23.03.lua -module load odb_api/0.18.1-cpeCray-23.03.lua -module load python-climatedt/3.11.3-cpeCray-23.03.lua -module load pyfdb/0.0.2-cpeCray-23.03.lua -module load cray-hdf5/1.12.2.3 -module load cray-netcdf/4.9.0.3 -module load rttov/13.2 -module load radsim/3.2 -module load fdb/5.11.94-cpeCray-23.03.lua -module load eckit/1.25.0-cpeCray-23.03.lua -module load metkit/1.11.0-cpeCray-23.03.lua +#module use /project/project_465000454/devaraju/modules/LUMI/23.03/C +#module load LUMI/23.03 +#module load partition/C +#module load PrgEnv-gnu +#module load ecCodes/2.32.0-cpeCray-23.03.lua +#module load odb_api/0.18.1-cpeCray-23.03.lua +#module load python-climatedt/3.11.3-cpeCray-23.03.lua +#module load pyfdb/0.0.2-cpeCray-23.03.lua +#module load cray-hdf5/1.12.2.3 +#module load cray-netcdf/4.9.0.3 +#module load rttov/13.2 +#module load radsim/3.2 +#module load fdb/5.11.94-cpeCray-23.03.lua +#module load eckit/1.25.0-cpeCray-23.03.lua +#module load metkit/1.11.0-cpeCray-23.03.lua + +#VVV for convenient testing in terminal VVV# +#source load_modules.sh +#AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA# ## Import required libraries ## import sys ## import subprocess +export SRC_OBSALL_DIR=$(pwd) + ## --- Processing ground-based observations (SYNOP) echo "**********************************************************" echo "DestinE Climate Digital Twin - OBSALL Apps" @@ -37,6 +43,7 @@ cd SYNOP pwd ./main_synop.sh pwd +exit # Only SYNOP prepared properly for containerization. ## --- Processing radiosounding observations (TEMP) echo "**********************************************************"