From a982de1ca9920602083078abd4a7d8e251754eaf Mon Sep 17 00:00:00 2001 From: Alexander Mahura Date: Sun, 14 Apr 2024 16:42:03 +0300 Subject: [PATCH] updated README-file for mother-request & exec py-scrip --- README.md | 499 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 386 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index 5e9e4e2c3..7574f1e27 100644 --- a/README.md +++ b/README.md @@ -75,116 +75,389 @@ module load fdb/5.11.94-cpeCray-23.03.lua module load eckit/1.25.0-cpeCray-23.03.lua module load metkit/1.11.0-cpeCray-23.03.lua ``` -**Data request (sfc):** file `request.yml` is needed for extracting selected modeled data -(example is for SYNOP Part, where only data at `sfc` is needed) +**Data request (sfc, pl, and 02d -levels):** file `request.yml` is needed for extracting selected modeled data from IFS and NEMO models at sfc-, pl-, and o2d-levels. The example is given as part of the mother_request.yml-file for extraction of all needed variables for synop (only at sfc-level), radiosounding (at both sfc- and pl-levels) and satellite (at sfc, pl-, and o2d-levels) parts of OBSALL Apps.
``` -EXPERIMENT: - DATELIST: 20200201 #Startdate - MEMBERS: fc0 - CHUNKSIZEUNIT: day #unit length of the chunk (hour, day, month, year) - CHUNKSIZE: 1 #number of unit lengths per chunk - NUMCHUNKS: 1 #chunkc to be run - CALENDAR: standard #including/ excluding leap years - -GSVREQUEST: #raw data - domain: g - class: rd - type: fc - expver: hz9n - stream: lwda - anoffset: 9 - levtype: "sfc" - date: "%CHUNK_START_DATE%/to/%CHUNK_SECOND_TO_LAST_DATE%" - time: 0000/to/2300/by/0100 # every hour - step: ["0"] # Irrelevant. step will be recalculated in workflow - param: #["2t"] - grid: 1/1 - method: nn - -APP: - other: "empty" - -OPAREQUEST: - NSTATS: "1" # number of variables to request - 1: - variable: "2t" - stat: "hourly" - stat_freq: "hourly" - output_freq: "daily" - time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically - save: True - checkpoint: True - checkpoint_filepath: "/scratch/project_465000454/tmp/%DEFAULT.EXPID%/" - out_filepath: "/scratch/project_465000454/tmp/%DEFAULT.EXPID%/" +OBSALL: + 1: + GSVREQUEST: #raw data (2t at sfc-level from IFS model) for SYNOP Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "sfc" # please fill this + date: split_day + time: "0000/to/2300/by/0100" + param: "2t" # 2t - air temperature at 2 m (167) + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "2t" # 2t - air temperature at 2 m (167) + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + 2: + GSVREQUEST: #raw data (t at 850 hPa from IFS model) for RADSOUND Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "pl" # please fill this + levelist: [850] + date: split_day + time: "0000/to/2300/by/1200" + param: "t" # t - air temperature (130) at 850 hPa + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "t" # t - air temperature (130) at 850 hPa + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 720 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + + 3: + GSVREQUEST: #raw data (2t at sfc-level from IFS model) for RADSOUND Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "sfc" # please fill this + date: split_day + time: "0000/to/2300/by/1200" + param: "2t" # 2t - air temperature at 2 m (167) + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "2t" # 2t - air temperature at 2 m (167) + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 720 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + + 4: + GSVREQUEST: #raw data (2d at sfc-level from IFS model) for SATELLITE Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "sfc" # please fill this + date: split_day + time: "0000/to/2300/by/0100" + param: "2d" # 2d - dewpoint temperature at 2 m (168) + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "2d" # 2d - dewpoint temperature at 2 m (168) + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + + 5: + GSVREQUEST: #raw data (10u at sfc-level from IFS model) for SATELLITE Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "sfc" # please fill this + date: split_day + time: "0000/to/2300/by/0100" + param: "10u" # 10u - U component of wind speed at 10 m (165) + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "10u" # 10u - U component of wind speed at 10 m (165) + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + + 6: + GSVREQUEST: #raw data (10v at sfc-level from IFS model) for SATELLITE Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "sfc" # please fill this + date: split_day + time: "0000/to/2300/by/0100" + param: "10v" # 10v - V component of wind speed at 10 m (166) + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "10v" # 10v - V component of wind speed at 10 m (166) + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + + 7: + GSVREQUEST: #raw data (sp at sfc-level from IFS model) for SATELLITE Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "sfc" # please fill this + date: split_day + time: "0000/to/2300/by/0100" + param: "sp" # sp - surface pressure (134) + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "sp" # sp - surface pressure (134) + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + + 8: + GSVREQUEST: #raw data (skt at sfc-level from IFS model) for SATELLITE Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "sfc" # please fill this + date: split_day + time: "0000/to/2300/by/0100" + param: "skt" # skt - skin temperature (235) + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "skt" # skt - skin temperature (235) + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + + 9: + GSVREQUEST: #raw data (t at pl-levels from IFS model) for SATELLITE Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "pl" # please fill this + levelist: [1000, 925, 850, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 20, 10, 5, 1] + date: split_day + time: "0000/to/2300/by/0100" + param: "t" # t - air temperature (130) at multiple hPa pressure levels (see levelist) + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "t" # t - air temperature (130) at multiple hPa pressure levels (see levelist) + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + + 10: + GSVREQUEST: #raw data (q at pl-levels from IFS model) for SATELLITE Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "pl" # please fill this + levelist: [1000, 925, 850, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 20, 10, 5, 1] + date: split_day + time: "0000/to/2300/by/0100" + param: "q" # q - specific humidity (133) at multiple hPa pressure levels (see levelist) + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "q" # q - specific humidity (133) at multiple hPa pressure levels (see levelist) + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + + 11: + GSVREQUEST: #raw data (clwc at pl-levels from IFS model) for SATELLITE Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "pl" # please fill this + levelist: [1000, 925, 850, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 20, 10, 5, 1] + date: split_day + time: "0000/to/2300/by/0100" + param: "clwc" # clwc - specific liquid water content (246) at multiple hPa pressure levels (see levelist) + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "clwc" # clwc - specific liquid water content (246) at multiple hPa pressure levels (see levelist) + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" + + 12: + GSVREQUEST: #raw data (avg_siconc at o2d-level from NEMO model) for SATELLITE Part of OBSALL Apps + dataset: climate-dt + class: d1 + type: fc + expver: "%APP.READ_EXPID%" + stream: clte + activity: CMIP6 + resolution: standard + generation: 1 + realization: 1 + experiment: hist + model: IFS-NEMO + levtype: "o2d" # please fill this + date: split_day + time: "0000" # note, outputting ocean variables as daily averages, from NEMO model + param: "avg_siconc" # avg_siconc - sea ice fraction, param: [263001] + grid: "1.0/1.0" #fill this if needed. 0.05 is 5km + method: nn + + OPAREQUEST: + variable: "avg_siconc" # avg_siconc - sea ice fraction, param: [263001] + stat: "raw" # please fill this + stat_freq: "hourly" # please fill this + output_freq: "daily" # please fill this + time_step: 0 # in minutes, 60*timestep length in hours TODO: do that automatically + save: True + checkpoint: True + checkpoint_filepath: "%APP.OUTPATH%" + save_filepath: "%APP.OUTPATH%" ```
-Note, for TEMP (radiosounding) and AMSU-A (satellite) Parts - modeled data at both `sfc` and `pl` levels are simulteniously required (such joint request is in development by BSC) - -**Data request (sfc+pl):** file `request.yml` is needed for extracting selected modeled data -(example below is for `2t` and `t` (it is in development by BSC) where modeled data at both `sfc` and `pl` levels are needed; note all other required modeled data are to be added) -
- -``` -APP: - project_details: "Test for the app-workflow streaming" - -DATA: - 1: - GSVREQUEST - param: "2t" - domain: g - class: rd - type: fc - expver: hz9n - stream: lwda - anoffset: 9 - levtype: "sfc" - date: split_day #will be specified in the dn template - time: 0000/to/2300/by/0100 - step: ["0"] # Irrelevant. Step will be recalculated in workflow - grid: 1/1 - method: nn - OPAREQUEST: - stat: "raw" - stat_freq: "hourly" - output_freq: "daily" - time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically - save: True - checkpoint: True - checkpoint_filepath: "%APP.OUTPATH% - save_filepath: "%APP.OUTPATH%" - 2: - GSVREQUEST: - param: "t" - domain: g - class: rd - type: fc - expver: hz9n - stream: lwda - anoffset: 9 - levtype: "pl" - levelist: [850] # Set of levels as int - date: split_day #will be specified in the dn template - time: 0000/to/2300/by/1200 - step: ["0"] # Irrelevant. Step will be recalculated in workflow - grid: 1/1 - method: nn - OPAREQUEST: - stat: "raw" - stat_freq: "hourly" - output_freq: "daily" - time_step: 720 # in minutes, 60*timestep length in hours TODO: do that automatically - save: True - checkpoint: True - checkpoint_filepath: "%APP.OUTPATH% - save_filepath: "%APP.OUTPATH%" - -``` -
**How to run:** runscript `run_obsall.py` is used to execute OBSALL application:
@@ -197,8 +470,7 @@ DATA: import sys import subprocess -# IN IMPLEMENTATION -# --- Processing ground-based observations (SYNOP) +## --- Processing ground-based observations (SYNOP) print('**********************************************************') print('DestinE Climate Digital Twin - OBSALL Apps') print('--- Processing ground-based observations (SYNOP)') @@ -206,8 +478,7 @@ print('**********************************************************') command_synop_run = "cd SYNOP; pwd; ./main_synop.sh; exit 0" subprocess.run(command_synop_run, shell=True, check=True, executable="/bin/bash") -# IN IMPLEMENTATION -# --- Processing radiosounding observations (TEMP) +## --- Processing radiosounding observations (TEMP) print('**********************************************************') print('DestinE Climate Digital Twin - OBSALL Apps') print('--- Processing radiosounding-based observations (TEMP)') @@ -215,13 +486,15 @@ print('**********************************************************') command_radsound_run = "cd RADSOUND; pwd; ./main_radsound.sh; exit 0" subprocess.run(command_radsound_run, shell=True, check=True, executable="/bin/bash") -# IN DEVELOPMENT -# Processing satellite observations (AMSU-A) -#command_satellite_run = "cd SATELLITE; pwd; ./main_amsua.sh; exit 0" -#subprocess.run(command_satellite_run, shell=True, check=True, executable="/bin/bash") +## Processing satellite observations (AMSU-A) +print('**********************************************************') +print('DestinE Climate Digital Twin - OBSALL Apps') +print('--- Processing satelitte-based observations (AMSU-A)') +print('**********************************************************') +command_satellite_run = "cd SATELLITE; pwd; ./main_amsua.sh; exit 0" +subprocess.run(command_satellite_run, shell=True, check=True, executable="/bin/bash") sys.exit(0) - ```
@@ -255,7 +528,7 @@ If you use the OBSALL package in your research or publications, please cite usin @software{obsall, author = {Heikki Järvinen, Jouni Räisänen, Lauri Tuppi, Madeleine Ekblom, Alexander Mahura}, title = {ObsAll: Processing ground-based, radiosounding and satellite observational data for streamed climate modeled data}, - year = {2023}, + year = {2024}, publisher = {GitLab}, journal = {Barcelona Supercomputing Center, Earth Sciences GitLab}, howpublished = {\url{https://earth.bsc.es/gitlab/digital-twins/de_340/obsall/newbranch}}, -- GitLab