diff --git a/README.md b/README.md
index 5e9e4e2c3094a7390a2fa51e0118d29a9c9e8d97..7574f1e27dcb815549649a75e1f617212291adb1 100644
--- a/README.md
+++ b/README.md
@@ -75,116 +75,389 @@ module load fdb/5.11.94-cpeCray-23.03.lua
module load eckit/1.25.0-cpeCray-23.03.lua
module load metkit/1.11.0-cpeCray-23.03.lua
```
-**Data request (sfc):** file `request.yml` is needed for extracting selected modeled data
-(example is for SYNOP Part, where only data at `sfc` is needed)
+**Data request (sfc, pl, and 02d -levels):** file `request.yml` is needed for extracting selected modeled data from IFS and NEMO models at sfc-, pl-, and o2d-levels. The example is given as part of the mother_request.yml-file for extraction of all needed variables for synop (only at sfc-level), radiosounding (at both sfc- and pl-levels) and satellite (at sfc, pl-, and o2d-levels) parts of OBSALL Apps.
```
-EXPERIMENT:
- DATELIST: 20200201 #Startdate
- MEMBERS: fc0
- CHUNKSIZEUNIT: day #unit length of the chunk (hour, day, month, year)
- CHUNKSIZE: 1 #number of unit lengths per chunk
- NUMCHUNKS: 1 #chunkc to be run
- CALENDAR: standard #including/ excluding leap years
-
-GSVREQUEST: #raw data
- domain: g
- class: rd
- type: fc
- expver: hz9n
- stream: lwda
- anoffset: 9
- levtype: "sfc"
- date: "%CHUNK_START_DATE%/to/%CHUNK_SECOND_TO_LAST_DATE%"
- time: 0000/to/2300/by/0100 # every hour
- step: ["0"] # Irrelevant. step will be recalculated in workflow
- param: #["2t"]
- grid: 1/1
- method: nn
-
-APP:
- other: "empty"
-
-OPAREQUEST:
- NSTATS: "1" # number of variables to request
- 1:
- variable: "2t"
- stat: "hourly"
- stat_freq: "hourly"
- output_freq: "daily"
- time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
- save: True
- checkpoint: True
- checkpoint_filepath: "/scratch/project_465000454/tmp/%DEFAULT.EXPID%/"
- out_filepath: "/scratch/project_465000454/tmp/%DEFAULT.EXPID%/"
+OBSALL:
+ 1:
+ GSVREQUEST: #raw data (2t at sfc-level from IFS model) for SYNOP Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "sfc" # please fill this
+ date: split_day
+ time: "0000/to/2300/by/0100"
+ param: "2t" # 2t - air temperature at 2 m (167)
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "2t" # 2t - air temperature at 2 m (167)
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+ 2:
+ GSVREQUEST: #raw data (t at 850 hPa from IFS model) for RADSOUND Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "pl" # please fill this
+ levelist: [850]
+ date: split_day
+ time: "0000/to/2300/by/1200"
+ param: "t" # t - air temperature (130) at 850 hPa
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "t" # t - air temperature (130) at 850 hPa
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 720 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+
+ 3:
+ GSVREQUEST: #raw data (2t at sfc-level from IFS model) for RADSOUND Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "sfc" # please fill this
+ date: split_day
+ time: "0000/to/2300/by/1200"
+ param: "2t" # 2t - air temperature at 2 m (167)
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "2t" # 2t - air temperature at 2 m (167)
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 720 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+
+ 4:
+ GSVREQUEST: #raw data (2d at sfc-level from IFS model) for SATELLITE Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "sfc" # please fill this
+ date: split_day
+ time: "0000/to/2300/by/0100"
+ param: "2d" # 2d - dewpoint temperature at 2 m (168)
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "2d" # 2d - dewpoint temperature at 2 m (168)
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+
+ 5:
+ GSVREQUEST: #raw data (10u at sfc-level from IFS model) for SATELLITE Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "sfc" # please fill this
+ date: split_day
+ time: "0000/to/2300/by/0100"
+ param: "10u" # 10u - U component of wind speed at 10 m (165)
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "10u" # 10u - U component of wind speed at 10 m (165)
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+
+ 6:
+ GSVREQUEST: #raw data (10v at sfc-level from IFS model) for SATELLITE Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "sfc" # please fill this
+ date: split_day
+ time: "0000/to/2300/by/0100"
+ param: "10v" # 10v - V component of wind speed at 10 m (166)
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "10v" # 10v - V component of wind speed at 10 m (166)
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+
+ 7:
+ GSVREQUEST: #raw data (sp at sfc-level from IFS model) for SATELLITE Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "sfc" # please fill this
+ date: split_day
+ time: "0000/to/2300/by/0100"
+ param: "sp" # sp - surface pressure (134)
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "sp" # sp - surface pressure (134)
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+
+ 8:
+ GSVREQUEST: #raw data (skt at sfc-level from IFS model) for SATELLITE Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "sfc" # please fill this
+ date: split_day
+ time: "0000/to/2300/by/0100"
+ param: "skt" # skt - skin temperature (235)
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "skt" # skt - skin temperature (235)
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+
+ 9:
+ GSVREQUEST: #raw data (t at pl-levels from IFS model) for SATELLITE Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "pl" # please fill this
+ levelist: [1000, 925, 850, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 20, 10, 5, 1]
+ date: split_day
+ time: "0000/to/2300/by/0100"
+ param: "t" # t - air temperature (130) at multiple hPa pressure levels (see levelist)
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "t" # t - air temperature (130) at multiple hPa pressure levels (see levelist)
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+
+ 10:
+ GSVREQUEST: #raw data (q at pl-levels from IFS model) for SATELLITE Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "pl" # please fill this
+ levelist: [1000, 925, 850, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 20, 10, 5, 1]
+ date: split_day
+ time: "0000/to/2300/by/0100"
+ param: "q" # q - specific humidity (133) at multiple hPa pressure levels (see levelist)
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "q" # q - specific humidity (133) at multiple hPa pressure levels (see levelist)
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+
+ 11:
+ GSVREQUEST: #raw data (clwc at pl-levels from IFS model) for SATELLITE Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "pl" # please fill this
+ levelist: [1000, 925, 850, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 20, 10, 5, 1]
+ date: split_day
+ time: "0000/to/2300/by/0100"
+ param: "clwc" # clwc - specific liquid water content (246) at multiple hPa pressure levels (see levelist)
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "clwc" # clwc - specific liquid water content (246) at multiple hPa pressure levels (see levelist)
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
+
+ 12:
+ GSVREQUEST: #raw data (avg_siconc at o2d-level from NEMO model) for SATELLITE Part of OBSALL Apps
+ dataset: climate-dt
+ class: d1
+ type: fc
+ expver: "%APP.READ_EXPID%"
+ stream: clte
+ activity: CMIP6
+ resolution: standard
+ generation: 1
+ realization: 1
+ experiment: hist
+ model: IFS-NEMO
+ levtype: "o2d" # please fill this
+ date: split_day
+ time: "0000" # note, outputting ocean variables as daily averages, from NEMO model
+ param: "avg_siconc" # avg_siconc - sea ice fraction, param: [263001]
+ grid: "1.0/1.0" #fill this if needed. 0.05 is 5km
+ method: nn
+
+ OPAREQUEST:
+ variable: "avg_siconc" # avg_siconc - sea ice fraction, param: [263001]
+ stat: "raw" # please fill this
+ stat_freq: "hourly" # please fill this
+ output_freq: "daily" # please fill this
+ time_step: 0 # in minutes, 60*timestep length in hours TODO: do that automatically
+ save: True
+ checkpoint: True
+ checkpoint_filepath: "%APP.OUTPATH%"
+ save_filepath: "%APP.OUTPATH%"
```
-Note, for TEMP (radiosounding) and AMSU-A (satellite) Parts - modeled data at both `sfc` and `pl` levels are simulteniously required (such joint request is in development by BSC)
-
-**Data request (sfc+pl):** file `request.yml` is needed for extracting selected modeled data
-(example below is for `2t` and `t` (it is in development by BSC) where modeled data at both `sfc` and `pl` levels are needed; note all other required modeled data are to be added)
-
-
-```
-APP:
- project_details: "Test for the app-workflow streaming"
-
-DATA:
- 1:
- GSVREQUEST
- param: "2t"
- domain: g
- class: rd
- type: fc
- expver: hz9n
- stream: lwda
- anoffset: 9
- levtype: "sfc"
- date: split_day #will be specified in the dn template
- time: 0000/to/2300/by/0100
- step: ["0"] # Irrelevant. Step will be recalculated in workflow
- grid: 1/1
- method: nn
- OPAREQUEST:
- stat: "raw"
- stat_freq: "hourly"
- output_freq: "daily"
- time_step: 60 # in minutes, 60*timestep length in hours TODO: do that automatically
- save: True
- checkpoint: True
- checkpoint_filepath: "%APP.OUTPATH%
- save_filepath: "%APP.OUTPATH%"
- 2:
- GSVREQUEST:
- param: "t"
- domain: g
- class: rd
- type: fc
- expver: hz9n
- stream: lwda
- anoffset: 9
- levtype: "pl"
- levelist: [850] # Set of levels as int
- date: split_day #will be specified in the dn template
- time: 0000/to/2300/by/1200
- step: ["0"] # Irrelevant. Step will be recalculated in workflow
- grid: 1/1
- method: nn
- OPAREQUEST:
- stat: "raw"
- stat_freq: "hourly"
- output_freq: "daily"
- time_step: 720 # in minutes, 60*timestep length in hours TODO: do that automatically
- save: True
- checkpoint: True
- checkpoint_filepath: "%APP.OUTPATH%
- save_filepath: "%APP.OUTPATH%"
-
-```
-
**How to run:** runscript `run_obsall.py` is used to execute OBSALL application:
@@ -197,8 +470,7 @@ DATA:
import sys
import subprocess
-# IN IMPLEMENTATION
-# --- Processing ground-based observations (SYNOP)
+## --- Processing ground-based observations (SYNOP)
print('**********************************************************')
print('DestinE Climate Digital Twin - OBSALL Apps')
print('--- Processing ground-based observations (SYNOP)')
@@ -206,8 +478,7 @@ print('**********************************************************')
command_synop_run = "cd SYNOP; pwd; ./main_synop.sh; exit 0"
subprocess.run(command_synop_run, shell=True, check=True, executable="/bin/bash")
-# IN IMPLEMENTATION
-# --- Processing radiosounding observations (TEMP)
+## --- Processing radiosounding observations (TEMP)
print('**********************************************************')
print('DestinE Climate Digital Twin - OBSALL Apps')
print('--- Processing radiosounding-based observations (TEMP)')
@@ -215,13 +486,15 @@ print('**********************************************************')
command_radsound_run = "cd RADSOUND; pwd; ./main_radsound.sh; exit 0"
subprocess.run(command_radsound_run, shell=True, check=True, executable="/bin/bash")
-# IN DEVELOPMENT
-# Processing satellite observations (AMSU-A)
-#command_satellite_run = "cd SATELLITE; pwd; ./main_amsua.sh; exit 0"
-#subprocess.run(command_satellite_run, shell=True, check=True, executable="/bin/bash")
+## Processing satellite observations (AMSU-A)
+print('**********************************************************')
+print('DestinE Climate Digital Twin - OBSALL Apps')
+print('--- Processing satelitte-based observations (AMSU-A)')
+print('**********************************************************')
+command_satellite_run = "cd SATELLITE; pwd; ./main_amsua.sh; exit 0"
+subprocess.run(command_satellite_run, shell=True, check=True, executable="/bin/bash")
sys.exit(0)
-
```
@@ -255,7 +528,7 @@ If you use the OBSALL package in your research or publications, please cite usin
@software{obsall,
author = {Heikki Järvinen, Jouni Räisänen, Lauri Tuppi, Madeleine Ekblom, Alexander Mahura},
title = {ObsAll: Processing ground-based, radiosounding and satellite observational data for streamed climate modeled data},
- year = {2023},
+ year = {2024},
publisher = {GitLab},
journal = {Barcelona Supercomputing Center, Earth Sciences GitLab},
howpublished = {\url{https://earth.bsc.es/gitlab/digital-twins/de_340/obsall/newbranch}},