From ed6a5fabcc0f71bf212b4768d5c3431fd43b4118 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Thu, 7 Apr 2022 09:11:38 +0200 Subject: [PATCH 1/7] Add grid description files and systems to the archive --- conf/archive.yml | 25 ++++++++++++++++---- conf/grid_description/griddes_eccc1.txt | 19 +++++++++++++++ conf/grid_description/griddes_system2c3s.txt | 19 +++++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 conf/grid_description/griddes_eccc1.txt create mode 100644 conf/grid_description/griddes_system2c3s.txt diff --git a/conf/archive.yml b/conf/archive.yml index 57574919..73379d5d 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -16,20 +16,37 @@ archive: reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" system7c3s: src: "exp/meteofrance/system7c3s/" - monthly_mean: {"tas":"_f6h/","g500":"_f12h/", - "prlr":"_f24h/", "sfcWind": "_f6h/"} + monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/", + "tasmax":"_f6h/", "tasmin": "_f6h/"} nmember: fcst: 51 hcst: 25 reference_grid: "/esarchive/scratch/vagudets/repos/auto-s2s/conf/grid_description/griddes_system7c3s.txt" system35c3s: src: "exp/cmcc/system35c3s/" - monthly_mean: {"tas":"_f6h/","g500":"_f12h/", - "prlr":"_f24h/", "sfcWind": "_f6h/"} + monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/"} nmember: fcst: 50 hcst: 40 reference_grid: "/esarchive/scratch/vagudets/repos/auto-s2s/conf/grid_description/griddes_system35c3s.txt" + system2c3s: + src: "exp/jma/system2c3s/" + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} + nmember: + fcst: 10 + hcst: 10 + reference_grid: "/esarchive/scratch/vagudets/repos/auto-s2s/conf/grid_description/griddes_system35c3s.txt" + eccc1: + src: "exp/eccc/eccc1/" + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} + nmember: + fcst: 10 + hcst: 10 + reference_grid: "/esarchive/scratch/vagudets/repos/auto-s2s/conf/grid_description/griddes_eccc1.txt" Reference: era5: src: "recon/ecmwf/era5/" diff --git a/conf/grid_description/griddes_eccc1.txt b/conf/grid_description/griddes_eccc1.txt new file mode 100644 index 00000000..58a9810c --- /dev/null +++ b/conf/grid_description/griddes_eccc1.txt @@ -0,0 +1,19 @@ +# +# Grid description file for ECCC1 +# +# gridID 2 +# +gridtype = lonlat +gridsize = 64800 +xsize = 360 +ysize = 180 +xname = lon +xlongname = "longitude" +xunits = "degrees_east" +yname = lat +ylongname = "latitude" +yunits = "degrees_north" +xfirst = 0.5 +xinc = 1 +yfirst = 89.5 +yinc = -1 diff --git a/conf/grid_description/griddes_system2c3s.txt b/conf/grid_description/griddes_system2c3s.txt new file mode 100644 index 00000000..cf1e3515 --- /dev/null +++ b/conf/grid_description/griddes_system2c3s.txt @@ -0,0 +1,19 @@ +# +# Grid description file for JMA System3c3s +# +# gridID 2 +# +gridtype = lonlat +gridsize = 10512 +xsize = 144 +ysize = 73 +xname = lon +xlongname = "longitude" +xunits = "degrees_east" +yname = lat +ylongname = "latitude" +yunits = "degrees_north" +xfirst = 0 +xinc = 2.5 +yfirst = 90 +yinc = -2.5 -- GitLab From 68ccbd7cb91f26b9d54de10296665971f6b1cc8d Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Thu, 7 Apr 2022 13:06:35 +0200 Subject: [PATCH 2/7] Fix for issue #11 --- modules/data_load/load.R | 2 +- .../data_load/testing_recipes/recipe_4.yml | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 modules/data_load/testing_recipes/recipe_4.yml diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 0083e9a9..1fbf623d 100755 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -230,7 +230,7 @@ load_datasets <- function(recipe_file){ split_multiselected_dims = split_multiselected_dims, retrieve = TRUE) - names(dim(fcst))[which(names(dim(fcst)) == 'file_date')] <- "fcst_year" + names(dim(fcst))[which(names(dim(fcst)) == 'file_date')] <- "fcst_syear" fcst <- startR_to_s2dv(fcst) } else { diff --git a/modules/data_load/testing_recipes/recipe_4.yml b/modules/data_load/testing_recipes/recipe_4.yml new file mode 100644 index 00000000..128560d4 --- /dev/null +++ b/modules/data_load/testing_recipes/recipe_4.yml @@ -0,0 +1,43 @@ +Description: + Author: V. Agudetse + +Analysis: + Horizon: Seasonal + Variables: + name: tas + freq: monthly_mean + Datasets: + System: + name: system7c3s + Multimodel: no + Reference: + name: era5 + Time: + sdate: + fcst_syear: '2020' + fcst_sday: '1101' + hcst_start: '1993' + hcst_end: '2016' + leadtimemin: 2 + leadtimemax: 4 + Region: + latmin: -10 + latmax: 10 + lonmin: 0 + lonmax: 20 + Regrid: + method: bilinear + type: to_system + Workflow: + Calibration: + method: SBC + Skill: + metric: RPSS + Indicators: + index: no + Output_format: S2S4E +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ -- GitLab From 2254f0786b6597a35b49ef05c35960731cfc9742 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 12 Apr 2022 16:32:46 +0200 Subject: [PATCH 3/7] format fixes --- modules/data_load/load.R | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 1fbf623d..6aec0614 100755 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -19,12 +19,12 @@ attr2array <- function(attr){ startR_to_s2dv <- function(startR_array){ dates_dims <- dim(Subset(startR_array, - along=c('dat','var', + along=c('dat', 'var', 'latitude', 'longitude', 'ensemble'), list(1,1,1,1,1), drop="selected")) #sdates_dims <- dim(Subset(startR_array, - # along=c('dat','var','time','sweek','sday', + # along=c('dat', 'var', 'time', 'sweek', 'sday', # 'latitude', 'longitude', 'ensemble'), # list(1,1,1,1,1,1,1,1), drop="selected")) @@ -44,18 +44,18 @@ startR_to_s2dv <- function(startR_array){ 'Variables')$dat1$longitude), lat = attr2array(attr(startR_array, 'Variables')$dat1$latitude), - Variable = list(varName=names(attr(startR_array, - 'Variables')$common)[2], + Variable = list(varName=names(attr(startR_array, + 'Variables')$common)[2], level = NULL), Dates = list(start = dates_start, end = dates_end), #sdate = sdates), - time_dims = time_dims, + time_dims = time_dims, when = Sys.time(), source_files = attr2array(attr(startR_array, "Files")) - #Datasets=list(exp1=list(InitializationsDates=list(Member_1="01011990", - # Members="Member_1"))) - ) + # Datasets=list(exp1=list(InitializationsDates=list( Member_1="01011990", + # Members="Member_1"))) + ) return(s2dv_object) } -- GitLab From 5f98c0dd0319a258343e6ba7e68603622d53c96b Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 12 Apr 2022 16:33:27 +0200 Subject: [PATCH 4/7] Add testing recipe for meteofrance system 7 monthly --- modules/data_load/testing_recipes/recipe_4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data_load/testing_recipes/recipe_4.yml b/modules/data_load/testing_recipes/recipe_4.yml index 128560d4..a82c4f2c 100644 --- a/modules/data_load/testing_recipes/recipe_4.yml +++ b/modules/data_load/testing_recipes/recipe_4.yml @@ -30,7 +30,7 @@ Analysis: type: to_system Workflow: Calibration: - method: SBC + method: mse_min Skill: metric: RPSS Indicators: -- GitLab From cf240f215e897c6a82f9e6f0b98716ec42c520ef Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 19 Apr 2022 14:47:05 +0200 Subject: [PATCH 5/7] added file to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5faf5e63..752f5cf9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ out-logs/ *.swp *.swo +/modules/Calibration/test_victoria.R -- GitLab From 73ecc687f21033b16f1833e64391fddf36e98006 Mon Sep 17 00:00:00 2001 From: lpalma Date: Tue, 19 Apr 2022 16:10:29 +0200 Subject: [PATCH 6/7] fixed leadtime selection, now the 0 leadmonth corresponds to the month starting when the forecast is initialized --- modules/data_load/dates2load.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data_load/dates2load.R b/modules/data_load/dates2load.R index 28d7905e..f6a5e27f 100644 --- a/modules/data_load/dates2load.R +++ b/modules/data_load/dates2load.R @@ -97,8 +97,8 @@ get_timeidx <- function(sdates, ltmin, ltmax, } else if (time_freq == "monthly_mean") { - idx_min <- ltmin - idx_max <- ltmax + idx_min <- ltmin + 1 + idx_max <- ltmax + 1 indxs <- indices(idx_min:idx_max) } -- GitLab From 8b99aa6d4d486b5721a546aa577242d960580d29 Mon Sep 17 00:00:00 2001 From: lpalma Date: Tue, 19 Apr 2022 18:06:03 +0200 Subject: [PATCH 7/7] Added @aho suggestions --- modules/data_load/load.R | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 6aec0614..25a3b82b 100755 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -244,8 +244,7 @@ load_datasets <- function(recipe_file){ # Separate Start() call for monthly vs daily data if (store.freq == "monthly_mean") { - dates_file <- sapply(dates, format, '%Y%m%d') - dates_file <- format(as.Date(dates_file, '%Y%m%d'), "%Y%m") + dates_file <- format(as.Date(dates, '%Y%m%d'), "%Y%m") dim(dates_file) <- dim(Subset(hcst$data, along=c('dat','var', 'latitude', 'longitude', 'ensemble'), @@ -254,8 +253,6 @@ load_datasets <- function(recipe_file){ obs <- Start(dat = obs.path, var = variable, file_date = dates_file, - merge_across_dims = TRUE, - merge_across_dims_narm = TRUE, latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(), longitude = values(list(lons.min, lons.max)), -- GitLab