From 597db0b7ba4612b4cd1468bbe991742623076610 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Mon, 7 Mar 2022 16:51:57 +0100 Subject: [PATCH 01/31] Changed recipe to recipe --- modules/data_load/load.R | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 9af22eec..4e47c33a 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -21,25 +21,25 @@ source("tools/libs.R") # ------------------------------------------------------------------------------------------- # Set params ----------------------------------------- -hcst.inityear <- recipe$params$Time$hcst_start -hcst.endyear <- recipe$params$Time$hcst_end -ltmin <- recipe$params$Time$leadtimemin -ltmax <- recipe$params$Time$leadtimemax -lats.min <- recipe$params$Region$latmin -lats.max <- recipe$params$Region$latmax -lons.min <- recipe$params$Region$lonmin -lons.max <- recipe$params$Region$lonmax -ref.name <- recipe$params$Datasets$Reference$name -exp.name <- recipe$params$Datasets$System$name - -variable <- recipe$params$Variables$name -store.freq <- recipe$params$Variables$freq +hcst.inityear <- recipe$Analysis$Time$hcst_start +hcst.endyear <- recipe$Analysis$Time$hcst_end +ltmin <- recipe$Analysis$Time$leadtimemin +ltmax <- recipe$Analysis$Time$leadtimemax +lats.min <- recipe$Analysis$Region$latmin +lats.max <- recipe$Analysis$Region$latmax +lons.min <- recipe$Analysis$Region$lonmin +lons.max <- recipe$Analysis$Region$lonmax +ref.name <- recipe$Analysis$Datasets$Reference$name +exp.name <- recipe$Analysis$Datasets$System$name + +variable <- recipe$Analysis$Variables$name +store.freq <- recipe$Analysis$Variables$freq stream <- verifications$stream sdates <- verifications$fcst.sdate ## TODO: define fcst.name -##fcst.name <- recipe$params$Datasets$System[[sys]]$name +##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name # get sdates array sdates <- dates2load(recipe, logger) -- GitLab From ab1bae507314c99a137a06f30b6d359873163aca Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Mon, 7 Mar 2022 17:05:58 +0100 Subject: [PATCH 02/31] Change recipe to read_yaml, adjust parameter names --- modules/data_load/load.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 4e47c33a..c15bc592 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -9,9 +9,9 @@ source("tools/libs.R") # RECIPE FOR TESTING # ------------------------------------------------------------------------------------------- -# recipe <- read_yaml("/esarchive/scratch/lpalma/git/auto-s2s/modules/data_load/recipe_1.yml") + recipe <- read_yaml("/esarchive/scratch/lpalma/git/auto-s2s/modules/data_load/recipe_1.yml") args <- NULL; args[1] <- "/esarchive/scratch/lpalma/git/auto-s2s/modules/data_load/recipe_1.yml" - recipe <- Recipe$new(filename=args[1]) + # recipe <- Recipe$new(filename=args[1]) recipe$filename <- args[1] # Create output folder and log: logger <- prepare_outputs(recipe = recipe) @@ -45,7 +45,7 @@ sdates <- verifications$fcst.sdate sdates <- dates2load(recipe, logger) # get esarchive datasets dict: -archive <- read_yaml(paste0(recipe$run_conf_conf$code_dir,"conf/archive.yml"))$archive +archive <- read_yaml(paste0(recipe$Run$code_dir,"conf/archive.yml"))$archive exp_descrip <- archive[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable]) -- GitLab From 9b3251fa05ff6292280b3e5db08ae87804544dee Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Mon, 7 Mar 2022 17:14:48 +0100 Subject: [PATCH 03/31] Changed recipe to recipe to work with read_yaml --- modules/data_load/regrid.R | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/data_load/regrid.R b/modules/data_load/regrid.R index a8091997..88c74236 100644 --- a/modules/data_load/regrid.R +++ b/modules/data_load/regrid.R @@ -1,27 +1,27 @@ get_regrid_params <- function(recipe,archive){ - exp.name <- recipe$params$Datasets$System$name - ref.name <- recipe$params$Datasets$Reference$name + exp.name <- recipe$Analysis$Datasets$System$name + ref.name <- recipe$Analysis$Datasets$Reference$name exp_descrip <- archive[[exp.name]] reference_descrip <- archive[[ref.name]] - if (tolower(recipe$params$Regrid$type) == 'to_reference') { + if (tolower(recipe$Analysis$Regrid$type) == 'to_reference') { regrid_params <- list(fcst.gridtype=reference_descrip$regrid, - fcst.gridmethod=recipe$params$Regrid$method, + fcst.gridmethod=recipe$Analysis$Regrid$method, fcst.transform=CDORemapper, obs.gridtype=NULL, obs.gridmethod=NULL, obs.transform=NULL) - } else if (tolower(recipe$params$Regrid$type) == 'to_system') { + } else if (tolower(recipe$Analysis$Regrid$type) == 'to_system') { regrid_params <- list(fcst.gridtype=NULL, fcst.gridmethod=NULL, fcst.transform=NULL, obs.gridtype=exp_descrip$regrid, - obs.gridmethod=recipe$params$Regrid$method, + obs.gridmethod=recipe$Analysis$Regrid$method, obs.transform=CDORemapper) } ##TODO: Else condition? middle interpolation? -- GitLab From a26d10c0d6f17c933988544285b1f53a6edfda51 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 8 Mar 2022 09:08:07 +0100 Subject: [PATCH 04/31] Changed params to Analysis to read recipe --- modules/data_load/dates2load.R | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/data_load/dates2load.R b/modules/data_load/dates2load.R index 17c5cb99..c7c392a5 100644 --- a/modules/data_load/dates2load.R +++ b/modules/data_load/dates2load.R @@ -3,7 +3,7 @@ # both for the hcst and fcst. dates2load <- function(recipe, logger){ - recipe <- recipe$params$Time + recipe <- recipe$Analysis$Time # hcst dates @@ -41,6 +41,3 @@ add_dims <- function(data, type){ return(data) } - - - -- GitLab From 624fc20c87d6c046414a4adb1bd005b007361d9f Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 8 Mar 2022 09:17:06 +0100 Subject: [PATCH 05/31] Added calls to yaml and CSTools, removed call to add_dims.R --- tools/libs.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/libs.R b/tools/libs.R index 77dcf7d7..7b010f54 100644 --- a/tools/libs.R +++ b/tools/libs.R @@ -2,12 +2,13 @@ library(log4r) library(startR) library(ClimProjDiags) library(multiApply) +library(yaml) # library(s2dverification) # library(ncdf4) # library(abind) # library(easyVerification) # library(easyNCDF) -# library(CSTools) + library(CSTools) # # library(parallel) # library(pryr) # To check mem usage. @@ -25,4 +26,4 @@ library(multiApply) source("tools/check_recipe.R") source("tools/prepare_outputs.R") source("tools/divide_recipe.R") -source("tools/add_dims.R") # Not sure if necessary yet +# source("tools/add_dims.R") # Not sure if necessary yet -- GitLab From 4eaa6ed61b8c6ee32ffad3a80f950d0829203b81 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 8 Mar 2022 09:33:18 +0100 Subject: [PATCH 06/31] Added second atomic recipe for testing --- modules/data_load/recipe_2.yml | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 modules/data_load/recipe_2.yml diff --git a/modules/data_load/recipe_2.yml b/modules/data_load/recipe_2.yml new file mode 100644 index 00000000..8569bfba --- /dev/null +++ b/modules/data_load/recipe_2.yml @@ -0,0 +1,43 @@ +Description: + Author: V. Agudetse + '': split version +Analysis: + Horizon: Seasonal + Variables: + name: tas + freq: monthly_mean + Datasets: + System: + name: system5c3s + Multimodel: no + Reference: + name: era5 + Time: + sdate: + fcst_syear: '2017' + fcst_sday: '0701' + hcst_start: '1993' + hcst_end: '2016' + leadtimemin: 2 + leadtimemax: 4 + Region: + latmin: -90 + latmax: 90 + lonmin: 0 + lonmax: 360 + Regrid: + method: bilinear + type: to_system + Workflow: + Calibration: + method: SBC + Skill: + metric: RPSS + Indicators: + index: no + Output_format: S2S4E +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ -- GitLab From 27a1840f799b1c1c3c8f8901323e6535cbac89cc Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 8 Mar 2022 09:49:29 +0100 Subject: [PATCH 07/31] Changed sourced modules, recipe, added TODOs --- modules/data_load/load.R | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index c15bc592..6b82924d 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -1,23 +1,23 @@ source("modules/data_load/dates2load.R") source("modules/data_load/regrid.R") -source("modules/data_load/s2dv_cube.R") -source("recipe.R") +# source("modules/data_load/s2dv_cube.R") +# source("recipe.R") # Load required libraries source("tools/libs.R") # RECIPE FOR TESTING # ------------------------------------------------------------------------------------------- - recipe <- read_yaml("/esarchive/scratch/lpalma/git/auto-s2s/modules/data_load/recipe_1.yml") - args <- NULL; args[1] <- "/esarchive/scratch/lpalma/git/auto-s2s/modules/data_load/recipe_1.yml" - # recipe <- Recipe$new(filename=args[1]) - recipe$filename <- args[1] +## TODO: Get only the last part of the path as the recipe$filename? +recipe <- read_yaml("/esarchive/scratch/vagudets/repos/auto-s2s/modules/data_load/recipe_2.yml") +args <- NULL; args[1] <- "/esarchive/scratch/vagudets/repos/auto-s2s/modules/data_load/recipe_2.yml" +recipe$filename <- args[1] # Create output folder and log: - logger <- prepare_outputs(recipe = recipe) - folder <- logger$foldername - log_file <- logger$logname - logger <- logger$logger +logger <- prepare_outputs(recipe = recipe) +folder <- logger$foldername +log_file <- logger$logname +logger <- logger$logger # ------------------------------------------------------------------------------------------- # Set params ----------------------------------------- @@ -35,8 +35,9 @@ exp.name <- recipe$Analysis$Datasets$System$name variable <- recipe$Analysis$Variables$name store.freq <- recipe$Analysis$Variables$freq -stream <- verifications$stream -sdates <- verifications$fcst.sdate +## TODO: Examine this verifications part, verify if it's necessary +# stream <- verifications$stream +# sdates <- verifications$fcst.sdate ## TODO: define fcst.name ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name @@ -75,7 +76,7 @@ hcst.path <- paste0(archive$src, # Regrid: #------------------------------------------------------------------- -regrid_params <- get_regrid_params(recipe,archive) +regrid_params <- get_regrid_params(recipe, archive) # Timeseries load -- GitLab From 17cebf13332c1fc35998101d607031914dbe5368 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 8 Mar 2022 13:03:17 +0100 Subject: [PATCH 08/31] Change lon bound from 360 to 359.9 --- modules/data_load/recipe_2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data_load/recipe_2.yml b/modules/data_load/recipe_2.yml index 8569bfba..2775cdf5 100644 --- a/modules/data_load/recipe_2.yml +++ b/modules/data_load/recipe_2.yml @@ -24,7 +24,7 @@ Analysis: latmin: -90 latmax: 90 lonmin: 0 - lonmax: 360 + lonmax: 359.9 Regrid: method: bilinear type: to_system -- GitLab From 99780b056182ab31fe538a3b9adc1aabe5789d87 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 8 Mar 2022 16:25:18 +0100 Subject: [PATCH 09/31] Changed param name regrid to reference_grid as discussed --- conf/archive.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/archive.yml b/conf/archive.yml index e70d4f2d..2b945396 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -15,7 +15,7 @@ archive: nmember: fcst: 51 hcst: 25 - regrid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" + reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" era5: src: "recon/ecmwf/era5/" @@ -26,15 +26,15 @@ archive: lon_circular_sort: ini: 0 end: 361 - regrid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" + reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" era5land: src: "recon/ecmwf/era5land/" daily_mean: {"tas":"_f1h/","rsds":"_f1h/", "prlr":"_f1h/","sfcWind":"_f1h/"} - regrid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" + reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" uerra: src: "recon/ecmwf/uerra_mescan/" daily_mean: {"tas":"_f6h/"} - grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" + reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" -- GitLab From 3f00139dc15278364d7fd05c6d0448dcd2428605 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 8 Mar 2022 16:33:19 +0100 Subject: [PATCH 10/31] Added a couple systems to the archive --- conf/archive.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/conf/archive.yml b/conf/archive.yml index 2b945396..eb5a0cfa 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -16,6 +16,30 @@ archive: fcst: 51 hcst: 25 reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" + system7c3s: + src: "exp/meteofrance/system7c3s/" + monthly_mean: {"tas":"_f6h/","g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/"} + lat_decreasing_sort: TRUE + lon_circular_sort: + ini: 0 + end: 361 + nmember: + fcst: 51 + hcst: 25 + reference_grid: "/esarchive/exp/meteofrance/system7c3s/monthly_mean/tas_f6h/tas_20191001.nc" + system35c3s: + src: "/exp/cmcc/system35c3s/" + monthly_mean: {"tas":"_f6h/","g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/"} + lat_decreasing_sort: TRUE + lon_circular_sort: + ini: 0 + end: 361 + nmember: + fcst: 50 + hcst: 40 + reference_grid: "/esarchive/exp/cmcc/system35c3s/monthly_mean/tas_f6h/tas_20210101.nc" era5: src: "recon/ecmwf/era5/" -- GitLab From 50c5867df17cdf6a205ae274954452718ca166e1 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 8 Mar 2022 17:00:52 +0100 Subject: [PATCH 11/31] Removed exp lat_decreasing_sort and lon_circular_sort from the archive --- conf/archive.yml | 12 ------------ modules/data_load/load.R | 17 +++++++++++------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/conf/archive.yml b/conf/archive.yml index eb5a0cfa..b689003c 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -8,10 +8,6 @@ archive: "prlr":"_s0-24h/","sfcWind":"_f6h/"} monthly_mean: {"tas":"_f6h/","rsds":"_s0-24h/", "prlr":"_s0-24h/","sfcWind":"_f6h/"} - lat_decreasing_sort: TRUE - lon_circular_sort: - ini: 0 - end: 361 nmember: fcst: 51 hcst: 25 @@ -20,10 +16,6 @@ archive: src: "exp/meteofrance/system7c3s/" monthly_mean: {"tas":"_f6h/","g500":"_f12h/", "prlr":"_f24h/", "sfcWind": "_f6h/"} - lat_decreasing_sort: TRUE - lon_circular_sort: - ini: 0 - end: 361 nmember: fcst: 51 hcst: 25 @@ -32,10 +24,6 @@ archive: src: "/exp/cmcc/system35c3s/" monthly_mean: {"tas":"_f6h/","g500":"_f12h/", "prlr":"_f24h/", "sfcWind": "_f6h/"} - lat_decreasing_sort: TRUE - lon_circular_sort: - ini: 0 - end: 361 nmember: fcst: 50 hcst: 40 diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 6b82924d..0af64c3e 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -78,6 +78,13 @@ hcst.path <- paste0(archive$src, #------------------------------------------------------------------- regrid_params <- get_regrid_params(recipe, archive) +# Longitude sort +#------------------------------------------------------------------- +if (lons.min >= 0) { + circularsort <- CircularSort(0, 360) +} else { + circularsort <- CircularSort(-180, 180) +} # Timeseries load #------------------------------------------------------------------- @@ -87,12 +94,9 @@ hcst <- Start(dat = hcst.path, file_date = sdates$hcst, time = indices(ltmin:ltmax), latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(decreasing = - exp_descrip$lat_decreasing_sort), + latitude_reorder = Sort(), longitude = values(list(lons.min, lons.max)), - longitude_reorder = CircularSort( - exp_descrip$lon_circular_sort$ini, - exp_descrip$lon_circular_sort$end), + longitude_reorder = circularsort, transform = regrid_params$fcst.transform, transform_params = list(grid = regrid_params$fcst.gridtype, method = regrid_params$fcst.gridmethod, @@ -155,8 +159,9 @@ dates_file <- format(as.Date(dates_file, '%Y%m%d'), "%Y%m") dim(dates_file) <- dim(Subset(hcst, along=c('dat','var', 'latitude', 'longitude', 'member'), - list(1,1,1,1,1), drop="selected")) + list(1,1,1,1,1), drop="selected")) +## TODO: Change latitude and longitude reorder of obs Start call obs <- Start(dat = obs.path, var = variable, file_date = dates_file, -- GitLab From 908455cb121e720f09bfacff1592670807ca0495 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Wed, 9 Mar 2022 12:38:22 +0100 Subject: [PATCH 12/31] Changed variable name to match archive --- modules/data_load/regrid.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data_load/regrid.R b/modules/data_load/regrid.R index 88c74236..5d0a6efe 100644 --- a/modules/data_load/regrid.R +++ b/modules/data_load/regrid.R @@ -8,7 +8,7 @@ get_regrid_params <- function(recipe,archive){ if (tolower(recipe$Analysis$Regrid$type) == 'to_reference') { - regrid_params <- list(fcst.gridtype=reference_descrip$regrid, + regrid_params <- list(fcst.gridtype=reference_descrip$reference_grid, fcst.gridmethod=recipe$Analysis$Regrid$method, fcst.transform=CDORemapper, obs.gridtype=NULL, @@ -20,7 +20,7 @@ get_regrid_params <- function(recipe,archive){ regrid_params <- list(fcst.gridtype=NULL, fcst.gridmethod=NULL, fcst.transform=NULL, - obs.gridtype=exp_descrip$regrid, + obs.gridtype=exp_descrip$reference_grid, obs.gridmethod=recipe$Analysis$Regrid$method, obs.transform=CDORemapper) -- GitLab From 185584172ba95a4a78e84eef8b9a92bbbe0eb5ba Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Wed, 9 Mar 2022 12:57:44 +0100 Subject: [PATCH 13/31] Formatting --- modules/data_load/load.R | 21 +++++++++++---------- modules/data_load/recipe_2.yml | 8 ++++---- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 0af64c3e..50b2cfb2 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -46,7 +46,7 @@ store.freq <- recipe$Analysis$Variables$freq sdates <- dates2load(recipe, logger) # get esarchive datasets dict: -archive <- read_yaml(paste0(recipe$Run$code_dir,"conf/archive.yml"))$archive +archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive exp_descrip <- archive[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable]) @@ -68,11 +68,11 @@ hcst.nmember <- exp_descrip$nmember$hcst # ----------- obs.path <- paste0(archive$src, obs.dir, store.freq, "/$var$", - reference_descrip[[store.freq]][[variable]],"$var$_$file_date$.nc") + reference_descrip[[store.freq]][[variable]], "$var$_$file_date$.nc") hcst.path <- paste0(archive$src, hcst.dir, store.freq, "/$var$", - exp_descrip[[store.freq]][[variable]],"$var$_$file_date$.nc") + exp_descrip[[store.freq]][[variable]], "$var$_$file_date$.nc") # Regrid: #------------------------------------------------------------------- @@ -103,8 +103,8 @@ hcst <- Start(dat = hcst.path, crop = c(lons.min, lons.max, lats.min, lats.max)), transform_vars = c('latitude', 'longitude'), - synonims = list(latitude = c('lat','latitude'), - longitude = c('lon','longitude'), + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude'), member = c('ensemble')), member = indices(1:hcst.nmember), return_vars = list(latitude = 'dat', @@ -128,9 +128,10 @@ startR_to_s2dv <- function(startR_array){ names(dim(dates_end)) <- dates_dims s2dv_object <- s2dv_cube(data = attr2array(startR_array), - lon = attr2array(attr(startR_array,'Variables')$dat1$longitude), - lat = attr2array(attr(startR_array,'Variables')$dat1$latitude), - Variable = list(varName = names(attr(startR_array, "Variables")$common)[2], + lon = attr2array(attr(startR_array, 'Variables')$dat1$longitude), + lat = attr2array(attr(startR_array, 'Variables')$dat1$latitude), + Variable = list(varName = names(attr(startR_array, + 'Variables')$common)[2], level = NULL), Dates = list(start = dates_start, end = dates_end), @@ -172,8 +173,8 @@ obs <- Start(dat = obs.path, longitude_reorder = CircularSort( reference_descrip$lon_circular_sort$ini, reference_descrip$lon_circular_sort$end), - synonims = list(latitude = c('lat','latitude'), - longitude = c('lon','longitude')), + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude')), transform = regrid_params$obs.transform, transform_params = list(grid = regrid_params$obs.gridtype, method = regrid_params$obs.gridmethod, diff --git a/modules/data_load/recipe_2.yml b/modules/data_load/recipe_2.yml index 2775cdf5..d2376b23 100644 --- a/modules/data_load/recipe_2.yml +++ b/modules/data_load/recipe_2.yml @@ -5,7 +5,7 @@ Analysis: Horizon: Seasonal Variables: name: tas - freq: monthly_mean + freq: daily_mean Datasets: System: name: system5c3s @@ -21,10 +21,10 @@ Analysis: leadtimemin: 2 leadtimemax: 4 Region: - latmin: -90 - latmax: 90 + latmin: -10 + latmax: 10 lonmin: 0 - lonmax: 359.9 + lonmax: 20 Regrid: method: bilinear type: to_system -- GitLab From 481f6ed6637492e76b00f2cd665968f2ef97a01a Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Wed, 9 Mar 2022 16:34:21 +0100 Subject: [PATCH 14/31] formatting --- modules/data_load/regrid.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data_load/regrid.R b/modules/data_load/regrid.R index 5d0a6efe..a9b4f62a 100644 --- a/modules/data_load/regrid.R +++ b/modules/data_load/regrid.R @@ -1,5 +1,5 @@ -get_regrid_params <- function(recipe,archive){ +get_regrid_params <- function(recipe, archive){ exp.name <- recipe$Analysis$Datasets$System$name ref.name <- recipe$Analysis$Datasets$Reference$name -- GitLab From 714426fd1354d30fb2b43c81462253ae33150b97 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Wed, 9 Mar 2022 16:34:35 +0100 Subject: [PATCH 15/31] Change latitude and longitude reorder of obs Start call --- modules/data_load/load.R | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 50b2cfb2..c9be5d82 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -113,7 +113,10 @@ hcst <- Start(dat = hcst.path, split_multiselected_dims = TRUE, retrieve = TRUE) +# Conversion from startR_array to s2dv_array +#------------------------------------------------------------------- +## TODO: Terminology? attr2array <- function(attr){ return(array(as.vector(attr), dim(attr))) } @@ -162,17 +165,14 @@ dim(dates_file) <- dim(Subset(hcst, 'latitude', 'longitude', 'member'), list(1,1,1,1,1), drop="selected")) -## TODO: Change latitude and longitude reorder of obs Start call + obs <- Start(dat = obs.path, var = variable, file_date = dates_file, latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(decreasing = - reference_descrip$lat_decreasing_sort), + latitude_reorder = Sort(), longitude = values(list(lons.min, lons.max)), - longitude_reorder = CircularSort( - reference_descrip$lon_circular_sort$ini, - reference_descrip$lon_circular_sort$end), + longitude_reorder = circularsort synonims = list(latitude = c('lat', 'latitude'), longitude = c('lon', 'longitude')), transform = regrid_params$obs.transform, -- GitLab From efa5555bbaf6cd8c8c9e79d23af875cab36bfd32 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Wed, 9 Mar 2022 16:35:24 +0100 Subject: [PATCH 16/31] Remove longitude and latitude sort for reference --- conf/archive.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/conf/archive.yml b/conf/archive.yml index b689003c..9fe54e15 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -34,10 +34,6 @@ archive: daily_mean: {"tas":"_f1h-r1440x721cds/","rsds":"_f1h-r1440x721cds/", "prlr":"_f1h-r1440x721cds/","sfcWind":"_f1h-r1440x721cds/"} monthly_mean: {"tas":"_f1h-r1440x721cds/"} - lat_decreasing_sort: TRUE - lon_circular_sort: - ini: 0 - end: 361 reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" era5land: src: "recon/ecmwf/era5land/" -- GitLab From ed75301565ec07533af6498a6e96496b8cd10ae7 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Wed, 9 Mar 2022 16:57:38 +0100 Subject: [PATCH 17/31] fixed typo and format --- modules/data_load/load.R | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index c9be5d82..3363f96b 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -172,20 +172,20 @@ obs <- Start(dat = obs.path, latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(), longitude = values(list(lons.min, lons.max)), - longitude_reorder = circularsort + longitude_reorder = circularsort, synonims = list(latitude = c('lat', 'latitude'), longitude = c('lon', 'longitude')), transform = regrid_params$obs.transform, transform_params = list(grid = regrid_params$obs.gridtype, - method = regrid_params$obs.gridmethod, - crop = c(lons.min, lons.max, - lats.min, lats.max)), + method = regrid_params$obs.gridmethod, + crop = c(lons.min, lons.max, + lats.min, lats.max)), transform_vars = c('latitude', 'longitude'), - return_vars = list(latitude = 'dat', - longitude = 'dat', - time = 'file_date'), - split_multiselected_dims = TRUE, - retrieve = TRUE) + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'file_date'), + split_multiselected_dims = TRUE, + retrieve = TRUE) ## TODO: Create s2dv object -- GitLab From c6a028664ff9f8696f7d98709b409d015cf8d5da Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Thu, 10 Mar 2022 09:45:59 +0100 Subject: [PATCH 18/31] Removed deprecated argument 'crop' from 'transform_params', added comments --- modules/data_load/load.R | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 3363f96b..2c6d5daa 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -116,7 +116,6 @@ hcst <- Start(dat = hcst.path, # Conversion from startR_array to s2dv_array #------------------------------------------------------------------- -## TODO: Terminology? attr2array <- function(attr){ return(array(as.vector(attr), dim(attr))) } @@ -165,6 +164,8 @@ dim(dates_file) <- dim(Subset(hcst, 'latitude', 'longitude', 'member'), list(1,1,1,1,1), drop="selected")) +## TODO: Not working for daily obs +## TODO: Not loading the correct dates? obs <- Start(dat = obs.path, var = variable, @@ -177,15 +178,13 @@ obs <- Start(dat = obs.path, longitude = c('lon', 'longitude')), transform = regrid_params$obs.transform, transform_params = list(grid = regrid_params$obs.gridtype, - method = regrid_params$obs.gridmethod, - crop = c(lons.min, lons.max, - lats.min, lats.max)), - transform_vars = c('latitude', 'longitude'), - return_vars = list(latitude = 'dat', - longitude = 'dat', - time = 'file_date'), - split_multiselected_dims = TRUE, - retrieve = TRUE) + method = regrid_params$obs.gridmethod), + transform_vars = c('latitude', 'longitude'), + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'file_date'), + split_multiselected_dims = TRUE, + retrieve = TRUE) ## TODO: Create s2dv object -- GitLab From f54eb608e0884014ae2e2633ca218f08a40dd0f2 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Thu, 10 Mar 2022 09:46:21 +0100 Subject: [PATCH 19/31] test monthly data --- modules/data_load/recipe_2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data_load/recipe_2.yml b/modules/data_load/recipe_2.yml index d2376b23..a63eaceb 100644 --- a/modules/data_load/recipe_2.yml +++ b/modules/data_load/recipe_2.yml @@ -5,7 +5,7 @@ Analysis: Horizon: Seasonal Variables: name: tas - freq: daily_mean + freq: monthly_mean Datasets: System: name: system5c3s -- GitLab From 48108e0237ada0d523a75c2e688b11e82e849be5 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Thu, 10 Mar 2022 13:07:04 +0100 Subject: [PATCH 20/31] Removed deprecated argument 'crop' from 'transform_params' from hcst start call --- modules/data_load/load.R | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 2c6d5daa..9cf29136 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -99,9 +99,7 @@ hcst <- Start(dat = hcst.path, longitude_reorder = circularsort, transform = regrid_params$fcst.transform, transform_params = list(grid = regrid_params$fcst.gridtype, - method = regrid_params$fcst.gridmethod, - crop = c(lons.min, lons.max, - lats.min, lats.max)), + method = regrid_params$fcst.gridmethod), transform_vars = c('latitude', 'longitude'), synonims = list(latitude = c('lat', 'latitude'), longitude = c('lon', 'longitude'), -- GitLab From c5b4d79de28c4f72a0f6997f9bb52b3325073d03 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Thu, 10 Mar 2022 15:41:47 +0100 Subject: [PATCH 21/31] Rearranged some parts of the code --- modules/data_load/load.R | 68 ++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 9cf29136..4d9fe10f 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -86,7 +86,7 @@ if (lons.min >= 0) { circularsort <- CircularSort(-180, 180) } -# Timeseries load +# Hindcast timeseries load #------------------------------------------------------------------- hcst <- Start(dat = hcst.path, @@ -111,6 +111,38 @@ hcst <- Start(dat = hcst.path, split_multiselected_dims = TRUE, retrieve = TRUE) +# Get forecast dates +dates <- attr(hcst, 'Variables')$common$time +dates_file <- sapply(dates, format, '%Y%m%d') +dates_file <- format(as.Date(dates_file, '%Y%m%d'), "%Y%m") +dim(dates_file) <- dim(Subset(hcst, + along=c('dat','var', + 'latitude', 'longitude', 'member'), + list(1,1,1,1,1), drop="selected")) + +# Reference timeseries load +#------------------------------------------------------------------- +## TODO: Not working for daily obs + +obs <- Start(dat = obs.path, + var = variable, + file_date = dates_file, + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = circularsort, + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude')), + transform = regrid_params$obs.transform, + transform_params = list(grid = regrid_params$obs.gridtype, + method = regrid_params$obs.gridmethod), + transform_vars = c('latitude', 'longitude'), + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'file_date'), + split_multiselected_dims = TRUE, + retrieve = TRUE) + # Conversion from startR_array to s2dv_array #------------------------------------------------------------------- @@ -147,45 +179,13 @@ startR_to_s2dv <- function(startR_array){ } hcst <- startR_to_s2dv(hcst) - +obs <- startR_to_s2dv(obs) ## TODO: new files checker? #hcst.NA_files <- c(attributes(hcst)$NotFoundFiles) #hcst.NA_files <- hcst.NA_files[!is.na(hcst.NA_files)] #try(hcst.NA_files <- hcst.NA_files[order(hcst.NA_files)]) -dates <- attr(hcst, 'Variables')$common$time -dates_file <- sapply(dates, format, '%Y%m%d') -dates_file <- format(as.Date(dates_file, '%Y%m%d'), "%Y%m") -dim(dates_file) <- dim(Subset(hcst, - along=c('dat','var', - 'latitude', 'longitude', 'member'), - list(1,1,1,1,1), drop="selected")) - -## TODO: Not working for daily obs -## TODO: Not loading the correct dates? - -obs <- Start(dat = obs.path, - var = variable, - file_date = dates_file, - latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(), - longitude = values(list(lons.min, lons.max)), - longitude_reorder = circularsort, - synonims = list(latitude = c('lat', 'latitude'), - longitude = c('lon', 'longitude')), - transform = regrid_params$obs.transform, - transform_params = list(grid = regrid_params$obs.gridtype, - method = regrid_params$obs.gridmethod), - transform_vars = c('latitude', 'longitude'), - return_vars = list(latitude = 'dat', - longitude = 'dat', - time = 'file_date'), - split_multiselected_dims = TRUE, - retrieve = TRUE) - - ## TODO: Create s2dv object - dates_file <- paste0(dates_file, '01') dim(dates_file) <- dim(Subset(hcst, along=c('dat','var', -- GitLab From 46d625ebeaa914bc5afa3f09b526663b2296b171 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Fri, 11 Mar 2022 15:01:22 +0100 Subject: [PATCH 22/31] formatting --- modules/data_load/load.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 4d9fe10f..7096170d 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -126,13 +126,13 @@ dim(dates_file) <- dim(Subset(hcst, obs <- Start(dat = obs.path, var = variable, - file_date = dates_file, + file_date = dates_file, latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(), longitude = values(list(lons.min, lons.max)), longitude_reorder = circularsort, synonims = list(latitude = c('lat', 'latitude'), - longitude = c('lon', 'longitude')), + longitude = c('lon', 'longitude')), transform = regrid_params$obs.transform, transform_params = list(grid = regrid_params$obs.gridtype, method = regrid_params$obs.gridmethod), -- GitLab From 8eef8ed66b1614c4894920bce754636ecac8f8ee Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Fri, 11 Mar 2022 16:25:25 +0100 Subject: [PATCH 23/31] formatting --- modules/data_load/load.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 7096170d..253d2de3 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -71,8 +71,8 @@ obs.path <- paste0(archive$src, reference_descrip[[store.freq]][[variable]], "$var$_$file_date$.nc") hcst.path <- paste0(archive$src, - hcst.dir, store.freq, "/$var$", - exp_descrip[[store.freq]][[variable]], "$var$_$file_date$.nc") + hcst.dir, store.freq, "/$var$", + exp_descrip[[store.freq]][[variable]], "$var$_$file_date$.nc") # Regrid: #------------------------------------------------------------------- @@ -88,7 +88,7 @@ if (lons.min >= 0) { # Hindcast timeseries load #------------------------------------------------------------------- - +## TODO: Adapt to daily data hcst <- Start(dat = hcst.path, var = variable, file_date = sdates$hcst, -- GitLab From f4c151c8ba7e5e22745d5a386c00e1be71e8e032 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Mon, 14 Mar 2022 10:50:07 +0100 Subject: [PATCH 24/31] Format fixes --- modules/data_load/load.R | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) mode change 100644 => 100755 modules/data_load/load.R diff --git a/modules/data_load/load.R b/modules/data_load/load.R old mode 100644 new mode 100755 index 253d2de3..35ddc9c6 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -94,9 +94,9 @@ hcst <- Start(dat = hcst.path, file_date = sdates$hcst, time = indices(ltmin:ltmax), latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(), + latitude_reorder = Sort(), longitude = values(list(lons.min, lons.max)), - longitude_reorder = circularsort, + longitude_reorder = circularsort, transform = regrid_params$fcst.transform, transform_params = list(grid = regrid_params$fcst.gridtype, method = regrid_params$fcst.gridmethod), @@ -126,7 +126,7 @@ dim(dates_file) <- dim(Subset(hcst, obs <- Start(dat = obs.path, var = variable, - file_date = dates_file, + file_date = dates_file, latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(), longitude = values(list(lons.min, lons.max)), @@ -171,7 +171,6 @@ startR_to_s2dv <- function(startR_array){ source_files = attr2array(attr(startR_array, "Files")) #Datasets = list(exp1 = list(InitializationsDates = list(Member_1 = "01011990", # Members = "Member_1"))) - ) return(s2dv_object) -- GitLab From 5d1c4a1e004f4129f4dcd221eb716b98b699f952 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Mon, 14 Mar 2022 10:50:55 +0100 Subject: [PATCH 25/31] Changes for testing --- modules/data_load/recipe_1.yml | 4 ++-- modules/data_load/recipe_2.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/data_load/recipe_1.yml b/modules/data_load/recipe_1.yml index 69ef8245..ca99311f 100644 --- a/modules/data_load/recipe_1.yml +++ b/modules/data_load/recipe_1.yml @@ -39,5 +39,5 @@ Analysis: Run: Loglevel: INFO Terminal: yes - output_dir: /esarchive/scratch/lpalma/git/auto-s2s/out-logs/ - code_dir: /esarchive/scratch/lpalma/git/auto-s2s/ + output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ diff --git a/modules/data_load/recipe_2.yml b/modules/data_load/recipe_2.yml index a63eaceb..d7272e90 100644 --- a/modules/data_load/recipe_2.yml +++ b/modules/data_load/recipe_2.yml @@ -5,13 +5,13 @@ Analysis: Horizon: Seasonal Variables: name: tas - freq: monthly_mean + freq: daily_mean Datasets: System: name: system5c3s Multimodel: no Reference: - name: era5 + name: era5land Time: sdate: fcst_syear: '2017' -- GitLab From c9c010c379c4680d0b1e3207942d34eeae18b3fe Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Mon, 14 Mar 2022 10:51:33 +0100 Subject: [PATCH 26/31] Added monthly mean to era5land --- conf/archive.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/archive.yml b/conf/archive.yml index 9fe54e15..ec0f743c 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -39,6 +39,7 @@ archive: src: "recon/ecmwf/era5land/" daily_mean: {"tas":"_f1h/","rsds":"_f1h/", "prlr":"_f1h/","sfcWind":"_f1h/"} + monthly_mean: {"tas":"_f1h/"} reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" uerra: src: "recon/ecmwf/uerra_mescan/" -- GitLab From ae0529429ff53276337fb9f74610aeae6b27da06 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 15 Mar 2022 11:43:44 +0100 Subject: [PATCH 27/31] Rename dim according to conventions --- modules/data_load/load.R | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 35ddc9c6..4f868ec9 100755 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -10,8 +10,8 @@ source("tools/libs.R") # RECIPE FOR TESTING # ------------------------------------------------------------------------------------------- ## TODO: Get only the last part of the path as the recipe$filename? -recipe <- read_yaml("/esarchive/scratch/vagudets/repos/auto-s2s/modules/data_load/recipe_2.yml") -args <- NULL; args[1] <- "/esarchive/scratch/vagudets/repos/auto-s2s/modules/data_load/recipe_2.yml" +recipe <- read_yaml("modules/data_load/recipe_1.yml") +args <- NULL; args[1] <- "modules/data_load/recipe_1.yml" recipe$filename <- args[1] # Create output folder and log: logger <- prepare_outputs(recipe = recipe) @@ -74,7 +74,7 @@ hcst.path <- paste0(archive$src, hcst.dir, store.freq, "/$var$", exp_descrip[[store.freq]][[variable]], "$var$_$file_date$.nc") -# Regrid: +# Define regrid parameters: #------------------------------------------------------------------- regrid_params <- get_regrid_params(recipe, archive) @@ -102,9 +102,8 @@ hcst <- Start(dat = hcst.path, method = regrid_params$fcst.gridmethod), transform_vars = c('latitude', 'longitude'), synonims = list(latitude = c('lat', 'latitude'), - longitude = c('lon', 'longitude'), - member = c('ensemble')), - member = indices(1:hcst.nmember), + longitude = c('lon', 'longitude')), + ensemble = indices(1:hcst.nmember), return_vars = list(latitude = 'dat', longitude = 'dat', time = 'file_date'), @@ -127,6 +126,7 @@ dim(dates_file) <- dim(Subset(hcst, obs <- Start(dat = obs.path, var = variable, file_date = dates_file, +# time = indices(ltmin:ltmax), latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(), longitude = values(list(lons.min, lons.max)), @@ -137,11 +137,11 @@ obs <- Start(dat = obs.path, transform_params = list(grid = regrid_params$obs.gridtype, method = regrid_params$obs.gridmethod), transform_vars = c('latitude', 'longitude'), - return_vars = list(latitude = 'dat', + return_vars = list(latitude = 'dat', longitude = 'dat', time = 'file_date'), - split_multiselected_dims = TRUE, - retrieve = TRUE) + split_multiselected_dims = TRUE, + retrieve = TRUE) # Conversion from startR_array to s2dv_array #------------------------------------------------------------------- -- GitLab From df4f75f1cc5b9b7659972f6d86385ace8631a1c5 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 15 Mar 2022 11:46:36 +0100 Subject: [PATCH 28/31] Finish renaming 'member' to 'ensemble' --- modules/data_load/load.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 4f868ec9..c394a007 100755 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -116,7 +116,7 @@ dates_file <- sapply(dates, format, '%Y%m%d') dates_file <- format(as.Date(dates_file, '%Y%m%d'), "%Y%m") dim(dates_file) <- dim(Subset(hcst, along=c('dat','var', - 'latitude', 'longitude', 'member'), + 'latitude', 'longitude', 'ensemble'), list(1,1,1,1,1), drop="selected")) # Reference timeseries load @@ -188,13 +188,13 @@ obs <- startR_to_s2dv(obs) dates_file <- paste0(dates_file, '01') dim(dates_file) <- dim(Subset(hcst, along=c('dat','var', - 'latitude', 'longitude', 'member'), + 'latitude', 'longitude', 'ensemble'), list(1,1,1,1,1), drop="selected")) file_dates <- paste0(file_dates, '01') dim(file_dates) <- dim(Subset(hcst, along=c('dat','var', - 'latitude', 'longitude', 'member', 'time'), + 'latitude', 'longitude', 'ensemble', 'time'), list(1,1,1,1,1,1), drop="selected")) @@ -240,7 +240,7 @@ info(logger, default_dims <- c(dat = 1, var = 1, sweek = 1, sday = 1, syear = 1, time = 1, - latitude = 1, longitude = 1, member = 1) + latitude = 1, longitude = 1, ensemble = 1) default_dims[names(dim(obs))] <- dim(obs) dim(obs) <- default_dims -- GitLab From 80f6b3aa56ed0c13b329686caf3ddc450410d666 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 15 Mar 2022 12:39:45 +0100 Subject: [PATCH 29/31] Removed duplicated dates from dates_file array --- modules/data_load/load.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/data_load/load.R b/modules/data_load/load.R index c394a007..541b2094 100755 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -114,8 +114,9 @@ hcst <- Start(dat = hcst.path, dates <- attr(hcst, 'Variables')$common$time dates_file <- sapply(dates, format, '%Y%m%d') dates_file <- format(as.Date(dates_file, '%Y%m%d'), "%Y%m") +dates_file <- dates_file[!duplicated(dates_file)] # Avoid duplicates dim(dates_file) <- dim(Subset(hcst, - along=c('dat','var', + along=c('dat', 'var', 'latitude', 'longitude', 'ensemble'), list(1,1,1,1,1), drop="selected")) @@ -123,7 +124,7 @@ dim(dates_file) <- dim(Subset(hcst, #------------------------------------------------------------------- ## TODO: Not working for daily obs -obs <- Start(dat = obs.path, +obs <- Start(dat = obs.path, var = variable, file_date = dates_file, # time = indices(ltmin:ltmax), -- GitLab From 27ed5c97ad6ab8ac4aae8edddc4c3ca48875adcd Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 15 Mar 2022 12:39:55 +0100 Subject: [PATCH 30/31] testing --- modules/data_load/recipe_1.yml | 6 +++--- modules/data_load/recipe_2.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/data_load/recipe_1.yml b/modules/data_load/recipe_1.yml index ca99311f..a6cdda67 100644 --- a/modules/data_load/recipe_1.yml +++ b/modules/data_load/recipe_1.yml @@ -21,10 +21,10 @@ Analysis: leadtimemin: 2 leadtimemax: 4 Region: - latmin: -90 - latmax: 90 + latmin: -10 + latmax: 10 lonmin: 0 - lonmax: 360 + lonmax: 20 Regrid: method: bilinear type: to_system diff --git a/modules/data_load/recipe_2.yml b/modules/data_load/recipe_2.yml index d7272e90..d2376b23 100644 --- a/modules/data_load/recipe_2.yml +++ b/modules/data_load/recipe_2.yml @@ -11,7 +11,7 @@ Analysis: name: system5c3s Multimodel: no Reference: - name: era5land + name: era5 Time: sdate: fcst_syear: '2017' -- GitLab From 85f296f49e4839374498b7d50c9321c69194fee6 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Wed, 16 Mar 2022 08:55:55 +0100 Subject: [PATCH 31/31] Add second level 'System' and 'Reference' to archive --- conf/archive.yml | 89 +++++++++++++++++++------------------- modules/data_load/load.R | 16 ++++--- modules/data_load/regrid.R | 4 +- 3 files changed, 56 insertions(+), 53 deletions(-) diff --git a/conf/archive.yml b/conf/archive.yml index ec0f743c..0feb34b8 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -1,49 +1,50 @@ archive: - src: "/esarchive/" - system5c3s: - src: "exp/ecmwf/system5c3s/" - daily_mean: {"tas":"_f6h/","rsds":"_s0-24h/", - "prlr":"_s0-24h/","sfcWind":"_f6h/"} - monthly_mean: {"tas":"_f6h/","rsds":"_s0-24h/", - "prlr":"_s0-24h/","sfcWind":"_f6h/"} - nmember: - fcst: 51 - hcst: 25 - reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" - system7c3s: - src: "exp/meteofrance/system7c3s/" - monthly_mean: {"tas":"_f6h/","g500":"_f12h/", - "prlr":"_f24h/", "sfcWind": "_f6h/"} - nmember: - fcst: 51 - hcst: 25 - reference_grid: "/esarchive/exp/meteofrance/system7c3s/monthly_mean/tas_f6h/tas_20191001.nc" - system35c3s: - src: "/exp/cmcc/system35c3s/" - monthly_mean: {"tas":"_f6h/","g500":"_f12h/", - "prlr":"_f24h/", "sfcWind": "_f6h/"} - nmember: - fcst: 50 - hcst: 40 - reference_grid: "/esarchive/exp/cmcc/system35c3s/monthly_mean/tas_f6h/tas_20210101.nc" - - era5: - src: "recon/ecmwf/era5/" - daily_mean: {"tas":"_f1h-r1440x721cds/","rsds":"_f1h-r1440x721cds/", - "prlr":"_f1h-r1440x721cds/","sfcWind":"_f1h-r1440x721cds/"} - monthly_mean: {"tas":"_f1h-r1440x721cds/"} - reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" - era5land: - src: "recon/ecmwf/era5land/" - daily_mean: {"tas":"_f1h/","rsds":"_f1h/", - "prlr":"_f1h/","sfcWind":"_f1h/"} - monthly_mean: {"tas":"_f1h/"} - reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" - uerra: - src: "recon/ecmwf/uerra_mescan/" - daily_mean: {"tas":"_f6h/"} - reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" + src: "/esarchive/" + System: + system5c3s: + src: "exp/ecmwf/system5c3s/" + daily_mean: {"tas":"_f6h/","rsds":"_s0-24h/", + "prlr":"_s0-24h/","sfcWind":"_f6h/"} + monthly_mean: {"tas":"_f6h/","rsds":"_s0-24h/", + "prlr":"_s0-24h/","sfcWind":"_f6h/"} + nmember: + fcst: 51 + hcst: 25 + reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" + system7c3s: + src: "exp/meteofrance/system7c3s/" + monthly_mean: {"tas":"_f6h/","g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/"} + nmember: + fcst: 51 + hcst: 25 + reference_grid: "/esarchive/exp/meteofrance/system7c3s/monthly_mean/tas_f6h/tas_20191001.nc" + system35c3s: + src: "/exp/cmcc/system35c3s/" + monthly_mean: {"tas":"_f6h/","g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/"} + nmember: + fcst: 50 + hcst: 40 + reference_grid: "/esarchive/exp/cmcc/system35c3s/monthly_mean/tas_f6h/tas_20210101.nc" + Reference: + era5: + src: "recon/ecmwf/era5/" + daily_mean: {"tas":"_f1h-r1440x721cds/","rsds":"_f1h-r1440x721cds/", + "prlr":"_f1h-r1440x721cds/","sfcWind":"_f1h-r1440x721cds/"} + monthly_mean: {"tas":"_f1h-r1440x721cds/"} + reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" + era5land: + src: "recon/ecmwf/era5land/" + daily_mean: {"tas":"_f1h/","rsds":"_f1h/", + "prlr":"_f1h/","sfcWind":"_f1h/"} + monthly_mean: {"tas":"_f1h/"} + reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" + uerra: + src: "recon/ecmwf/uerra_mescan/" + daily_mean: {"tas":"_f6h/"} + reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 541b2094..1e841c98 100755 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -47,10 +47,10 @@ sdates <- dates2load(recipe, logger) # get esarchive datasets dict: archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive -exp_descrip <- archive[[exp.name]] +exp_descrip <- archive$System[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable]) -reference_descrip <- archive[[ref.name]] +reference_descrip <- archive$Reference[[ref.name]] freq.obs <- unlist(reference_descrip[[store.freq]][variable]) obs.dir <- reference_descrip$src fcst.dir <- exp_descrip$src @@ -88,7 +88,6 @@ if (lons.min >= 0) { # Hindcast timeseries load #------------------------------------------------------------------- -## TODO: Adapt to daily data hcst <- Start(dat = hcst.path, var = variable, file_date = sdates$hcst, @@ -110,7 +109,8 @@ hcst <- Start(dat = hcst.path, split_multiselected_dims = TRUE, retrieve = TRUE) -# Get forecast dates +# Get forecast dates +## TODO: Adapt for daily data dates <- attr(hcst, 'Variables')$common$time dates_file <- sapply(dates, format, '%Y%m%d') dates_file <- format(as.Date(dates_file, '%Y%m%d'), "%Y%m") @@ -118,11 +118,11 @@ dates_file <- dates_file[!duplicated(dates_file)] # Avoid duplicates dim(dates_file) <- dim(Subset(hcst, along=c('dat', 'var', 'latitude', 'longitude', 'ensemble'), - list(1,1,1,1,1), drop="selected")) + list(1,1,1,1,1), drop="selected")) -# Reference timeseries load +# Reference load #------------------------------------------------------------------- -## TODO: Not working for daily obs +## TODO: Time dimension not working for daily obs obs <- Start(dat = obs.path, var = variable, @@ -181,6 +181,8 @@ startR_to_s2dv <- function(startR_array){ hcst <- startR_to_s2dv(hcst) obs <- startR_to_s2dv(obs) +## TODO: Review/modify code from here onwards. + ## TODO: new files checker? #hcst.NA_files <- c(attributes(hcst)$NotFoundFiles) #hcst.NA_files <- hcst.NA_files[!is.na(hcst.NA_files)] diff --git a/modules/data_load/regrid.R b/modules/data_load/regrid.R index a9b4f62a..9aaab6e4 100644 --- a/modules/data_load/regrid.R +++ b/modules/data_load/regrid.R @@ -3,8 +3,8 @@ get_regrid_params <- function(recipe, archive){ exp.name <- recipe$Analysis$Datasets$System$name ref.name <- recipe$Analysis$Datasets$Reference$name - exp_descrip <- archive[[exp.name]] - reference_descrip <- archive[[ref.name]] + exp_descrip <- archive$System[[exp.name]] + reference_descrip <- archive$Reference[[ref.name]] if (tolower(recipe$Analysis$Regrid$type) == 'to_reference') { -- GitLab