From 9be0bd83b4bccdb7de4f83c2941210fbf120d574 Mon Sep 17 00:00:00 2001 From: nperez Date: Mon, 16 Aug 2021 18:15:48 +0200 Subject: [PATCH 1/8] Accept None to define Indicators or Variables --- conf/indicators_table.yml | 19 +++++++++++++++++++ main.R | 40 --------------------------------------- recipes/conf_user.yml | 4 ++-- tools/check_recipe.R | 21 +++++++++++++++++++- tools/libs.R | 4 ++-- 5 files changed, 43 insertions(+), 45 deletions(-) create mode 100644 conf/indicators_table.yml delete mode 100644 main.R diff --git a/conf/indicators_table.yml b/conf/indicators_table.yml new file mode 100644 index 00000000..a2777461 --- /dev/null +++ b/conf/indicators_table.yml @@ -0,0 +1,19 @@ + +gdd: + longname: 'Growing Degree Days' + ECVs: tas + freq: daily + function: AccumulationExceedingThreshold + +gst: + longname: 'Growing Season Temperature' + ECVs: tas + freq: daily + function: PeriodMean + +spr32: + longname: 'Spring Heat Stress Days - 32°C' + ECVs: tasmax + freq: daily + function: TotalTimeExceedingThreshold + diff --git a/main.R b/main.R deleted file mode 100644 index 648b943a..00000000 --- a/main.R +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env Rscript -args = commandArgs(trailingOnly = TRUE) - -# Author: N.Pérez-Zanón 26th July 2021 -# execution: Rscript OperationalCS.R conf_user.yml recipe.yml -# This code should process and check the recipe and build the workflow -print(args) -library(yaml) -conf <- read_yaml(args[1]) # conf <- read_yaml("recipes/conf_user.yml") -recipe <- read_yaml(args[2]) # recipe <- read_yaml("recipes/seasonal_oper.yml") - -# Load required libraries -source("tools/libs.R") - -#check_conf(conf, file = args[1]) -#verifications <- check_recipe(recipe, file = args[2], conf) -# check_conf(conf, file = "conf_user.yml") -# verifications <- check_recipe(recipe, file = "recipe.yml", conf) - -recipe <- recipe$Analysis - -variable <- recipe$Variables$ECVs[[1]]$name -fcst.sdate <- recipe$Time$sdates -fcst.freq <- recipe$Variables$ECVs[[1]]$freq -hcst.inityear <- recipe$Time$hcst_start -hcst.endyear <- recipe$Time$hcst_end - -fcst.name <- recipe$Datasets$System[[1]]$name -ref.name <- recipe$Datasets$Reference[[1]]$name -ltmin <- recipe$Time$leadtimemin -ltmax <- recipe$Time$leadtimemax -lats.min <- recipe$Region$Regional[[1]]$latmin -lats.max <- recipe$Region$Regional[[1]]$latmax -lons.min <- recipe$Region$Regional[[1]]$lonmin -lons.max <- recipe$Region$Regional[[1]]$lonmax -stream <- "fcst" - -source("modules/data_load/load.R") - - diff --git a/recipes/conf_user.yml b/recipes/conf_user.yml index 2860105d..490da5e2 100644 --- a/recipes/conf_user.yml +++ b/recipes/conf_user.yml @@ -1,4 +1,4 @@ # Users configuration to point correct directories -output_dir: /esarchive/scratch/lpalma/tmp/ -code_dir: /esarchive/scratch/lpalma/git/auto-s2s/ +output_dir: /esarchive/scratch/nperez/git/auto-s2s/out-logs/ +code_dir: /esarchive/scratch/nperez/git/auto-s2s/ diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 76759233..9015453d 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -7,7 +7,8 @@ check_recipe <- function(recipe, file, conf) { if (!any(names(recipe) %in% "Analysis")) { stop("The recipe should has a element 'Analysis'.") } - params <- c('Horizon', 'Time', 'Variables', 'Region', 'Regrid', 'Workflow', 'Datasets') + params <- c('Horizon', 'Time', 'Variables', 'Region', + 'Regrid', 'Workflow', 'Datasets') if (!all(params %in% names(recipe$Analysis))) { stop(paste("The element 'Analysis' in the recipe should contain these", "elements:", paste(params, collapse = " "))) @@ -24,6 +25,24 @@ check_recipe <- function(recipe, file, conf) { stop("Only 1 Horizon can be specified in the recipe") } nvar <- length(recipe$Analysis$Variables) + if (nvar > 2) { + stop("Only two type of Variables can be listed: ECVs and Indicators.") + } + # remove NULL or None Indicators or ECVs from the recipe: + if (!is.list(recipe$Analysis$Variables$Indicators)) { + recipe$Analysis$Variables <- recipe$Analysis$Variables[ + -which(names(recipe$Analysis$Variables) == 'Indicators')] + #assign(recipe$Analysis$Variables, recipe$Analysis$Variables[ + # -which(names(recipe$Analysis$Variables) == 'Indicators')], + # envir = parent.frame()) + } + if (!is.list(recipe$Analysis$Variables$ECVs)) { + recipe$Analysis$Variables <- recipe$Analysis$Variables[ + -which(names(recipe$Analysis$Variables) == 'ECVs')] + #assign(recipe$Analysis$Variables, recipe$Analysis$Variables[ + # -which(names(recipe$Analysis$Variables) == 'ECVs')], + # envir = parent.frame()) + } if (length(recipe$Analysis$Region) > 3) { stop("The section 'Region' in the recipe should be checked.", "Global, Countries and Regional elements are expected.") diff --git a/tools/libs.R b/tools/libs.R index 9ac904f1..c975d393 100644 --- a/tools/libs.R +++ b/tools/libs.R @@ -21,6 +21,6 @@ # source("R_CST_MergeDims.R") #setwd(conf$code_dir) # # To be removed when new package is done by library(CSOperational) - source("/esarchive/scratch/lpalma/git/auto-s2s/tools/check_recipe.R") - source("/esarchive/scratch/lpalma/git/auto-s2s/tools/check_conf.R") + source("/esarchive/scratch/nperez/git/auto-s2s/tools/check_recipe.R") + source("/esarchive/scratch/nperez/git/auto-s2s/tools/check_conf.R") -- GitLab From 48646f3c90baa26986babda65c6067425d32c601 Mon Sep 17 00:00:00 2001 From: nperez Date: Tue, 17 Aug 2021 18:22:52 +0200 Subject: [PATCH 2/8] Reviewing check_recipe --- OperationalCS.R | 1 + recipes/seasonal_complex.yml | 41 +++++++++++++++ tools/check_recipe.R | 99 ++++++++++++++++-------------------- tools/libs.R | 4 +- 4 files changed, 89 insertions(+), 56 deletions(-) create mode 100644 recipes/seasonal_complex.yml diff --git a/OperationalCS.R b/OperationalCS.R index 03bfe7d1..7ddf9d48 100644 --- a/OperationalCS.R +++ b/OperationalCS.R @@ -8,6 +8,7 @@ print(args) library(yaml) conf <- read_yaml(args[1]) # conf <- read_yaml("recipes/conf_user.yml") recipe <- read_yaml(args[2]) # recipe <- read_yaml("recipes/seasonal_oper.yml") +# recipe <- read_yaml("recipes/seasonal_complex.yml") # Load required libraries source("tools/libs.R") diff --git a/recipes/seasonal_complex.yml b/recipes/seasonal_complex.yml new file mode 100644 index 00000000..c6ce15ed --- /dev/null +++ b/recipes/seasonal_complex.yml @@ -0,0 +1,41 @@ +Description: + Author: N.Pérez-Zanón + Info: This is a test to transform s2s4e data-analysis for SEAS5 + +Analysis: + Horizon: Seasonal + Variables: + ECVs: + - {name: tas, freq: monthly_mean} + - {name: tas, freq: daily_mean} + Indicators: + - {name: gdd} + Datasets: + System: + - name: system5c3s + Multimodel: False + Reference: + - {name: ERA5} + Time: + sdates: 20210701 + hcst_start: 2000 + hcst_end: 2002 + leadtimemin: 2 + leadtimemax: 4 + Region: + Global: TRUE + Aggregation: False + Regional: + - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} + - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} + Regrid: + method: ycon + type: 180x90 + Workflow: + Calibration: + method: SBC + Skill: + metric: RPSS + Indicators: + index: False + Output_format: S2S4E diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 9015453d..e88e41fd 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -83,75 +83,66 @@ check_number_of_dependent_verifications <- function(recipe, conf) { independent_verifications <- NULL dependent_verifications <- NULL dep <- 1 - if (is.null(recipe$Analysis$Variables$Indicators) || - (length(recipe$Analysis$Variables$Indicators) == 1 && - is.null(recipe$Analysis$Variables$ECVs))) { - # look for duplications - vars <- unlist(recipe$Analysis$Variables) - if (any(duplicated(vars))) { - stop("There are duplicated variables.") + # check workflow order: + if (all(c('Calibration', 'Indicators') %in% names(recipe$Analysis$Workflow))) { + cal_pos <- which(names(recipe$Analysis$Workflow) == 'Calibration') + ind_pos <- which(names(recipe$Analysis$Workflow) == 'Indicators') + if (cal_pos < ind_pos) { + workflow_independent <- FALSE + } else { + workflow_independent <- TRUE } - independent_verifications <- as.vector(vars) - } else if (length(names(recipe$Analysis$Variables)) == 2 || - length(recipe$Analysis$Variables$Indicators) > 1) { - ecvs <- recipe$Analysi$Variables$ECVs$name - inds <- recipe$Analysi$Variables$Indicators$name - ind_table <- read_yaml(paste0(conf$code_dir, - "conf/vars-dict.yml")) - ind_table <- ind_table[unlist(inds)] - vars_for_ind <- NULL - # look for ecvs required to compute indicators - for (ind in unlist(inds)) { - if (is.null(ind_table[ind][[1]])) { - stop("Check the name of the indicators.") - } - vars_for_ind <- c(vars_for_ind, ind_table[ind][[1]]$ECVs) - } - if (any(vars_for_ind %in% unlist(ecvs)) || - any(duplicated(vars_for_ind))) { - # check workflow order to know the number of verifications to calculate - if (all(c('Calibration', 'Indicators') %in% names(recipe$Analysis$Workflow))) { - cal_pos <- which(names(recipe$Analysis$Workflow) == 'Calibration') - ind_pos <- which(names(recipe$Analysis$Workflow) == 'Indicators') - if (cal_pos < ind_pos) { - workflow_independent <- FALSE - } else { - workflow_independent <- TRUE + } + if (workflow_independent) { + independent_verifications <- recipe$Analysis$Variables + } else { + if (is.null(recipe$Analysis$Variables$Indicators) || + (length(recipe$Analysis$Variables$Indicators) == 1 && + is.null(recipe$Analysis$Variables$ECVs))) { + independent_verifications <- recipe$Analysis$Variables + } else { + ecvs <- recipe$Analysi$Variables$ECVs + inds <- recipe$Analysi$Variables$Indicators + ind_table <- read_yaml(paste0(conf$code_dir, + "conf/indicators_table.yml")) + ind_table <- ind_table[unlist(inds)] + vars_for_ind <- NULL + freq_for_ind <- NULL + # look for ecvs required to compute indicators + for (ind in unlist(inds)) { + if (is.null(ind_table[ind][[1]])) { + stop("Check the name of the indicators.") } + vars_for_ind <- c(vars_for_ind, ind_table[ind][[1]]$ECVs) + freq_for_ind <- c(freq_for_ind, ind_table[ind][[1]]$freq) } - } - vars <- unique(c(unlist(ecvs), unlist(vars_for_ind))) - for (var in unlist(vars)) { - if (var %in% vars_for_ind) { - if (workflow_independent) { - if (var %in% unlist(ecvs)) { - varname <- var - } else { - varname <- NULL # avoid writting a ecv not requested by needed by the ind - } - independent_verifications <- c(independent_verifications, varname, - as.vector(unlist(inds[which(unlist(vars_for_ind %in% var))]))) - } else { +# To review from here + # first, check if indicators need common variables + if (any(duplicated(vars_for_ind))) { + } + if (any(vars_for_ind %in% unlist(ecvs)) || + any(duplicated(vars_for_ind))) { + } + vars <- unique(c(unlist(ecvs), unlist(vars_for_ind))) + for (var in unlist(vars)) { + if (var %in% vars_for_ind) { if (var %in% unlist(ecvs)) { varname <- var } else { varname <- NULL } dependent_verifications[[dep]] <- c(varname, - as.vector(unlist(inds[which(unlist(vars_for_ind %in% var))]))) - dep = dep + 1 + as.vector(unlist(inds[which(unlist(vars_for_ind %in% var))]))) + dep = dep + 1 + } else { + independent_verifications <- var } - } else { - independent_verifications <- var } } - } else { - stop("Only two types of 'Variables', named 'ECVs' and 'Indicators',", - " are allawed.") } if (!is.null(independent_verifications)) { message("The variables for independent verification are ", - paste(independent_verifications, collapse = " ")) + paste(independent_verifications, collapse = " ")) } if (!is.null(dependent_verifications)) { message("The variables for dependent verification are: ", diff --git a/tools/libs.R b/tools/libs.R index c975d393..7b0d5bb2 100644 --- a/tools/libs.R +++ b/tools/libs.R @@ -21,6 +21,6 @@ # source("R_CST_MergeDims.R") #setwd(conf$code_dir) # # To be removed when new package is done by library(CSOperational) - source("/esarchive/scratch/nperez/git/auto-s2s/tools/check_recipe.R") - source("/esarchive/scratch/nperez/git/auto-s2s/tools/check_conf.R") + source(paste0(conf$code_dir, "tools/check_recipe.R")) + source(paste0(conf$code_dir, "tools/check_conf.R")) -- GitLab From 210f27f4df24a6d98087ab7c848cf1a21d170a3c Mon Sep 17 00:00:00 2001 From: nperez Date: Thu, 19 Aug 2021 09:20:05 +0200 Subject: [PATCH 3/8] recipe seasonal_oper loads the data --- OperationalCS.R | 16 ++- conf/archive.yml | 19 +-- conf/indicators_table.yml | 6 +- modules/data_load/{load.R => seas5.load.R} | 119 +++++++++------- recipes/seasonal_oper.yml | 2 +- tools/check_recipe.R | 128 +++++++++++++---- tools/libs.R | 2 + ...heck_number_of_independent_verifications.R | 129 ++++++++++++++++++ 8 files changed, 318 insertions(+), 103 deletions(-) rename modules/data_load/{load.R => seas5.load.R} (69%) create mode 100644 tools/test_check_number_of_independent_verifications.R diff --git a/OperationalCS.R b/OperationalCS.R index 7ddf9d48..febab0ce 100644 --- a/OperationalCS.R +++ b/OperationalCS.R @@ -17,12 +17,14 @@ check_conf(conf, file = args[1]) # check_conf(conf, file = "conf_user.yml") verifications <- check_recipe(recipe, file = args[2], conf) # verifications <- check_recipe(recipe, file = "recipe.yml", conf) -for (indep in verifications$independent) { - variable <- indep$name - fcst.sdate <- recipe$Analysis$Time$forecast_sdate - fcst.freq <- indep$freq - hcst.inityear <- recipe$Analysis$Time$hindcast_start - hcst.endyear <- recipe$Analysis$Time$hindcast_end +for (indep in verifications$independent) { + #if (names(indep) + variable <- indep[[1]]$name + store.freq <- indep[[1]]$freq + fcst.sdate <- recipe$Analysis$Time$sdate + #fcst.freq <- indep$freq + hcst.inityear <- recipe$Analysis$Time$hcst_start + hcst.endyear <- recipe$Analysis$Time$hcst_end fcst.name <- recipe$Analysis$Datasets$System[[1]]$name ref.name <- recipe$Analysis$Datasets$Reference[[1]]$name ltmin <- recipe$Analysis$Time$leadtimemin @@ -32,6 +34,6 @@ for (indep in verifications$independent) { lons.min <- recipe$Analysis$Region$Regional[[1]]$lonmin lons.max <- recipe$Analysis$Region$Regional[[1]]$lonmax stream <- "fcst" - #source("modules/data_load/load.R") + source("modules/data_load/seas5.load.R") } diff --git a/conf/archive.yml b/conf/archive.yml index 0981e0d9..d4f1f4af 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -4,10 +4,10 @@ archive: esarchive: "/esarchive/oper/S2S-downscale/" system5c3s: src: "exp/ecmwf/system5c3s/" - daily_mean: {"tas":"tas_f6h/","rsds":"rsds_s0-24h/", - "prlr":"prlr_s0-24h/","sfcWind":"sfcWind_f6h/"} - monthly_mean: {"tas":"tas_f6h/","rsds":"rsds_s0-24h/", - "prlr":"prlr_s0-24h/","sfcWind":"sfcWind_f6h/"} + daily_mean: {"tas":"_f6h/","rsds":"_s0-24h/", + "prlr":"_s0-24h/","sfcWind":"_f6h/"} + monthly_mean: {"tas":"_f6h/","rsds":"_s0-24h/", + "prlr":"_s0-24h/","sfcWind":"_f6h/"} lat_decreasing_sort: TRUE lon_circular_sort: ini: 0 @@ -18,14 +18,15 @@ archive: era5: src: "recon/ecmwf/era5/" - daily_mean: {"tas":"tas_f1h-r1440x721cds/","rsds":"rsds_f1h-r1440x721cds/", - "prlr":"prlr_f1h-r1440x721cds/","sfcWind":"sfcWind_f1h-r1440x721cds/"} + daily_mean: {"tas":"_f1h-r1440x721cds/","rsds":"_f1h-r1440x721cds/", + "prlr":"_f1h-r1440x721cds/","sfcWind":"_f1h-r1440x721cds/"} + monthly_mean: {"tas":"_f1h-r1440x721cds/"} era5land: src: "recon/ecmwf/era5land/" - daily_mean: {"tas":"tas_f1h/","rsds":"rsds_f1h/", - "prlr":"prlr_f1h/","sfcWind":"sfcWind_f1h/"} + daily_mean: {"tas":"_f1h/","rsds":"_f1h/", + "prlr":"_f1h/","sfcWind":"_f1h/"} uerra: src: "recon/ecmwf/uerra_mescan/" - daily_mean: {"tas":"tas_f6h/"} + daily_mean: {"tas":"_f6h/"} diff --git a/conf/indicators_table.yml b/conf/indicators_table.yml index a2777461..f4debcc8 100644 --- a/conf/indicators_table.yml +++ b/conf/indicators_table.yml @@ -2,18 +2,18 @@ gdd: longname: 'Growing Degree Days' ECVs: tas - freq: daily + freq: daily_mean function: AccumulationExceedingThreshold gst: longname: 'Growing Season Temperature' ECVs: tas - freq: daily + freq: daily_mean function: PeriodMean spr32: longname: 'Spring Heat Stress Days - 32°C' ECVs: tasmax - freq: daily + freq: daily_mean function: TotalTimeExceedingThreshold diff --git a/modules/data_load/load.R b/modules/data_load/seas5.load.R similarity index 69% rename from modules/data_load/load.R rename to modules/data_load/seas5.load.R index fac981c6..86af832d 100644 --- a/modules/data_load/load.R +++ b/modules/data_load/seas5.load.R @@ -1,35 +1,48 @@ + + #------------------------------------------------------------------- + # Init + #------------------------------------------------------------------- -fcst.month <- substr(fcst.sdate,5,6) -fcst.year <- substr(fcst.sdate,1,4) - -file_dates.fcst <- paste(fcst.year, fcst.month, sep = "") -file_dates <- paste(strtoi(hcst.inityear):strtoi(hcst.endyear), - fcst.month, sep = "") - -add_dims <- function(data){ - default_dims <- c(sweek = 1, sday = 1, syear = length(data)) - default_dims[names(dim(data))] <- dim(data) - dim(data) <- default_dims - return(data) -} - -file_dates <- add_dims(file_dates) -file_dates.fcst <- add_dims(file_dates.fcst) + fcst.month <- substr(fcst.sdate,5,6) + fcst.year <- substr(fcst.sdate,1,4) + + file_dates.fcst <- paste(fcst.year, fcst.month, sep = "") + file_dates <- paste(strtoi(hcst.inityear):strtoi(hcst.endyear), + fcst.month, sep = "") + + add_dims <- function(data){ + default_dims <- c(sweek = 1, sday = 1, syear = length(data)) + default_dims[names(dim(data))] <- dim(data) + dim(data) <- default_dims + return(data) + } + file_dates <- add_dims(file_dates) + file_dates.fcst <- add_dims(file_dates.fcst) + +# REPLACED +# accum <- FALSE +# switch (tolower(variable), +# "tas" = {freq.hcst <- 'f6h'; freq.obs <- 'f1h'}, +# "tasmin" = {freq.hcst <- 'f24h'; freq.obs <- 'f24h'}, +# "tasmax" = {freq.hcst <- 'f24h'; freq.obs <- 'f24h'}, +# "psl" = {freq.hcst <- 'f6h'; freq.obs <- 'f1h'}, +# "prlr" = {freq.hcst <- 's0-24h'; freq.obs <- 's0-24h'; accum <- TRUE}, +# "rsds" = {freq.hcst <- 's0-24h'; freq.obs <- 's0-24h'; accum <- TRUE}, +# "sfcwind" = {freq.hcst <- 'f6h'; freq.obs <- 'f1h'} +# ) +#------------- table <- read_yaml(paste0(conf$code_dir, "conf/archive.yml")) -dataset_descrip <- table$archive[[fcst.name]] -fcst.nmember <- dataset_descrip$nmember$fcst -hcst.nmember <- dataset_descrip$nmember$hcst +dataset_descrip <- table$archive[which(names(table$archive) == fcst.name)][[1]] +freq.hcst <- unlist(dataset_descrip[store_freq][[1]][variable]) +refere_descrip <- table$archive[which(names(table$archive) == ref.name)][[1]] +freq.obs <- unlist(refere_descrip[store_freq][[1]][variable]) +obs.dir <- refere_descrip$src fcst.dir <- dataset_descrip$src hcst.dir <- dataset_descrip$src - -store_freq <- dataset_descrip[[fcst.freq]][[variable]] -freq.hcst <- unlist(dataset_descrip$store_freq[store_freq]) - -refere_descrip <- table[which(names(table) == ref.name)][[1]] -freq.obs <- unlist(refere_descrip$store_freq[store_freq]) -obs.dir <- refere_descrip$path +fcst.nmember <- dataset_descrip$nmember$fcst +hcst.nmember <- dataset_descrip$nmember$hcst if ("accum" %in% names(refere_descrip)) { accum <- unlist(refere_descrip$accum[store_freq][[1]]) @@ -37,18 +50,18 @@ if ("accum" %in% names(refere_descrip)) { accum <- FALSE } # ----------- - obs.path <- paste0( - obs.dir,fcst.freq,"/$var$_", - freq.obs,"/$var$_$file_date$.nc") + obs.path <- paste0("/esarchive/", + obs.dir, store.freq, "/$var$", + freq.obs,"$var$_$file_date$.nc") - hcst.path <- paste0( - hcst.dir,fcst.freq,"/", "/$var$_", - freq.hcst,"/$var$_$file_date$01", + hcst.path <- paste0("/esarchive/", + hcst.dir, store.freq, "/$var$", + freq.hcst,"$var$_$file_date$01", ".nc") - fcst.path <- paste0( - fcst.dir,fcst.freq,"/", "/$var$_", - freq.hcst,"/$var$_$file_date$01", + fcst.path <- paste0("/esarchive/", + fcst.dir, store.freq, "/$var$", + freq.hcst,"$var$_$file_date$01", ".nc") #------------------------------------------------------------------- # Timeseries load @@ -60,7 +73,7 @@ if ("accum" %in% names(refere_descrip)) { var = variable, file_date = file_dates.fcst, time = indices(ltmin:ltmax), - latitude = values(list(lats.min, lats.max)), + latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(decreasing = TRUE), longitude = values(list(lons.min, lons.max)), longitude_reorder = CircularSort(0, 361), @@ -83,9 +96,9 @@ if ("accum" %in% names(refere_descrip)) { latitude_reorder = Sort(decreasing = TRUE), longitude = values(list(lons.min, lons.max)), longitude_reorder = CircularSort(0, 361), - synonims = list(latitude=c('lat','latitude'), - longitude=c('lon','longitude'), - member=c('ensemble')), + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude'), + member = c('ensemble')), member = indices(1:hcst.nmember), return_vars = list(latitude = 'dat', longitude = 'dat', @@ -105,9 +118,9 @@ if ("accum" %in% names(refere_descrip)) { 'latitude', 'longitude', 'member'), list(1,1,1,1,1), drop="selected")) - obs <- Start(dat = obs.path, + obs <- Start(dat = obs.path, var = variable, - file_date = dates_file, + file_date = dates_file, latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(decreasing = TRUE), longitude = values(list(lons.min, lons.max)), @@ -115,20 +128,20 @@ if ("accum" %in% names(refere_descrip)) { synonims = list(latitude=c('lat','latitude'), longitude=c('lon','longitude')), transform = CDORemapper, - transform_params = list( - grid = paste0( - "/esarchive/exp/ecmwf/system5c3s/",fcst.freq, - "/",variable,"_",freq.hcst, - "/",variable,"_","201805","01.nc" - ), - method = 'con', - crop = c(lons.min,lons.max,lats.min,lats.max)), - transform_vars = c('latitude', 'longitude'), - return_vars = list(latitude = 'dat', + transform_params = list( + grid = paste0( + "/esarchive/exp/ecmwf/system5c3s/", store.freq, + "/",variable, freq.hcst, + "/",variable,"_","201805","01.nc" + ), + method = 'con', + crop = c(lons.min,lons.max,lats.min,lats.max)), + transform_vars = c('latitude', 'longitude'), + return_vars = list(latitude = 'dat', longitude = 'dat', time = 'file_date'), - split_multiselected_dims = TRUE, - retrieve = TRUE) + split_multiselected_dims = TRUE, + retrieve = TRUE) dates_file <- paste0(dates_file, '01') dim(dates_file) <- dim(Subset(hcst, diff --git a/recipes/seasonal_oper.yml b/recipes/seasonal_oper.yml index b9115974..ee9f0114 100644 --- a/recipes/seasonal_oper.yml +++ b/recipes/seasonal_oper.yml @@ -14,7 +14,7 @@ Analysis: - name: system5c3s Multimodel: False Reference: - - {name: ERA5} + - {name: era5} Time: sdates: 20210701 hcst_start: 2000 diff --git a/tools/check_recipe.R b/tools/check_recipe.R index e88e41fd..ee2faed9 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -94,50 +94,118 @@ check_number_of_dependent_verifications <- function(recipe, conf) { } } if (workflow_independent) { - independent_verifications <- recipe$Analysis$Variables +print(recipe$Analysis$Variables$Indicators) + independent_verifications <- append(recipe$Analysis$Variables$ECVs, + recipe$Analysis$Variables$Indicators) } else { if (is.null(recipe$Analysis$Variables$Indicators) || (length(recipe$Analysis$Variables$Indicators) == 1 && is.null(recipe$Analysis$Variables$ECVs))) { +print(recipe$Analysis$Variables) independent_verifications <- recipe$Analysis$Variables } else { ecvs <- recipe$Analysi$Variables$ECVs inds <- recipe$Analysi$Variables$Indicators ind_table <- read_yaml(paste0(conf$code_dir, "conf/indicators_table.yml")) - ind_table <- ind_table[unlist(inds)] - vars_for_ind <- NULL - freq_for_ind <- NULL - # look for ecvs required to compute indicators - for (ind in unlist(inds)) { - if (is.null(ind_table[ind][[1]])) { - stop("Check the name of the indicators.") - } - vars_for_ind <- c(vars_for_ind, ind_table[ind][[1]]$ECVs) - freq_for_ind <- c(freq_for_ind, ind_table[ind][[1]]$freq) - } +# ind_table <- ind_table[unlist(inds)] +# vars_for_ind <- NULL +# freq_for_ind <- NULL +# # look for ecvs required to compute indicators +# for (ind in unlist(inds)) { +# if (is.null(ind_table[ind][[1]])) { +# stop("Check the name of the indicators.") +# } +# vars_for_ind <- c(vars_for_ind, ind_table[ind][[1]]$ECVs) +# freq_for_ind <- c(freq_for_ind, ind_table[ind][[1]]$freq) +# } + # To review from here - # first, check if indicators need common variables - if (any(duplicated(vars_for_ind))) { - } - if (any(vars_for_ind %in% unlist(ecvs)) || - any(duplicated(vars_for_ind))) { - } - vars <- unique(c(unlist(ecvs), unlist(vars_for_ind))) - for (var in unlist(vars)) { - if (var %in% vars_for_ind) { - if (var %in% unlist(ecvs)) { - varname <- var + # first, loop on ecvs if any and compare to indicators + done <- NULL # to gather the indicators reviewed + if (!is.null(ecvs)) { + for (i in 1:length(ecvs)) { + dependent <- ecvs[[i]] +print(i) +print(dependent) + for (j in 1:length(inds)) { +print(j) +print(ind_table[inds[[j]]$name][[1]]$ECVs == ecvs[[i]]$name) +print(ind_table[inds[[j]]$name][[1]]$freq == ecvs[[i]]$freq) + if (ind_table[inds[[j]]$name][[1]]$ECVs == ecvs[[i]]$name) { + if (ind_table[inds[[j]]$name][[1]]$freq == ecvs[[i]]$freq) { + # they are dependent +print("Dependent") + dependent <- list(dependent, inds[[j]]) + done <- append(done, inds[[j]]) + } + } + } + if (!is.null(dependent$name)) { +print("Is it empty?") + dependent <- NULL + independent_verifications <- append(independent_verifications, + list(ecvs[[i]])) } else { - varname <- NULL +print("Dependent") + dependent_verifications <- append(dependent_verifications, + list(dependent)) + } + } + # There are indicators not reviewed yet? + if (length(done) < length(inds)) { + if (length(inds) == 1) { + independent_verifications <- append(independent_verifications, + inds) + } else { + done <- NULL + for (i in 1:(length(inds) - 1)) { + dependent <- list(inds[[i]]$name) + if (is.na(match(unlist(dependent), unlist(done)))) { + for (j in (i+1):length(inds)) { + if (ind_table[inds[[i]]$name][[1]]$ECVs == + ind_table[inds[[j]]$name][[1]]$ECVs) { + if (ind_table[inds[[i]]$name][[1]]$freq == + ind_table[inds[[j]]$name][[1]]$freq) { + dependent <- append(dependent, inds[[j]]$name) + done <- dependent + } + } + } + } + if (length(dependent) == 1) { + independent_verifications <- dependent + dependent <- NULL + } else { + dependent_verifications <- dependent + } + } } - dependent_verifications[[dep]] <- c(varname, - as.vector(unlist(inds[which(unlist(vars_for_ind %in% var))]))) - dep = dep + 1 - } else { - independent_verifications <- var } - } + } else { # there are only Indicators: + done <- NULL + for (i in 1:(length(inds) - 1)) { + dependent <- list(inds[[i]]$name) + if (is.na(match(unlist(dependent), unlist(done)))) { + for (j in (i+1):length(inds)) { + if (ind_table[inds[[i]]$name][[1]]$ECVs == + ind_table[inds[[j]]$name][[1]]$ECVs) { + if (ind_table[inds[[i]]$name][[1]]$freq == + ind_table[inds[[j]]$name][[1]]$freq) { + dependent <- append(dependent, inds[[j]]$name) + done <- dependent + } + } + } + } + if (length(dependent) == 1) { + independent_verifications <- dependent + dependent <- NULL + } else { + dependent_verifications <- dependent + } + } + } } } if (!is.null(independent_verifications)) { diff --git a/tools/libs.R b/tools/libs.R index 7b0d5bb2..d94c3862 100644 --- a/tools/libs.R +++ b/tools/libs.R @@ -1,4 +1,6 @@ +library(startR) +library(ClimProjDiags) # library(s2dverification) # library(ncdf4) # library(abind) diff --git a/tools/test_check_number_of_independent_verifications.R b/tools/test_check_number_of_independent_verifications.R new file mode 100644 index 00000000..cf5acb0e --- /dev/null +++ b/tools/test_check_number_of_independent_verifications.R @@ -0,0 +1,129 @@ +library(testthat) +test_that("A few combinations", { + source("/esarchive/scratch/nperez/git/auto-s2s/tools/check_recipe.R") + recipe <- list(Analysis = + list(Variables = list(ECVs = list( + list(name = 'tas', freq = 'daily_mean')), + Indicators = list(list(name = 'gdd'))), + Workflow = list(Calibration = 'SBC', Indicators = TRUE))) + conf <- list(code_dir = "/esarchive/scratch/nperez/git/auto-s2s/") + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = NULL, + dependent = list(list(list(name = 'tas', freq = 'daily_mean'), + list(name = 'gdd'))))) + recipe <- list(Analysis = + list(Variables = list(ECVs = list( + list(name = 'tas', freq = 'daily_mean')), + Indicators = list(name = 'gdd')), + Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = list(list(name = 'tas', freq = 'daily_mean'), + name = 'gdd'), dependent = NULL)) + +# Cases to review: + recipe <- list(Analysis = + list(Variables = list(ECVs = list( + list(name = list('tas', freq = 'daily')))), + Workflow = list(Indicators = FALSE, Calibration = 'SBC'))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = list(list(list(name = list('tas'), freq = 'daily_mean'))), + dependent = NULL)) + + recipe <- list(Analysis = + list(Variables = list(ECVs = list(list(name = 'tas'), + list(name = 'tasmax'))), + Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = c('tas', 'tasmax'), dependent = NULL)) + + recipe <- list(Analysis = + list(Variables = list(Indicators = list(list(name = 'gdd'))), + Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = c('gdd'), dependent = NULL)) + + recipe <- list(Analysis = + list(Variables = list(ECVs = list(list(name = 'tas')), + Indicators = list(list(name = 'gdd'), + list(name = 'gst'))), + Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = c('tas', 'gdd', 'gst'), dependent = NULL)) + + recipe <- list(Analysis = + list(Variables = list(ECVs = list(list(name = 'tas')), + Indicators = list(list(name = 'gdd'), + list(name = 'gst'))), + Workflow = list(Calibration = 'SBC', Indicators = TRUE))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = NULL, dependent = list(c('tas', 'gdd', 'gst')))) +# Dependent workflow: cal & ind + recipe <- list(Analysis = + list(Variables = list( + Indicators = list(list(name = 'gdd'), + list(name = 'gst'))), + Workflow = list(Calibration = 'SBC', Indicators = TRUE))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = NULL, dependent = list(c('gdd', 'gst')))) + + recipe <- list(Analysis = + list(Variables = list(ECVs = list(list(name = 'tasmax')), + Indicators = list(list(name = 'gdd'), + list(name = 'gst'))), + Workflow = list(Calibration = 'SBC', Indicators = TRUE))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = 'tasmax', dependent = list(c('gdd', 'gst')))) + + recipe <- list(Analysis = + list(Variables = list(ECVs = list(list(name = 'tasmax'), + list(name = 'tas')), + Indicators = list(list(name = 'gdd'), + list(name = 'gst'))), + Workflow = list(Calibration = 'SBC', Indicators = TRUE))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = 'tasmax', dependent = list(c('tas', 'gdd', 'gst')))) + + recipe <- list(Analysis = + list(Variables = list(ECVs = list(list(name = 'tasmax'), + list(name = 'tas')), + Indicators = list(list(name = 'gdd'), + list(name = 'gst'), + list(name = 'spr32'))), + Workflow = list(Calibration = 'SBC', Indicators = TRUE))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = NULL, + dependent = list(c('tasmax', 'spr32'), c('tas', 'gdd', 'gst')))) +# Independent workflow: ind & cal + recipe <- list(Analysis = + list(Variables = list( + Indicators = list(list(name = 'gdd'), + list(name = 'gst'))), + Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = c('gdd', 'gst'), dependent = NULL)) + + recipe <- list(Analysis = + list(Variables = list(ECVs = list(list(name ='tasmax')), + Indicators = list(list(name = 'gdd'), + list(name = 'gst'))), + Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = c('tasmax', 'gdd', 'gst'), dependent = NULL)) + + recipe <- list(Analysis = + list(Variables = list(ECVs = list(list(name = 'tasmax'), list(name = 'tas')), + Indicators = list(list(name = 'gdd'), list(name = 'gst'))), + Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = c('tasmax', 'tas', 'gdd', 'gst'), dependent = NULL)) + + recipe <- list(Analysis = + list(Variables = list(ECVs = list(list(name = 'tasmax'), list(name = 'tas')), + Indicators = list(list(name = 'gdd'), + list(name = 'gst'), list(name = 'spr32'))), + Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) + expect_equal(check_number_of_dependent_verifications(recipe, conf), + list(independent = c('tasmax', 'spr32', 'tas', 'gdd', 'gst'), + dependent = NULL)) +}) + -- GitLab From e341806e74c682857e757f4f02a0b22e1d3a5102 Mon Sep 17 00:00:00 2001 From: nperez Date: Thu, 19 Aug 2021 15:56:04 +0200 Subject: [PATCH 4/8] Output folder and logs with log4r in prepare_outputs function --- .gitignore | 1 + OperationalCS.R | 53 +++++++++++++------------ modules/data_load/seas5.load.R | 27 +++---------- modules/seasonal_verifications.R | 19 +++++++++ recipes/seasonal_oper.yml | 6 ++- tools/check_conf.R | 16 ++++---- tools/check_recipe.R | 66 ++++++++++---------------------- tools/libs.R | 5 +-- tools/prepare_outputs.R | 33 ++++++++++++++++ 9 files changed, 124 insertions(+), 102 deletions(-) create mode 100644 .gitignore create mode 100644 modules/seasonal_verifications.R create mode 100644 tools/prepare_outputs.R diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..d95b20cb --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +out-logs/ diff --git a/OperationalCS.R b/OperationalCS.R index febab0ce..d7bd1c63 100644 --- a/OperationalCS.R +++ b/OperationalCS.R @@ -6,34 +6,37 @@ args = commandArgs(trailingOnly = TRUE) # This code should process and check the recipe and build the workflow print(args) library(yaml) -conf <- read_yaml(args[1]) # conf <- read_yaml("recipes/conf_user.yml") -recipe <- read_yaml(args[2]) # recipe <- read_yaml("recipes/seasonal_oper.yml") -# recipe <- read_yaml("recipes/seasonal_complex.yml") +library(log4r) + +# To test: +# args <- NULL; args[1] <- "recipes/conf_user.yml"; args[2] <- "recipes/seasonal_oper.yml" +# args <- NULL; args[1] <- "recipes/conf_user.yml"; args[2] <- "recipes/seasonal_complex.yml" + +conf <- read_yaml(args[1]) +recipe <- read_yaml(args[2]) # Load required libraries source("tools/libs.R") -check_conf(conf, file = args[1]) # check_conf(conf, file = "conf_user.yml") -verifications <- check_recipe(recipe, file = args[2], conf) -# verifications <- check_recipe(recipe, file = "recipe.yml", conf) - -for (indep in verifications$independent) { - #if (names(indep) - variable <- indep[[1]]$name - store.freq <- indep[[1]]$freq - fcst.sdate <- recipe$Analysis$Time$sdate - #fcst.freq <- indep$freq - hcst.inityear <- recipe$Analysis$Time$hcst_start - hcst.endyear <- recipe$Analysis$Time$hcst_end - fcst.name <- recipe$Analysis$Datasets$System[[1]]$name - ref.name <- recipe$Analysis$Datasets$Reference[[1]]$name - ltmin <- recipe$Analysis$Time$leadtimemin - ltmax <- recipe$Analysis$Time$leadtimemax - lats.min <- recipe$Analysis$Region$Regional[[1]]$latmin - lats.max <- recipe$Analysis$Region$Regional[[1]]$latmax - lons.min <- recipe$Analysis$Region$Regional[[1]]$lonmin - lons.max <- recipe$Analysis$Region$Regional[[1]]$lonmax - stream <- "fcst" - source("modules/data_load/seas5.load.R") +# Create output folder and log: +prepare_outputs(recipe = recipe, file = args[2], conf = conf) + +# Checks: +check_conf(conf, file = args[1], logger) +verifications <- check_recipe(recipe, file = args[2], conf, logger) + +# Hace falta dividir por horizonte? Espero que no! +# De momento aquí se queda esto: +if (recipe$Analysis$Horizon == 'Seasonal' || + recipe$Analysis$Horizon == 'seasonal') { + source("modules/seasonal_verifications.R") +} else if (recipe$Analysis$Horizon == 'Subseasonal' || + recipe$Analysis$Horizon == 'subseasonal') { + source("modules/subseasonal_verifications.R") +} else if (recipe$Analysis$Horizon == 'Decadal' || + recipe$Analysis$Horizon == 'decadal') { + source("modules/decadal_verifications.R") +} else { + error(logger, "Unknown 'Horizon' defined in the recipe.") } diff --git a/modules/data_load/seas5.load.R b/modules/data_load/seas5.load.R index 86af832d..04ae4cd4 100644 --- a/modules/data_load/seas5.load.R +++ b/modules/data_load/seas5.load.R @@ -1,7 +1,3 @@ - - #------------------------------------------------------------------- - # Init - #------------------------------------------------------------------- fcst.month <- substr(fcst.sdate,5,6) @@ -21,23 +17,12 @@ file_dates <- add_dims(file_dates) file_dates.fcst <- add_dims(file_dates.fcst) -# REPLACED -# accum <- FALSE -# switch (tolower(variable), -# "tas" = {freq.hcst <- 'f6h'; freq.obs <- 'f1h'}, -# "tasmin" = {freq.hcst <- 'f24h'; freq.obs <- 'f24h'}, -# "tasmax" = {freq.hcst <- 'f24h'; freq.obs <- 'f24h'}, -# "psl" = {freq.hcst <- 'f6h'; freq.obs <- 'f1h'}, -# "prlr" = {freq.hcst <- 's0-24h'; freq.obs <- 's0-24h'; accum <- TRUE}, -# "rsds" = {freq.hcst <- 's0-24h'; freq.obs <- 's0-24h'; accum <- TRUE}, -# "sfcwind" = {freq.hcst <- 'f6h'; freq.obs <- 'f1h'} -# ) -#------------- table <- read_yaml(paste0(conf$code_dir, "conf/archive.yml")) dataset_descrip <- table$archive[which(names(table$archive) == fcst.name)][[1]] -freq.hcst <- unlist(dataset_descrip[store_freq][[1]][variable]) -refere_descrip <- table$archive[which(names(table$archive) == ref.name)][[1]] -freq.obs <- unlist(refere_descrip[store_freq][[1]][variable]) +freq.hcst <- unlist(dataset_descrip[store.freq][[1]][variable]) +refere_descrip <- table$archive[which(names(table$archive) == + tolower(ref.name))][[1]] +freq.obs <- unlist(refere_descrip[store.freq][[1]][variable]) obs.dir <- refere_descrip$src fcst.dir <- dataset_descrip$src hcst.dir <- dataset_descrip$src @@ -45,7 +30,7 @@ fcst.nmember <- dataset_descrip$nmember$fcst hcst.nmember <- dataset_descrip$nmember$hcst if ("accum" %in% names(refere_descrip)) { - accum <- unlist(refere_descrip$accum[store_freq][[1]]) + accum <- unlist(refere_descrip$accum[store.freq][[1]]) } else { accum <- FALSE } @@ -162,7 +147,7 @@ if ("accum" %in% names(refere_descrip)) { target_dims=c('time', 'latitude', 'longitude'))[[1]] obs.NA_dates <- dates_file[obs.NA_dates.ind] obs.NA_dates <- obs.NA_dates[order(obs.NA_dates)] - obs.NA_files <- paste0(obs.dir,fcst.freq,"/",variable,"_", + obs.NA_files <- paste0(obs.dir, store.freq,"/",variable,"_", freq.obs,"obs.grid","/",variable,"_",obs.NA_dates,".nc") if (any(is.na(hcst))){ diff --git a/modules/seasonal_verifications.R b/modules/seasonal_verifications.R new file mode 100644 index 00000000..494969ba --- /dev/null +++ b/modules/seasonal_verifications.R @@ -0,0 +1,19 @@ +for (indep in verifications$independent) { + #if (names(indep) + variable <- indep[[1]]$name + store.freq <- indep[[1]]$freq + fcst.sdate <- recipe$Analysis$Time$sdate + hcst.inityear <- recipe$Analysis$Time$hcst_start + hcst.endyear <- recipe$Analysis$Time$hcst_end + fcst.name <- recipe$Analysis$Datasets$System[[1]]$name + ref.name <- recipe$Analysis$Datasets$Reference[[1]]$name + ltmin <- recipe$Analysis$Time$leadtimemin + ltmax <- recipe$Analysis$Time$leadtimemax + lats.min <- recipe$Analysis$Region$Regional[[1]]$latmin + lats.max <- recipe$Analysis$Region$Regional[[1]]$latmax + lons.min <- recipe$Analysis$Region$Regional[[1]]$lonmin + lons.max <- recipe$Analysis$Region$Regional[[1]]$lonmax + stream <- "fcst" + source("modules/data_load/seas5.load.R") +} + diff --git a/recipes/seasonal_oper.yml b/recipes/seasonal_oper.yml index ee9f0114..9775bb39 100644 --- a/recipes/seasonal_oper.yml +++ b/recipes/seasonal_oper.yml @@ -36,5 +36,9 @@ Analysis: Skill: metric: RPSS Indicators: - index: False + index: FALSE Output_format: S2S4E + +Run: + Loglevel: INFO + Terminal: TRUE diff --git a/tools/check_conf.R b/tools/check_conf.R index 38ea5236..70ffa9f0 100644 --- a/tools/check_conf.R +++ b/tools/check_conf.R @@ -1,19 +1,21 @@ -check_conf <- function(conf, file) { +check_conf <- function(conf, file, logger) { # conf: yaml user config already read it # file: name of the file to clarify messages - message(paste("Checking user configuration", file)) + info(logger, paste("Checking user configuration", file)) if (!('output_dir' %in% names(conf))) { - stop(paste("The user configuration file should have element 'output_dir'.")) + error(logger, + paste("The user configuration file should have element 'output_dir'.")) } if (!is.character(conf$output_dir)) { - stop (paste("The element 'output_dir' in", file,"file ", - "should be a character string indicating the path ", - "where to save the outputs.")) + error(logger, + paste("The element 'output_dir' in", file,"file ", + "should be a character string indicating the path ", + "where to save the outputs.")) } - message("User configuration checked succsessfully.") + info(logger, "User configuration checked succsessfully.") } diff --git a/tools/check_recipe.R b/tools/check_recipe.R index ee2faed9..221a0122 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -1,54 +1,52 @@ -check_recipe <- function(recipe, file, conf) { +check_recipe <- function(recipe, file, conf, logger) { # recipe: yaml recipe already read it # file: name of the file to clarify messages # conf: configuration file read to load the tables of definitions # output: errors or the total number of workflow (vars x regions) to compute - message(paste("Checking recipe", file)) + info(logger, paste("Checking recipe", file)) + # create output dirs: if (!any(names(recipe) %in% "Analysis")) { - stop("The recipe should has a element 'Analysis'.") + error(logger, "The recipe should has a element 'Analysis'.") } params <- c('Horizon', 'Time', 'Variables', 'Region', 'Regrid', 'Workflow', 'Datasets') if (!all(params %in% names(recipe$Analysis))) { - stop(paste("The element 'Analysis' in the recipe should contain these", + error(logger, + paste("The element 'Analysis' in the recipe should contain these", "elements:", paste(params, collapse = " "))) } horizons <- c('Subseasonal', 'Seasonal', 'Decadal') if (!any(horizons %in% recipe$Analysis$Horizon)) { - stop("The element 'Horizon' in the recipe should be one of the followings:", + error(logger, + "The element 'Horizon' in the recipe should be one of the followings:", paste(horizons, collapse = " ")) } # more checks # ... # calculate number of workflows to create for each variable and if (length(recipe$Analysis$Horizon) > 1) { - stop("Only 1 Horizon can be specified in the recipe") + error(logger, "Only 1 Horizon can be specified in the recipe") } nvar <- length(recipe$Analysis$Variables) if (nvar > 2) { - stop("Only two type of Variables can be listed: ECVs and Indicators.") + error(logger, + "Only two type of Variables can be listed: ECVs and Indicators.") } # remove NULL or None Indicators or ECVs from the recipe: if (!is.list(recipe$Analysis$Variables$Indicators)) { recipe$Analysis$Variables <- recipe$Analysis$Variables[ -which(names(recipe$Analysis$Variables) == 'Indicators')] - #assign(recipe$Analysis$Variables, recipe$Analysis$Variables[ - # -which(names(recipe$Analysis$Variables) == 'Indicators')], - # envir = parent.frame()) } if (!is.list(recipe$Analysis$Variables$ECVs)) { recipe$Analysis$Variables <- recipe$Analysis$Variables[ -which(names(recipe$Analysis$Variables) == 'ECVs')] - #assign(recipe$Analysis$Variables, recipe$Analysis$Variables[ - # -which(names(recipe$Analysis$Variables) == 'ECVs')], - # envir = parent.frame()) } if (length(recipe$Analysis$Region) > 3) { - stop("The section 'Region' in the recipe should be checked.", + error(logger, "The section 'Region' in the recipe should be checked.", "Global, Countries and Regional elements are expected.") } if (!is.logical(recipe$Analysis$Region$Global)) { - stop("The element 'Global' in Region should be logical.") + error(logger, "The element 'Global' in Region should be logical.") } nregions <- 0 if (recipe$Analysis$Region$Global == TRUE) { @@ -62,8 +60,9 @@ check_recipe <- function(recipe, file, conf) { limits <- c('latmin', 'latmax', 'lonmin', 'lonmax') for (i in 1:length(recipe$Analysis$Region$Regional)) { if (!all(limits %in% names(recipe$Analysis$Region$Regional[[i]]))) { - stop("Each region defined in element 'Regional' should have 4 elements:", - paste(limits, collapse = " ")) + error(logger, + "Each region defined in element 'Regional' should have 4 elements:", + paste(limits, collapse = " ")) } # are numeric? class list mode list } @@ -71,7 +70,7 @@ check_recipe <- function(recipe, file, conf) { # Check workflow: need to define restrictions? # e.g. only one calibration method nverifications <- check_number_of_dependent_verifications(recipe, conf) - message("Recipe checked succsessfully.") + info(logger, "Recipe checked succsessfully.") return(nverifications) } @@ -94,60 +93,37 @@ check_number_of_dependent_verifications <- function(recipe, conf) { } } if (workflow_independent) { -print(recipe$Analysis$Variables$Indicators) independent_verifications <- append(recipe$Analysis$Variables$ECVs, recipe$Analysis$Variables$Indicators) } else { if (is.null(recipe$Analysis$Variables$Indicators) || (length(recipe$Analysis$Variables$Indicators) == 1 && is.null(recipe$Analysis$Variables$ECVs))) { -print(recipe$Analysis$Variables) independent_verifications <- recipe$Analysis$Variables } else { ecvs <- recipe$Analysi$Variables$ECVs inds <- recipe$Analysi$Variables$Indicators ind_table <- read_yaml(paste0(conf$code_dir, "conf/indicators_table.yml")) -# ind_table <- ind_table[unlist(inds)] -# vars_for_ind <- NULL -# freq_for_ind <- NULL -# # look for ecvs required to compute indicators -# for (ind in unlist(inds)) { -# if (is.null(ind_table[ind][[1]])) { -# stop("Check the name of the indicators.") -# } -# vars_for_ind <- c(vars_for_ind, ind_table[ind][[1]]$ECVs) -# freq_for_ind <- c(freq_for_ind, ind_table[ind][[1]]$freq) -# } - -# To review from here # first, loop on ecvs if any and compare to indicators done <- NULL # to gather the indicators reviewed if (!is.null(ecvs)) { for (i in 1:length(ecvs)) { dependent <- ecvs[[i]] -print(i) -print(dependent) for (j in 1:length(inds)) { -print(j) -print(ind_table[inds[[j]]$name][[1]]$ECVs == ecvs[[i]]$name) -print(ind_table[inds[[j]]$name][[1]]$freq == ecvs[[i]]$freq) if (ind_table[inds[[j]]$name][[1]]$ECVs == ecvs[[i]]$name) { if (ind_table[inds[[j]]$name][[1]]$freq == ecvs[[i]]$freq) { # they are dependent -print("Dependent") dependent <- list(dependent, inds[[j]]) done <- append(done, inds[[j]]) } } } if (!is.null(dependent$name)) { -print("Is it empty?") dependent <- NULL independent_verifications <- append(independent_verifications, list(ecvs[[i]])) } else { -print("Dependent") dependent_verifications <- append(dependent_verifications, list(dependent)) } @@ -209,12 +185,12 @@ print("Dependent") } } if (!is.null(independent_verifications)) { - message("The variables for independent verification are ", - paste(independent_verifications, collapse = " ")) + info(logger, "The variables for independent verification are ", + paste(independent_verifications, collapse = " ")) } if (!is.null(dependent_verifications)) { - message("The variables for dependent verification are: ", - paste(dependent_verifications, collapse = " ")) + info(logger, "The variables for dependent verification are: ", + paste(dependent_verifications, collapse = " ")) } # remove unnecessary names in objects to be removed return(list(independent = independent_verifications, diff --git a/tools/libs.R b/tools/libs.R index d94c3862..45584fa2 100644 --- a/tools/libs.R +++ b/tools/libs.R @@ -1,12 +1,11 @@ library(startR) library(ClimProjDiags) +library(multiApply) # library(s2dverification) # library(ncdf4) # library(abind) # library(easyVerification) -# library(multiApply) -# library(startR) # library(easyNCDF) # library(CSTools) # @@ -25,4 +24,4 @@ library(ClimProjDiags) # # To be removed when new package is done by library(CSOperational) source(paste0(conf$code_dir, "tools/check_recipe.R")) source(paste0(conf$code_dir, "tools/check_conf.R")) - + source(paste0(conf$code_dir, "tools/prepare_outputs.R")) diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R new file mode 100644 index 00000000..5255af3a --- /dev/null +++ b/tools/prepare_outputs.R @@ -0,0 +1,33 @@ +prepare_outputs <- function(recipe, file, conf) { +# recipe: the content of the readed recipe +# file: the recipe file name +# conf: the content of the readed config user file + # Create output folders: + folder_name <- paste0(gsub(".yml", "", gsub("/", "_", file)), "_", + gsub(" ", "", gsub(":", "", gsub("-", "", Sys.time())))) + dir.create(file.path(conf$output_dir, folder_name, 'plots'), recursive = TRUE) + dir.create(file.path(conf$output_dir, folder_name, 'outputs')) + dir.create(file.path(conf$output_dir, folder_name, 'logs')) + file.copy(file, file.path(conf$output_dir, folder_name, 'logs')) + logfile <- file.path(conf$output_dir, folder_name, 'logs', 'log.txt') + # Set default behaviour of log output file: + if (is.null(recipe$Run)) { + recipe$Run <- list(Loglevel = 'INFO', Terminal = TRUE) + } + if (is.null(recipe$Run$Loglevel)) { + recipe$Run$Loglevel <- 'INFO' + } + if (!is.logical(recipe$Run$Terminal)) { + recipe$Run$Terminal <- TRUE + } + if (recipe$Run$Terminal) { + logger <- logger(threshold = recipe$Run$Loglevel, + appenders = list(console_appender(layout = default_log_layout()), + file_appender(logfile, append = TRUE, + layout = default_log_layout()))) + } else { + logger <- logger(threshold = recipe$Run$Loglevel, + appenders = list(file_appender(logfile, append = TRUE, + layout = default_log_layout()))) + } +} -- GitLab From b437c0f8026cb9e966c49b99f635bd82411ca288 Mon Sep 17 00:00:00 2001 From: nperez Date: Thu, 19 Aug 2021 17:59:20 +0200 Subject: [PATCH 5/8] tests for independent_verifications functions --- OperationalCS.R | 2 +- modules/seasonal_verifications.R | 1 + recipes/seasonal_complex.yml | 4 + tools/check_recipe.R | 14 +-- tools/prepare_outputs.R | 1 + ...heck_number_of_independent_verifications.R | 92 +++++++++++++------ 6 files changed, 76 insertions(+), 38 deletions(-) diff --git a/OperationalCS.R b/OperationalCS.R index d7bd1c63..797a4259 100644 --- a/OperationalCS.R +++ b/OperationalCS.R @@ -19,7 +19,7 @@ recipe <- read_yaml(args[2]) source("tools/libs.R") # Create output folder and log: -prepare_outputs(recipe = recipe, file = args[2], conf = conf) +logger <- prepare_outputs(recipe = recipe, file = args[2], conf = conf) # Checks: check_conf(conf, file = args[1], logger) diff --git a/modules/seasonal_verifications.R b/modules/seasonal_verifications.R index 494969ba..00c3fc10 100644 --- a/modules/seasonal_verifications.R +++ b/modules/seasonal_verifications.R @@ -15,5 +15,6 @@ for (indep in verifications$independent) { lons.max <- recipe$Analysis$Region$Regional[[1]]$lonmax stream <- "fcst" source("modules/data_load/seas5.load.R") + } diff --git a/recipes/seasonal_complex.yml b/recipes/seasonal_complex.yml index c6ce15ed..ba4b0fa4 100644 --- a/recipes/seasonal_complex.yml +++ b/recipes/seasonal_complex.yml @@ -39,3 +39,7 @@ Analysis: Indicators: index: False Output_format: S2S4E + +Run: + Loglevel: INFO + Terminal: TRUE diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 221a0122..4d3c0bd0 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -109,17 +109,17 @@ check_number_of_dependent_verifications <- function(recipe, conf) { done <- NULL # to gather the indicators reviewed if (!is.null(ecvs)) { for (i in 1:length(ecvs)) { - dependent <- ecvs[[i]] + dependent <- list(ecvs[[i]]) for (j in 1:length(inds)) { if (ind_table[inds[[j]]$name][[1]]$ECVs == ecvs[[i]]$name) { if (ind_table[inds[[j]]$name][[1]]$freq == ecvs[[i]]$freq) { # they are dependent - dependent <- list(dependent, inds[[j]]) + dependent <- append(dependent, inds[[j]]) done <- append(done, inds[[j]]) } } } - if (!is.null(dependent$name)) { + if (length(dependent) == 1) { dependent <- NULL independent_verifications <- append(independent_verifications, list(ecvs[[i]])) @@ -185,12 +185,12 @@ check_number_of_dependent_verifications <- function(recipe, conf) { } } if (!is.null(independent_verifications)) { - info(logger, "The variables for independent verification are ", - paste(independent_verifications, collapse = " ")) + info(logger, paste("The variables for independent verification are ", + paste(independent_verifications, collapse = " "))) } if (!is.null(dependent_verifications)) { - info(logger, "The variables for dependent verification are: ", - paste(dependent_verifications, collapse = " ")) + info(logger, paste("The variables for dependent verification are: ", + paste(dependent_verifications, collapse = " "))) } # remove unnecessary names in objects to be removed return(list(independent = independent_verifications, diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R index 5255af3a..08f0063e 100644 --- a/tools/prepare_outputs.R +++ b/tools/prepare_outputs.R @@ -30,4 +30,5 @@ prepare_outputs <- function(recipe, file, conf) { appenders = list(file_appender(logfile, append = TRUE, layout = default_log_layout()))) } + return(logger) } diff --git a/tools/test_check_number_of_independent_verifications.R b/tools/test_check_number_of_independent_verifications.R index cf5acb0e..846dc5be 100644 --- a/tools/test_check_number_of_independent_verifications.R +++ b/tools/test_check_number_of_independent_verifications.R @@ -10,7 +10,7 @@ test_that("A few combinations", { expect_equal(check_number_of_dependent_verifications(recipe, conf), list(independent = NULL, dependent = list(list(list(name = 'tas', freq = 'daily_mean'), - list(name = 'gdd'))))) + name = 'gdd')))) recipe <- list(Analysis = list(Variables = list(ECVs = list( list(name = 'tas', freq = 'daily_mean')), @@ -20,43 +20,50 @@ test_that("A few combinations", { list(independent = list(list(name = 'tas', freq = 'daily_mean'), name = 'gdd'), dependent = NULL)) -# Cases to review: recipe <- list(Analysis = list(Variables = list(ECVs = list( - list(name = list('tas', freq = 'daily')))), + list(name = 'tas', freq = 'daily_mean'))), Workflow = list(Indicators = FALSE, Calibration = 'SBC'))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = list(list(list(name = list('tas'), freq = 'daily_mean'))), + list(independent = list(list(name = 'tas', freq = 'daily_mean')), dependent = NULL)) recipe <- list(Analysis = - list(Variables = list(ECVs = list(list(name = 'tas'), - list(name = 'tasmax'))), + list(Variables = list(ECVs = list( + list(name = 'tas', freq = 'daily_mean'), + list(name = 'tasmax', freq = 'daily_mean'))), Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = c('tas', 'tasmax'), dependent = NULL)) + list(independent = list(list(name = 'tas', freq = 'daily_mean'), + list(name = 'tasmax', freq = 'daily_mean')), + dependent = NULL)) recipe <- list(Analysis = list(Variables = list(Indicators = list(list(name = 'gdd'))), Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = c('gdd'), dependent = NULL)) + list(independent = list(list(name = 'gdd')), dependent = NULL)) recipe <- list(Analysis = - list(Variables = list(ECVs = list(list(name = 'tas')), + list(Variables = list(ECVs = list( + list(name = 'tas', freq = 'daily_mean')), Indicators = list(list(name = 'gdd'), list(name = 'gst'))), Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = c('tas', 'gdd', 'gst'), dependent = NULL)) + list(independent = list(list(name = 'tas', freq = 'daily_mean'), + list(name = 'gdd'), list(name = 'gst')), dependent = NULL)) recipe <- list(Analysis = - list(Variables = list(ECVs = list(list(name = 'tas')), + list(Variables = list(ECVs = list( + list(name = 'tas', freq = 'daily_mean')), Indicators = list(list(name = 'gdd'), list(name = 'gst'))), Workflow = list(Calibration = 'SBC', Indicators = TRUE))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = NULL, dependent = list(c('tas', 'gdd', 'gst')))) + list(independent = NULL, + dependent = list(list(list(name = 'tas', freq = 'daily_mean'), + name = 'gdd', name = 'gst')))) # Dependent workflow: cal & ind recipe <- list(Analysis = list(Variables = list( @@ -64,35 +71,44 @@ test_that("A few combinations", { list(name = 'gst'))), Workflow = list(Calibration = 'SBC', Indicators = TRUE))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = NULL, dependent = list(c('gdd', 'gst')))) + list(independent = NULL, dependent = list('gdd', 'gst'))) recipe <- list(Analysis = - list(Variables = list(ECVs = list(list(name = 'tasmax')), + list(Variables = list(ECVs = list( + list(name = 'tasmax', freq = 'daily_mean')), Indicators = list(list(name = 'gdd'), list(name = 'gst'))), Workflow = list(Calibration = 'SBC', Indicators = TRUE))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = 'tasmax', dependent = list(c('gdd', 'gst')))) + list(independent = list(list(name = 'tasmax', freq = 'daily_mean')), + dependent = list('gdd', 'gst'))) recipe <- list(Analysis = - list(Variables = list(ECVs = list(list(name = 'tasmax'), - list(name = 'tas')), + list(Variables = list(ECVs = list( + list(name = 'tasmax', freq = 'daily_mean'), + list(name = 'tas', freq = 'daily_mean')), Indicators = list(list(name = 'gdd'), list(name = 'gst'))), Workflow = list(Calibration = 'SBC', Indicators = TRUE))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = 'tasmax', dependent = list(c('tas', 'gdd', 'gst')))) + list(independent = list(list(name = 'tasmax', freq = 'daily_mean')), + dependent = list(list(list(name = 'tas', freq = 'daily_mean'), + name = 'gdd', name = 'gst')))) recipe <- list(Analysis = - list(Variables = list(ECVs = list(list(name = 'tasmax'), - list(name = 'tas')), + list(Variables = list(ECVs = list( + list(name = 'tasmax', freq = 'daily_mean'), + list(name = 'tas', freq = 'daily_mean')), Indicators = list(list(name = 'gdd'), list(name = 'gst'), list(name = 'spr32'))), Workflow = list(Calibration = 'SBC', Indicators = TRUE))) expect_equal(check_number_of_dependent_verifications(recipe, conf), list(independent = NULL, - dependent = list(c('tasmax', 'spr32'), c('tas', 'gdd', 'gst')))) + dependent = list(list(list(name = 'tasmax', freq = 'daily_mean'), + name = 'spr32'), + list(list(name = 'tas', freq = 'daily_mean'), + name = 'gdd', name ='gst')))) # Independent workflow: ind & cal recipe <- list(Analysis = list(Variables = list( @@ -100,30 +116,46 @@ test_that("A few combinations", { list(name = 'gst'))), Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = c('gdd', 'gst'), dependent = NULL)) + list(independent = list(list(name = 'gdd'), list(name = 'gst')), + dependent = NULL)) recipe <- list(Analysis = - list(Variables = list(ECVs = list(list(name ='tasmax')), + list(Variables = list(ECVs = list( + list(name ='tasmax', freq = 'daily_freq')), Indicators = list(list(name = 'gdd'), list(name = 'gst'))), Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = c('tasmax', 'gdd', 'gst'), dependent = NULL)) + list(independent = list(list(name = 'tasmax', freq = 'daily_freq'), + list(name = 'gdd'), list(name = 'gst')), + dependent = NULL)) recipe <- list(Analysis = - list(Variables = list(ECVs = list(list(name = 'tasmax'), list(name = 'tas')), - Indicators = list(list(name = 'gdd'), list(name = 'gst'))), + list(Variables = list(ECVs = list( + list(name = 'tasmax', freq = 'daily_mean'), + list(name = 'tas', freq = 'daily_mean')), + Indicators = list(list(name = 'gdd'), + list(name = 'gst'))), Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = c('tasmax', 'tas', 'gdd', 'gst'), dependent = NULL)) + list(independent = list(list(name = 'tasmax', freq = 'daily_mean'), + list(name = 'tas', freq = 'daily_mean'), + list(name = 'gdd'), + list(name = 'gst')), dependent = NULL)) recipe <- list(Analysis = - list(Variables = list(ECVs = list(list(name = 'tasmax'), list(name = 'tas')), + list(Variables = list(ECVs = list( + list(name = 'tasmax', freq = 'daily_mean'), + list(name = 'tas', freq = 'daily_mean')), Indicators = list(list(name = 'gdd'), - list(name = 'gst'), list(name = 'spr32'))), + list(name = 'gst'), + list(name = 'spr32'))), Workflow = list(Indicators = TRUE, Calibration = 'SBC'))) expect_equal(check_number_of_dependent_verifications(recipe, conf), - list(independent = c('tasmax', 'spr32', 'tas', 'gdd', 'gst'), + list(independent = list(list(name = 'tasmax', freq = 'daily_mean'), + list(name = 'tas', freq = 'daily_mean'), + list(name = 'gdd'), list(name = 'gst'), + list(name = 'spr32')), dependent = NULL)) }) -- GitLab From 9084b29272bf5424a7eb670aff711b8903e54b32 Mon Sep 17 00:00:00 2001 From: nperez Date: Fri, 20 Aug 2021 18:52:59 +0200 Subject: [PATCH 6/8] Check Time and return fcst_sdates in check_recipe --- OperationalCS.R | 1 - conf/archive.yml | 4 ++ modules/data_load/.seas5.load.R.swp | Bin 0 -> 16384 bytes modules/data_load/seas5.load.R | 87 ++++++++++++++-------------- modules/seasonal_verifications.R | 23 +++++--- recipes/seasonal_complex.yml | 7 ++- recipes/seasonal_oper.yml | 5 +- tools/add_dims.R | 9 +++ tools/check_recipe.R | 41 ++++++++++++- tools/libs.R | 3 +- 10 files changed, 123 insertions(+), 57 deletions(-) create mode 100644 modules/data_load/.seas5.load.R.swp create mode 100644 tools/add_dims.R diff --git a/OperationalCS.R b/OperationalCS.R index 797a4259..d9897cb6 100644 --- a/OperationalCS.R +++ b/OperationalCS.R @@ -6,7 +6,6 @@ args = commandArgs(trailingOnly = TRUE) # This code should process and check the recipe and build the workflow print(args) library(yaml) -library(log4r) # To test: # args <- NULL; args[1] <- "recipes/conf_user.yml"; args[2] <- "recipes/seasonal_oper.yml" diff --git a/conf/archive.yml b/conf/archive.yml index d4f1f4af..ecfccea9 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -21,6 +21,10 @@ archive: daily_mean: {"tas":"_f1h-r1440x721cds/","rsds":"_f1h-r1440x721cds/", "prlr":"_f1h-r1440x721cds/","sfcWind":"_f1h-r1440x721cds/"} monthly_mean: {"tas":"_f1h-r1440x721cds/"} + lat_decreasing_sort: TRUE + lon_circular_sort: + ini: 0 + end: 361 era5land: src: "recon/ecmwf/era5land/" daily_mean: {"tas":"_f1h/","rsds":"_f1h/", diff --git a/modules/data_load/.seas5.load.R.swp b/modules/data_load/.seas5.load.R.swp new file mode 100644 index 0000000000000000000000000000000000000000..c6cf430de60965198f0f5206d3823cd29e6bbfb5 GIT binary patch literal 16384 zcmeHNONbmr7%q*k#6-mx9>i2;C(Lx{%p-{kA&cZGib*7|%VZsTyKA$JJ>8S8n#7F+ z6+MWcm)t}|@dfdqMnNwk2zpQu$ted98VLw0g5V_(#v@_s;6s*9@@?CuWVvCZei@ZFIPLY7j~{Yb)2!VEn;zk8pa>f(iKjuW;brQf2?3t zNGEn@{bQmk-Plp?Y&CIFJ>sjX(^1iSxj|N2k=F@?ta^@ennC1vRVf^KZ-u^A_Ko#m z8Vv)6fzb@CVjDNqYt%kbSi^6-_26hx(_$Df3>XFs1BL;^fMLKeU>GnA{O=f0?PcsK z^y=!gXN~m#Rek?2r^li6`r*FzKhySm((7mX+S60>&oE#ZFbo(53GHFkl!k z3>XFs1BQWrfdR*3Y#scbp@JX#|K0lk(#?!r1kM8Q0;hoxuz^cAG4>;H9{3b^33w6M z3tV2s*k{1gz=ObY;OCW$eFdBbP6Cetdx5pUFE_##a2EI$_#Ai@I0|e7wgPtpzpj8U z;2q#?;2B^aPyki}Uo2*b8yNclph8 zI0@_oZUoL<%h)HtN5F@`DL?_cfi=K|YtSd41#ANTSO!_(N#J4NPT&VDtegf;08_wW zpalE|zCQ$r_Ya8s#UGrjc&$mjwK?PCT?xKrJ@2{Dh+5dXo%Ui`gk=hk?LT?wvWP#}3BNmVs#{{3`2Pl-DL$(87HCvrP z`BDVJRl;j}eoK<@zJojMe)%5d-G=-#bh1nr!zs$^_aw44vpN=Rqpq_ znAC%rrDPw(v!sK&?7OVc&=|-LULeVW^&}+lH?jVoA7q9fz;jLtJW3^!%JHtCZJs|0mC!8=Jf~-3akN6SkzFW) zuwt{AF1gLv3^b6FfJp7(E#r*X^`OVb|72_U~jTxd^1- zGzznpPN<)>v6#UtzD-ZyeO$j?)Xk0g`!v;P)Ti*Pk%o6cHx~F>KU~X02P3fDt8H0u z!8Bv(v`ZmiqIaZFnTxckNODZy`9%nmTw)nq4YMj^H!(K4owJl{((jGDsMTp)Lxh@# zvNOO-1H)fGvi0xkp7A+w(w!XO7(cID!m<)>S#KodT+y$9V)~+VDr@-W(HGAPh_HSJ|( zt`!t4T`**mNit27DaT7HfpT9PO-=KcMo11;A@lbJ`XdhAuT~V|=N6ZujmJkp&xF11 zbkAtMSCOAoOn&IAIlK#RZO&g6p{Fm>+&FJRKZYpk#Ef=HVzz$fR0h10GV-88GNdLu z({NJO*Cg(vu2s(APoUgJ}wUhArmy+&d*YlA@?DD)e;*w0@l z4|l>3MH`P6N)2P^2D}PvPXdpopc=q^}nP5Cn&;+r0&A1J?b){bX{g+Js3n_ zJs6_cIniq_bsJkDUL3V2OWlpG5|y$OBRWV;)Q8M6oh;2VdYWnD`x2Dle8`cMo;I)J zuV$fePq!8I5YROzZToaW|7b^)rPFQ)b7ehbdv=D literal 0 HcmV?d00001 diff --git a/modules/data_load/seas5.load.R b/modules/data_load/seas5.load.R index 04ae4cd4..fa6b429f 100644 --- a/modules/data_load/seas5.load.R +++ b/modules/data_load/seas5.load.R @@ -7,12 +7,6 @@ file_dates <- paste(strtoi(hcst.inityear):strtoi(hcst.endyear), fcst.month, sep = "") - add_dims <- function(data){ - default_dims <- c(sweek = 1, sday = 1, syear = length(data)) - default_dims[names(dim(data))] <- dim(data) - dim(data) <- default_dims - return(data) - } file_dates <- add_dims(file_dates) file_dates.fcst <- add_dims(file_dates.fcst) @@ -73,23 +67,23 @@ if ("accum" %in% names(refere_descrip)) { retrieve = T) } - hcst<-Start(dat = hcst.path, - var = variable, - file_date = file_dates, - time = indices(ltmin:ltmax), - latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lons.min, lons.max)), - longitude_reorder = CircularSort(0, 361), - synonims = list(latitude = c('lat','latitude'), - longitude = c('lon','longitude'), - member = c('ensemble')), - member = indices(1:hcst.nmember), - return_vars = list(latitude = 'dat', - longitude = 'dat', - time = 'file_date'), - split_multiselected_dims = TRUE, - retrieve = T) + hcst <-Start(dat = hcst.path, + var = variable, + file_date = file_dates, + time = indices(ltmin:ltmax), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = CircularSort(0, 361), + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude'), + member = c('ensemble')), + member = indices(1:hcst.nmember), + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'file_date'), + split_multiselected_dims = TRUE, + retrieve = T) hcst.NA_files <- c(attributes(hcst)$NotFoundFiles) hcst.NA_files <- hcst.NA_files[!is.na(hcst.NA_files)] @@ -151,34 +145,39 @@ if ("accum" %in% names(refere_descrip)) { freq.obs,"obs.grid","/",variable,"_",obs.NA_dates,".nc") if (any(is.na(hcst))){ - print(" ERROR: MISSING HCST VALUES FOUND DURING LOADING # " ) - print(" ################################################# " ) - print(" ###### MISSING FILES #### " ) - print(" ################################################# " ) - print("hcst files:") - print(hcst.NA_files) - print(" ################################################# " ) - print(" ################################################# " ) - quit(status=1) + fatal(logger, + paste(" ERROR: MISSING HCST VALUES FOUND DURING LOADING # ", + " ################################################# ", + " ###### MISSING FILES #### ", + " ################################################# ", + "hcst files:", + hcst.NA_files, + " ################################################# ", + " ################################################# ", + sep="\n")) + quit(status = 1) } if (any(is.na(obs)) && !identical(obs.NA_dates,character(0))){ - print(" ERROR: MISSING OBS VALUES FOUND DURING LOADING # " ) - print(" ################################################# " ) - print(" ###### MISSING FILES #### " ) - print(" ################################################# " ) - print("obs files:") - print(obs.NA_files) - print(" ################################################# " ) - print(" ################################################# " ) + fatal(logger, + paste(" ERROR: MISSING OBS VALUES FOUND DURING LOADING # ", + " ################################################# ", + " ###### MISSING FILES #### ", + " ################################################# ", + "obs files:", + obs.NA_files, + " ################################################# ", + " ################################################# ", + sep="\n")) quit(status=1) } - print("######### DATA LOADING COMPLETED SUCCESFULLY ##############") + info(logger, + "######### DATA LOADING COMPLETED SUCCESFULLY ##############") - default_dims <- c(dat=1, var=1, sweek = 1, - sday = 1, syear = 1, time=1, - latitude=1, longitude = 1, member = 1) + default_dims <- c(dat = 1, var = 1, sweek = 1, + sday = 1, syear = 1, time = 1, + latitude = 1, longitude = 1, member = 1) default_dims[names(dim(obs))] <- dim(obs) dim(obs) <- default_dims diff --git a/modules/seasonal_verifications.R b/modules/seasonal_verifications.R index 00c3fc10..d87f30af 100644 --- a/modules/seasonal_verifications.R +++ b/modules/seasonal_verifications.R @@ -1,20 +1,29 @@ for (indep in verifications$independent) { - #if (names(indep) variable <- indep[[1]]$name store.freq <- indep[[1]]$freq - fcst.sdate <- recipe$Analysis$Time$sdate hcst.inityear <- recipe$Analysis$Time$hcst_start hcst.endyear <- recipe$Analysis$Time$hcst_end - fcst.name <- recipe$Analysis$Datasets$System[[1]]$name - ref.name <- recipe$Analysis$Datasets$Reference[[1]]$name ltmin <- recipe$Analysis$Time$leadtimemin ltmax <- recipe$Analysis$Time$leadtimemax lats.min <- recipe$Analysis$Region$Regional[[1]]$latmin lats.max <- recipe$Analysis$Region$Regional[[1]]$latmax lons.min <- recipe$Analysis$Region$Regional[[1]]$lonmin lons.max <- recipe$Analysis$Region$Regional[[1]]$lonmax - stream <- "fcst" - source("modules/data_load/seas5.load.R") - + stream <- verifications$stream + # Is multimodel FALSE + if (recipe$Analysis$Datasets$Multimodel) { + # is there a multimodel load step in S2S4E backend? + } else { + for (sys in 1:length(recipe$Analysis$Datasets$System)) { + fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name + for (ref in 1:length(recipe$Analysis$Datasets$Reference)) { + ref.name <- recipe$Analysis$Datasets$Reference[[ref]]$name + for (sdate in verifications$fcst.sdate) { + fcst.sdate <- sdate + source("modules/data_load/seas5.load.R") + } + } + } + } } diff --git a/recipes/seasonal_complex.yml b/recipes/seasonal_complex.yml index ba4b0fa4..8b633cb0 100644 --- a/recipes/seasonal_complex.yml +++ b/recipes/seasonal_complex.yml @@ -17,7 +17,10 @@ Analysis: Reference: - {name: ERA5} Time: - sdates: 20210701 + sdate: + fcst_year: 2021 + fcst_month: [07, 08] + fcst_day: 01 hcst_start: 2000 hcst_end: 2002 leadtimemin: 2 @@ -42,4 +45,4 @@ Analysis: Run: Loglevel: INFO - Terminal: TRUE + Terminal: True diff --git a/recipes/seasonal_oper.yml b/recipes/seasonal_oper.yml index 9775bb39..d58c3237 100644 --- a/recipes/seasonal_oper.yml +++ b/recipes/seasonal_oper.yml @@ -16,7 +16,10 @@ Analysis: Reference: - {name: era5} Time: - sdates: 20210701 + sdate: + fcst_year: 2021 + fcst_month: 07 + fcst_day: 01 hcst_start: 2000 hcst_end: 2002 leadtimemin: 2 diff --git a/tools/add_dims.R b/tools/add_dims.R new file mode 100644 index 00000000..aa770eb8 --- /dev/null +++ b/tools/add_dims.R @@ -0,0 +1,9 @@ +# Function created in S2S$E-backend. Copied from: +# https://earth.bsc.es/gitlab/es/S2S4E-backend-BSC/-/blob/master/data-analysis/SEAS5/seas5.load.R +add_dims <- function(data){ + default_dims <- c(sweek = 1, sday = 1, syear = length(data)) + default_dims[names(dim(data))] <- dim(data) + dim(data) <- default_dims + return(data) + } + diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 4d3c0bd0..4a1e5612 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -21,6 +21,44 @@ check_recipe <- function(recipe, file, conf, logger) { "The element 'Horizon' in the recipe should be one of the followings:", paste(horizons, collapse = " ")) } + # Check temporal settings and + # count the number of verifications + time_settings <- c('sdate', 'leadtimemin', 'leadtimemax', 'hcst_start', 'hcst_end') + if (!all(time_settings %in% names(recipe$Analysis$Time))) { + error(logger, + paste("The element 'Time' in the recipe should contain these elements:", + paste(time_settings, collapse = " "))) + } + if (is.null(recipe$Analysis$Time$sdate$fcst_year) || + recipe$Analysis$Time$sdate$fcst_year == 'None') { + stream <- "hindcast" + recipe$Analysis$Time$sdate$fcst_year <- 'YYYY' + } else { + stream <- "fcst" + } + if (length(recipe$Analysis$Time$sdate$fcst_day) > 1 && + tolower(recipe$Analysis$Horizon) != "subseasonal") { + warn(logger, + paste("Only subseasonal verification allows multiple forecast days."), + "Element fcst_day in recipe set as 1.") + recipe$Analysis$Time$sdate$fcst_day <- '01' + } + if (is.null(recipe$Analysis$Time$sdate$fcst_month)) { + error(logger, + paste("The element 'fcst_month' in the recipe should be defined.")) + } + fcst.sdate <- NULL + for (year in recipe$Analysis$Time$sdate$fcst_year) { + for (month in recipe$Analysis$Time$sdate$fcst_month) { + for (day in recipe$Analysis$Time$sdate$fcst_day) { + fcst.sdate <- c(fcst.sdate, + paste0(year, + sprintf("%02d", as.numeric(month)), + sprintf("%02d", as.numeric(day)))) + } + } + } + fcst.sdate <- list(stream = stream, fcst.sdate = fcst.sdate) # more checks # ... # calculate number of workflows to create for each variable and @@ -70,8 +108,9 @@ check_recipe <- function(recipe, file, conf, logger) { # Check workflow: need to define restrictions? # e.g. only one calibration method nverifications <- check_number_of_dependent_verifications(recipe, conf) + info(logger, paste("Start Dates", paste(fcst.sdate, collapse = " "))) info(logger, "Recipe checked succsessfully.") - return(nverifications) + return(append(nverifications, fcst.sdate)) } check_number_of_dependent_verifications <- function(recipe, conf) { diff --git a/tools/libs.R b/tools/libs.R index 45584fa2..8d9856d7 100644 --- a/tools/libs.R +++ b/tools/libs.R @@ -1,4 +1,4 @@ - +library(log4r) library(startR) library(ClimProjDiags) library(multiApply) @@ -25,3 +25,4 @@ library(multiApply) source(paste0(conf$code_dir, "tools/check_recipe.R")) source(paste0(conf$code_dir, "tools/check_conf.R")) source(paste0(conf$code_dir, "tools/prepare_outputs.R")) + source(paste0(conf$code_dir, "tools/add_dims.R")) # Not sure if necessary yet -- GitLab From a9b1878cab31dbfecd78bdb1677488b21b9802b7 Mon Sep 17 00:00:00 2001 From: nperez Date: Mon, 23 Aug 2021 10:05:46 +0200 Subject: [PATCH 7/8] Independent verification ind vs ecv --- OperationalCS.R | 9 +++------ modules/data_load/.seas5.load.R.swp | Bin 16384 -> 0 bytes modules/seasonal_verifications.R | 23 +++++++++++++++++++++-- 3 files changed, 24 insertions(+), 8 deletions(-) delete mode 100644 modules/data_load/.seas5.load.R.swp diff --git a/OperationalCS.R b/OperationalCS.R index d9897cb6..6b66120a 100644 --- a/OperationalCS.R +++ b/OperationalCS.R @@ -26,14 +26,11 @@ verifications <- check_recipe(recipe, file = args[2], conf, logger) # Hace falta dividir por horizonte? Espero que no! # De momento aquí se queda esto: -if (recipe$Analysis$Horizon == 'Seasonal' || - recipe$Analysis$Horizon == 'seasonal') { +if (tolower(recipe$Analysis$Horizon) == 'seasonal') { source("modules/seasonal_verifications.R") -} else if (recipe$Analysis$Horizon == 'Subseasonal' || - recipe$Analysis$Horizon == 'subseasonal') { +} else if (tolower(recipe$Analysis$Horizon == 'subseasonal') { source("modules/subseasonal_verifications.R") -} else if (recipe$Analysis$Horizon == 'Decadal' || - recipe$Analysis$Horizon == 'decadal') { +} else if (tolower(recipe$Analysis$Horizon == 'decadal') { source("modules/decadal_verifications.R") } else { error(logger, "Unknown 'Horizon' defined in the recipe.") diff --git a/modules/data_load/.seas5.load.R.swp b/modules/data_load/.seas5.load.R.swp deleted file mode 100644 index c6cf430de60965198f0f5206d3823cd29e6bbfb5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHNONbmr7%q*k#6-mx9>i2;C(Lx{%p-{kA&cZGib*7|%VZsTyKA$JJ>8S8n#7F+ z6+MWcm)t}|@dfdqMnNwk2zpQu$ted98VLw0g5V_(#v@_s;6s*9@@?CuWVvCZei@ZFIPLY7j~{Yb)2!VEn;zk8pa>f(iKjuW;brQf2?3t zNGEn@{bQmk-Plp?Y&CIFJ>sjX(^1iSxj|N2k=F@?ta^@ennC1vRVf^KZ-u^A_Ko#m z8Vv)6fzb@CVjDNqYt%kbSi^6-_26hx(_$Df3>XFs1BL;^fMLKeU>GnA{O=f0?PcsK z^y=!gXN~m#Rek?2r^li6`r*FzKhySm((7mX+S60>&oE#ZFbo(53GHFkl!k z3>XFs1BQWrfdR*3Y#scbp@JX#|K0lk(#?!r1kM8Q0;hoxuz^cAG4>;H9{3b^33w6M z3tV2s*k{1gz=ObY;OCW$eFdBbP6Cetdx5pUFE_##a2EI$_#Ai@I0|e7wgPtpzpj8U z;2q#?;2B^aPyki}Uo2*b8yNclph8 zI0@_oZUoL<%h)HtN5F@`DL?_cfi=K|YtSd41#ANTSO!_(N#J4NPT&VDtegf;08_wW zpalE|zCQ$r_Ya8s#UGrjc&$mjwK?PCT?xKrJ@2{Dh+5dXo%Ui`gk=hk?LT?wvWP#}3BNmVs#{{3`2Pl-DL$(87HCvrP z`BDVJRl;j}eoK<@zJojMe)%5d-G=-#bh1nr!zs$^_aw44vpN=Rqpq_ znAC%rrDPw(v!sK&?7OVc&=|-LULeVW^&}+lH?jVoA7q9fz;jLtJW3^!%JHtCZJs|0mC!8=Jf~-3akN6SkzFW) zuwt{AF1gLv3^b6FfJp7(E#r*X^`OVb|72_U~jTxd^1- zGzznpPN<)>v6#UtzD-ZyeO$j?)Xk0g`!v;P)Ti*Pk%o6cHx~F>KU~X02P3fDt8H0u z!8Bv(v`ZmiqIaZFnTxckNODZy`9%nmTw)nq4YMj^H!(K4owJl{((jGDsMTp)Lxh@# zvNOO-1H)fGvi0xkp7A+w(w!XO7(cID!m<)>S#KodT+y$9V)~+VDr@-W(HGAPh_HSJ|( zt`!t4T`**mNit27DaT7HfpT9PO-=KcMo11;A@lbJ`XdhAuT~V|=N6ZujmJkp&xF11 zbkAtMSCOAoOn&IAIlK#RZO&g6p{Fm>+&FJRKZYpk#Ef=HVzz$fR0h10GV-88GNdLu z({NJO*Cg(vu2s(APoUgJ}wUhArmy+&d*YlA@?DD)e;*w0@l z4|l>3MH`P6N)2P^2D}PvPXdpopc=q^}nP5Cn&;+r0&A1J?b){bX{g+Js3n_ zJs6_cIniq_bsJkDUL3V2OWlpG5|y$OBRWV;)Q8M6oh;2VdYWnD`x2Dle8`cMo;I)J zuV$fePq!8I5YROzZToaW|7b^)rPFQ)b7ehbdv=D diff --git a/modules/seasonal_verifications.R b/modules/seasonal_verifications.R index d87f30af..8f713a98 100644 --- a/modules/seasonal_verifications.R +++ b/modules/seasonal_verifications.R @@ -1,6 +1,25 @@ for (indep in verifications$independent) { - variable <- indep[[1]]$name - store.freq <- indep[[1]]$freq + if (length(indep[[1]]) == 1) { + info(logger, + paste(" #*****************************************#", + " # Starting Independent verification of Indicator ", + " indep[[1]]$name", + sep = "\n")) + ind_table <- read_yaml(paste0(conf$code_dir, + "conf/indicators_table.yml")) + variable <- ind_table[indep[[1]]][[1]]$ECVs + store.freq <- ind_table[indep[[1]]][[1]]$freq + ind.fun <- ind_table[indep[[1]]][[1]]$function + } else { + info(logger, + paste(" #*****************************************#", + " # Starting Independent verification of ECV ", + " indep[[1]]$name, indep[[1]]$freq", + sep = "\n")) + variable <- indep[[1]]$name + store.freq <- indep[[1]]$freq + ind.fun <- NULL + } hcst.inityear <- recipe$Analysis$Time$hcst_start hcst.endyear <- recipe$Analysis$Time$hcst_end ltmin <- recipe$Analysis$Time$leadtimemin -- GitLab From e2f551fe1087d0705e9acf5d57fb38cad1ab1d84 Mon Sep 17 00:00:00 2001 From: nperez Date: Mon, 23 Aug 2021 16:16:31 +0200 Subject: [PATCH 8/8] Transformation get from recipe --- OperationalCS.R | 13 +- conf/archive.yml | 4 + conf/indicators_table.yml | 6 +- modules/data_load/seas5.load.R | 138 ++++++++++++------ ...asonal_verifications.R => verifications.R} | 9 +- recipes/seasonal_complex.yml | 4 +- recipes/seasonal_oper.yml | 4 +- tools/check_recipe.R | 12 +- 8 files changed, 117 insertions(+), 73 deletions(-) rename modules/{seasonal_verifications.R => verifications.R} (91%) diff --git a/OperationalCS.R b/OperationalCS.R index 6b66120a..1d79b4ff 100644 --- a/OperationalCS.R +++ b/OperationalCS.R @@ -24,15 +24,6 @@ logger <- prepare_outputs(recipe = recipe, file = args[2], conf = conf) check_conf(conf, file = args[1], logger) verifications <- check_recipe(recipe, file = args[2], conf, logger) -# Hace falta dividir por horizonte? Espero que no! -# De momento aquí se queda esto: -if (tolower(recipe$Analysis$Horizon) == 'seasonal') { - source("modules/seasonal_verifications.R") -} else if (tolower(recipe$Analysis$Horizon == 'subseasonal') { - source("modules/subseasonal_verifications.R") -} else if (tolower(recipe$Analysis$Horizon == 'decadal') { - source("modules/decadal_verifications.R") -} else { - error(logger, "Unknown 'Horizon' defined in the recipe.") -} +# Go to verification code: +source("modules/verifications.R") diff --git a/conf/archive.yml b/conf/archive.yml index ecfccea9..f55a2e90 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -15,6 +15,7 @@ archive: nmember: fcst: 51 hcst: 25 + regrid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" era5: src: "recon/ecmwf/era5/" @@ -25,12 +26,15 @@ archive: lon_circular_sort: ini: 0 end: 361 + regrid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" era5land: src: "recon/ecmwf/era5land/" daily_mean: {"tas":"_f1h/","rsds":"_f1h/", "prlr":"_f1h/","sfcWind":"_f1h/"} + regrid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" uerra: src: "recon/ecmwf/uerra_mescan/" daily_mean: {"tas":"_f6h/"} + grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" diff --git a/conf/indicators_table.yml b/conf/indicators_table.yml index f4debcc8..8e8ce0fd 100644 --- a/conf/indicators_table.yml +++ b/conf/indicators_table.yml @@ -3,17 +3,17 @@ gdd: longname: 'Growing Degree Days' ECVs: tas freq: daily_mean - function: AccumulationExceedingThreshold + fun: AccumulationExceedingThreshold gst: longname: 'Growing Season Temperature' ECVs: tas freq: daily_mean - function: PeriodMean + fun: PeriodMean spr32: longname: 'Spring Heat Stress Days - 32°C' ECVs: tasmax freq: daily_mean - function: TotalTimeExceedingThreshold + fun: TotalTimeExceedingThreshold diff --git a/modules/data_load/seas5.load.R b/modules/data_load/seas5.load.R index fa6b429f..4ebd922b 100644 --- a/modules/data_load/seas5.load.R +++ b/modules/data_load/seas5.load.R @@ -10,21 +10,21 @@ file_dates <- add_dims(file_dates) file_dates.fcst <- add_dims(file_dates.fcst) - +# Take parameters from conf/archive for datasets: table <- read_yaml(paste0(conf$code_dir, "conf/archive.yml")) dataset_descrip <- table$archive[which(names(table$archive) == fcst.name)][[1]] freq.hcst <- unlist(dataset_descrip[store.freq][[1]][variable]) -refere_descrip <- table$archive[which(names(table$archive) == +reference_descrip <- table$archive[which(names(table$archive) == tolower(ref.name))][[1]] -freq.obs <- unlist(refere_descrip[store.freq][[1]][variable]) -obs.dir <- refere_descrip$src +freq.obs <- unlist(reference_descrip[store.freq][[1]][variable]) +obs.dir <- reference_descrip$src fcst.dir <- dataset_descrip$src hcst.dir <- dataset_descrip$src fcst.nmember <- dataset_descrip$nmember$fcst hcst.nmember <- dataset_descrip$nmember$hcst -if ("accum" %in% names(refere_descrip)) { - accum <- unlist(refere_descrip$accum[store.freq][[1]]) +if ("accum" %in% names(reference_descrip)) { + accum <- unlist(reference_descrip$accum[store.freq][[1]]) } else { accum <- FALSE } @@ -42,7 +42,32 @@ if ("accum" %in% names(refere_descrip)) { fcst.dir, store.freq, "/$var$", freq.hcst,"$var$_$file_date$01", ".nc") - #------------------------------------------------------------------- +#------------------------------------------------------------------- +# Regrid: +if (tolower(recipe$Analysis$Regrid$type) == 'reference') { + fcst.gridtype <- reference_descrip$regrid + fcst.gridmethod <- recipe$Analysis$Regrid$method + fcst.tranform <- CDORemapper + obs.gridtype <- NULL + obs.gridmethod <- NULL + obs.tranform <- NULL +} else if (tolower(recipe$Analysis$Regrid$type) == 'system') { + fcst.gridtype <- NULL + fcst.gridmethod <- NULL + fcst.transform <- NULL + obs.gridtype <- dataset_descrip$regrid + obs.gridmethod <- recipe$Analysis$Regrid$method + obs.transform <- CDORemapper +} else { + fcst.gridtype <- recipe$Analysis$Regrid$type + fcst.gridmethod <- recipe$Analysis$Regrid$method + fcst.transform <- CDORemapper + obs.gridtype <- recipe$Analysis$Regrid$type + obs.gridmethod <- recipe$Analysis$Regrid$method + obs.transform <- CDORemapper +} + + # Timeseries load #------------------------------------------------------------------- @@ -53,37 +78,55 @@ if ("accum" %in% names(refere_descrip)) { file_date = file_dates.fcst, time = indices(ltmin:ltmax), latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(decreasing = TRUE), + latitude_reorder = Sort(decreasing = + dataset_descrip$lat_decreasing_sort), longitude = values(list(lons.min, lons.max)), - longitude_reorder = CircularSort(0, 361), - synonims = list(latitude=c('lat','latitude'), - longitude=c('lon','longitude'), - member=c('ensemble')), + longitude_reorder = CircularSort( + dataset_descrip$lon_circular_sort$ini, + dataset_descrip$lon_circular_sort$end), + transform = fcst.transform, + transform_params = list(grid = fcst.gridtype, + method = fcst.gridmethod, + crop = c(lons.min, lons.max, + lats.min, lats.max)), + transform_vars = c('latitude', 'longitude'), + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude'), + member = c('ensemble')), member = indices(1:fcst.nmember), return_vars = list(latitude = 'dat', longitude = 'dat', time = 'file_date'), split_multiselected_dims = TRUE, - retrieve = T) + retrieve = TRUE) } - hcst <-Start(dat = hcst.path, - var = variable, - file_date = file_dates, - time = indices(ltmin:ltmax), - latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lons.min, lons.max)), - longitude_reorder = CircularSort(0, 361), - synonims = list(latitude = c('lat','latitude'), - longitude = c('lon','longitude'), - member = c('ensemble')), - member = indices(1:hcst.nmember), - return_vars = list(latitude = 'dat', - longitude = 'dat', - time = 'file_date'), - split_multiselected_dims = TRUE, - retrieve = T) + hcst <- Start(dat = hcst.path, + var = variable, + file_date = file_dates, + time = indices(ltmin:ltmax), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = + dataset_descrip$lat_decreasing_sort), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = CircularSort( + dataset_descrip$lon_circular_sort$ini, + dataset_descrip$lon_circular_sort$end), + transform = fcst.transform, + transform_params = list(grid = fcst.gridtype, + method = fcst.gridmethod, + crop = c(lons.min, lons.max, + lats.min, lats.max)), + transform_vars = c('latitude', 'longitude'), + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude'), + member = c('ensemble')), + member = indices(1:hcst.nmember), + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'file_date'), + split_multiselected_dims = TRUE, + retrieve = TRUE) hcst.NA_files <- c(attributes(hcst)$NotFoundFiles) hcst.NA_files <- hcst.NA_files[!is.na(hcst.NA_files)] @@ -99,28 +142,27 @@ if ("accum" %in% names(refere_descrip)) { obs <- Start(dat = obs.path, var = variable, - file_date = dates_file, + file_date = dates_file, latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(decreasing = TRUE), + latitude_reorder = Sort(decreasing = + reference_descrip$lat_decreasing_sort), longitude = values(list(lons.min, lons.max)), - longitude_reorder = CircularSort(0, 361), - synonims = list(latitude=c('lat','latitude'), - longitude=c('lon','longitude')), - transform = CDORemapper, - transform_params = list( - grid = paste0( - "/esarchive/exp/ecmwf/system5c3s/", store.freq, - "/",variable, freq.hcst, - "/",variable,"_","201805","01.nc" - ), - method = 'con', - crop = c(lons.min,lons.max,lats.min,lats.max)), - transform_vars = c('latitude', 'longitude'), - return_vars = list(latitude = 'dat', + longitude_reorder = CircularSort( + reference_descrip$lon_circular_sort$ini, + reference_descrip$lon_circular_sort$end), + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude')), + transform = obs.transform, + transform_params = list(grid = obs.gridtype, + method = obs.gridmethod, + crop = c(lons.min, lons.max, + lats.min, lats.max)), + transform_vars = c('latitude', 'longitude'), + return_vars = list(latitude = 'dat', longitude = 'dat', time = 'file_date'), - split_multiselected_dims = TRUE, - retrieve = TRUE) + split_multiselected_dims = TRUE, + retrieve = TRUE) dates_file <- paste0(dates_file, '01') dim(dates_file) <- dim(Subset(hcst, diff --git a/modules/seasonal_verifications.R b/modules/verifications.R similarity index 91% rename from modules/seasonal_verifications.R rename to modules/verifications.R index 8f713a98..2749201a 100644 --- a/modules/seasonal_verifications.R +++ b/modules/verifications.R @@ -1,5 +1,6 @@ for (indep in verifications$independent) { - if (length(indep[[1]]) == 1) { +print(indep) + if (length(indep) == 1) { info(logger, paste(" #*****************************************#", " # Starting Independent verification of Indicator ", @@ -9,15 +10,15 @@ for (indep in verifications$independent) { "conf/indicators_table.yml")) variable <- ind_table[indep[[1]]][[1]]$ECVs store.freq <- ind_table[indep[[1]]][[1]]$freq - ind.fun <- ind_table[indep[[1]]][[1]]$function + ind.fun <- ind_table[indep[[1]]][[1]]$fun } else { info(logger, paste(" #*****************************************#", " # Starting Independent verification of ECV ", " indep[[1]]$name, indep[[1]]$freq", sep = "\n")) - variable <- indep[[1]]$name - store.freq <- indep[[1]]$freq + variable <- indep$name + store.freq <- indep$freq ind.fun <- NULL } hcst.inityear <- recipe$Analysis$Time$hcst_start diff --git a/recipes/seasonal_complex.yml b/recipes/seasonal_complex.yml index 8b633cb0..dd0137cd 100644 --- a/recipes/seasonal_complex.yml +++ b/recipes/seasonal_complex.yml @@ -32,8 +32,8 @@ Analysis: - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} Regrid: - method: ycon - type: 180x90 + method: bicubic + type: system Workflow: Calibration: method: SBC diff --git a/recipes/seasonal_oper.yml b/recipes/seasonal_oper.yml index d58c3237..9598f449 100644 --- a/recipes/seasonal_oper.yml +++ b/recipes/seasonal_oper.yml @@ -31,8 +31,8 @@ Analysis: - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} Regrid: - method: ycon - type: 180x90 + method: bilinear + type: system Workflow: Calibration: method: SBC diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 4a1e5612..3fea5594 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -58,8 +58,13 @@ check_recipe <- function(recipe, file, conf, logger) { } } } - fcst.sdate <- list(stream = stream, fcst.sdate = fcst.sdate) - # more checks + fcst.sdate <- list(stream = stream, fcst.sdate = fcst.sdate) + # Regrid checks: + if (length(recipe$Analysis$Regrid) != 2) { + error(logger, + "The 'Regrid' element should specified the 'method' and 'type'.") + } +# more checks # ... # calculate number of workflows to create for each variable and if (length(recipe$Analysis$Horizon) > 1) { @@ -138,7 +143,8 @@ check_number_of_dependent_verifications <- function(recipe, conf) { if (is.null(recipe$Analysis$Variables$Indicators) || (length(recipe$Analysis$Variables$Indicators) == 1 && is.null(recipe$Analysis$Variables$ECVs))) { - independent_verifications <- recipe$Analysis$Variables + independent_verifications <- append(recipe$Analysis$Variables$ECVs, + recipe$Analysis$Variables$Indicators) } else { ecvs <- recipe$Analysi$Variables$ECVs inds <- recipe$Analysi$Variables$Indicators -- GitLab