From eef452421f3ed8203caea5100917eb1e83c3c766 Mon Sep 17 00:00:00 2001 From: nperez Date: Thu, 26 Aug 2021 18:31:49 +0200 Subject: [PATCH 1/6] First version for workflow development --- modules/Calibration/Calibration.R | 38 +++++++++++++++++++++++++++ modules/Skill/Skill.R | 43 +++++++++++++++++++++++++++++++ modules/verifications.R | 34 ++++++++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 modules/Calibration/Calibration.R create mode 100644 modules/Skill/Skill.R diff --git a/modules/Calibration/Calibration.R b/modules/Calibration/Calibration.R new file mode 100644 index 00000000..c837fa42 --- /dev/null +++ b/modules/Calibration/Calibration.R @@ -0,0 +1,38 @@ +# Code to apply any correction method: +# simple bias adjustment +# variance inflation +# quantile mapping + +## Which parameter are required? +if ("obs" %in% ls() || is.null(obs)) { + error(logger, + "There is no object 'obs' in the global environment or it is NULL") +} +if (stream == "fcst" && ("fcst" %in% ls() || is.null(fcst))) { + error(logger, + "There is no object 'fcst' in the global environment or it is NULL") +} +if ("hcst" %in% ls() || is.null(hcst)) { + error(logger, + "There is no object 'hcst' in the global environment or it is NULL") +} +if ("method" %in% ls() || is.null(method)) { + warn(logger, + "Calibration method not found and it is set as 'SBC'.") + method <- 'SBC' +} +if (method %in% c('SBC', 'SimpleBiasCorrection')) { + cal_fun <- "CSTools::Calibration" + cal_method <- "bias" +} else if (method %in% c("Inflation", "VarianceInflation")) { + cal_fun <- "CSTools::Calibration" + cal_method <- "evmos" +} else if (method %in% c("QM", "QuantileMapping")) { + cal_fun <- "CSTools::QuantileMapping" +} else { + error(logger, "Unknown calibration method definde in the recipe.") +} +info(logger, paste("#-------------------------- ", + " running Calibration module ", + " it can call ", cal_fun )) + diff --git a/modules/Skill/Skill.R b/modules/Skill/Skill.R new file mode 100644 index 00000000..f1e1a63c --- /dev/null +++ b/modules/Skill/Skill.R @@ -0,0 +1,43 @@ +# This module should calculate verification metrics at any time of the workflow +# It should implement different verification metrics: +# - FRPSS and RPSS +# - FCRPSS abd CRPSS +# - enscorr +# - bias +# - reliability diagram +# - ask Carlos which decadal metrics he is currently using + +## Which parameter are required? +if ("obs" %in% ls() || is.null(obs)) { + error(logger, + "There is no object 'obs' in the global environment or it is NULL") +} +if (stream == "fcst" && ("fcst" %in% ls() || is.null(fcst))) { + error(logger, + "There is no object 'fcst' in the global environment or it is NULL") +} +if ("hcst" %in% ls() || is.null(hcst)) { + error(logger, + "There is no object 'hcst' in the global environment or it is NULL") +} +if ("method" %in% ls() || is.null(metric)) { + warn(logger, + "Verification metric not found and it is set as 'EnsCorr'.") + metric <- 'EnsCorr' +} +if (metric %in% c('FRPSS', 'RPSS')) { + metric_fun <- "veriApply" + metric_method <- "FairRpss" +} else if (metric %in% c("FCRPSS", "CRPSS")) { + metric_fun <- "veriApply" +} else if (metric %in% c("EnsCorr", "EnsCor")) { + metric_fun <- "veriApply" + metric_method <- "EnsCorr" +#... +} else { + error(logger, "Unknown verification metric defined in the recipe.") +} +info(logger, paste("#-------------------------- ", + " running Skill module ", + " it can call ", metric_fun )) + diff --git a/modules/verifications.R b/modules/verifications.R index 8078762c..bb45d552 100644 --- a/modules/verifications.R +++ b/modules/verifications.R @@ -41,6 +41,40 @@ print(indep) for (sdate in verifications$fcst.sdate) { fcst.sdate <- sdate source("modules/data_load/seas5.load.R") + # Translate $Workflow to call modules: + ## 1) Clean step of the workflow set as FALSE or NULL or None: + modules <- names(recipe$Analysis$Workflow) + for (mod in modules) { + if ((is.logical(recipe$Analysis$Workflow[[mod]][[1]]) && + recipe$Analysis$Workflow[[mod]][[1]] == FALSE) || + recipe$Analysis$Workflow[[mod]][[1]] == 'None' || + is.null(recipe$Analysis$Workflow[[mod]][[1]])) { + info(logger, + paste("The module", mod, "won't be executed.")) + recipe$Analysis$Workflow <- recipe$Analysis$Workflow[ + -which(names(recipe$Analysis$Workflow) == mod)] + } + } + modules <- names(recipe$Analysis$Workflow) + ## 2) Call each module and pass arguments: + for (mod in modules) { + info(logger, paste("Start running module", mod)) + module_code <- file.path(conf$code_dir, "modules", mod, + paste0(mod, ".R")) + print(module_code) + # Define variables setup in the recipe + for (param in names(recipe$Analysis$Workflow[[mod]])) { + info(logger, paste("Variable", param, + "definded in the Global environment.")) + assign(as.character(param), + recipe$Analysis$Workflow[[mod]][[param]]) + } + source(module_code) + # TO DO: + # Check the arguments of each module can be an option here: + #... + info(logger, paste(mod, "run successfully.")) + } } } } -- GitLab From 6fd789de2c9f613a4f3e8a74746bfc2c239ba98f Mon Sep 17 00:00:00 2001 From: nperez Date: Fri, 27 Aug 2021 13:10:37 +0200 Subject: [PATCH 2/6] Allow multiple skill metrics --- OperationalCS.R | 1 + modules/Calibration/Calibration.R | 8 ++-- modules/Skill/Skill.R | 9 +++-- modules/verifications.R | 31 ++++++++++---- recipes/tests/seasonal_testWorkflow1.yml | 51 ++++++++++++++++++++++++ 5 files changed, 84 insertions(+), 16 deletions(-) create mode 100644 recipes/tests/seasonal_testWorkflow1.yml diff --git a/OperationalCS.R b/OperationalCS.R index 2f034dbc..ffbb7ebf 100644 --- a/OperationalCS.R +++ b/OperationalCS.R @@ -8,6 +8,7 @@ print(args) library(yaml) # To test: +# args <- NULL; args[1] <- "recipes/conf_user.yml"; args[2] <- "recipes/tests/seasonal_testWorkflow1.yml" # args <- NULL; args[1] <- "recipes/conf_user.yml"; args[2] <- "recipes/seasonal_oper.yml" # args <- NULL; args[1] <- "recipes/conf_user.yml"; args[2] <- "recipes/seasonal_complex.yml" diff --git a/modules/Calibration/Calibration.R b/modules/Calibration/Calibration.R index c837fa42..8eb88250 100644 --- a/modules/Calibration/Calibration.R +++ b/modules/Calibration/Calibration.R @@ -4,19 +4,19 @@ # quantile mapping ## Which parameter are required? -if ("obs" %in% ls() || is.null(obs)) { +if (!("obs" %in% ls()) || is.null(obs)) { error(logger, "There is no object 'obs' in the global environment or it is NULL") } -if (stream == "fcst" && ("fcst" %in% ls() || is.null(fcst))) { +if (stream == "fcst" && (!("fcst" %in% ls()) || is.null(fcst))) { error(logger, "There is no object 'fcst' in the global environment or it is NULL") } -if ("hcst" %in% ls() || is.null(hcst)) { +if (!("hcst" %in% ls()) || is.null(hcst)) { error(logger, "There is no object 'hcst' in the global environment or it is NULL") } -if ("method" %in% ls() || is.null(method)) { +if (!("method" %in% ls()) || is.null(method)) { warn(logger, "Calibration method not found and it is set as 'SBC'.") method <- 'SBC' diff --git a/modules/Skill/Skill.R b/modules/Skill/Skill.R index f1e1a63c..4be37ccc 100644 --- a/modules/Skill/Skill.R +++ b/modules/Skill/Skill.R @@ -8,19 +8,19 @@ # - ask Carlos which decadal metrics he is currently using ## Which parameter are required? -if ("obs" %in% ls() || is.null(obs)) { +if (!("obs" %in% ls()) || is.null(obs)) { error(logger, "There is no object 'obs' in the global environment or it is NULL") } -if (stream == "fcst" && ("fcst" %in% ls() || is.null(fcst))) { +if (stream == "fcst" && (!("fcst" %in% ls()) || is.null(fcst))) { error(logger, "There is no object 'fcst' in the global environment or it is NULL") } -if ("hcst" %in% ls() || is.null(hcst)) { +if (!("hcst" %in% ls()) || is.null(hcst)) { error(logger, "There is no object 'hcst' in the global environment or it is NULL") } -if ("method" %in% ls() || is.null(metric)) { +if (!("method" %in% ls()) || is.null(metric)) { warn(logger, "Verification metric not found and it is set as 'EnsCorr'.") metric <- 'EnsCorr' @@ -36,6 +36,7 @@ if (metric %in% c('FRPSS', 'RPSS')) { #... } else { error(logger, "Unknown verification metric defined in the recipe.") + metric_fun <- 'NotFound' } info(logger, paste("#-------------------------- ", " running Skill module ", diff --git a/modules/verifications.R b/modules/verifications.R index bb45d552..ec908625 100644 --- a/modules/verifications.R +++ b/modules/verifications.R @@ -1,5 +1,4 @@ for (indep in verifications$independent) { -print(indep) if (length(indep) == 1) { info(logger, paste(" #*****************************************#", @@ -56,16 +55,32 @@ print(indep) } } modules <- names(recipe$Analysis$Workflow) - ## 2) Call each module and pass arguments: + ## 2) Create a common format for all modules + tmp_modules <- list() for (mod in modules) { - info(logger, paste("Start running module", mod)) - module_code <- file.path(conf$code_dir, "modules", mod, - paste0(mod, ".R")) - print(module_code) + if (length(recipe$Analysis$Workflow[[mod]]) > 1) { + names(recipe$Analysis$Workflow[[mod]]) <- + rep(mod, length(recipe$Analysis$Workflow[[mod]])) + tmp_modules <- append(tmp_modules, + recipe$Analysis$Workflow[[mod]]) + } else { + tmp_modules <- append(tmp_modules, + recipe$Analysis$Workflow[mod]) + } + } + modules <- tmp_modules + recipe$Analysis$Workflow <- tmp_modules + ## 3) Call each module and pass arguments: + for (mod in 1:length(modules)) { + info(logger, paste("Start running module", names(modules)[mod])) + module_code <- file.path(conf$code_dir, "modules", + names(modules)[mod], + paste0(names(modules)[mod], ".R")) # Define variables setup in the recipe for (param in names(recipe$Analysis$Workflow[[mod]])) { - info(logger, paste("Variable", param, - "definded in the Global environment.")) + print(param) + info(logger, paste("Variable *", param, "* set as ", + recipe$Analysis$Workflow[[mod]][[param]])) assign(as.character(param), recipe$Analysis$Workflow[[mod]][[param]]) } diff --git a/recipes/tests/seasonal_testWorkflow1.yml b/recipes/tests/seasonal_testWorkflow1.yml new file mode 100644 index 00000000..c3b87613 --- /dev/null +++ b/recipes/tests/seasonal_testWorkflow1.yml @@ -0,0 +1,51 @@ +Description: + Author: N.Pérez-Zanón + Info: This is a test to transform s2s4e data-analysis for SEAS5 + +Analysis: + Horizon: Seasonal + Variables: + ECVs: + - {name: tas, freq: monthly_mean} + Indicators: + - None + Datasets: + System: + - name: system5c3s + Multimodel: False + Reference: + - {name: era5} + Time: + sdate: + fcst_year: 2021 + fcst_month: 07 + fcst_day: 01 + hcst_start: 2000 + hcst_end: 2002 + leadtimemin: 2 + leadtimemax: 4 + Region: + Global: TRUE + Aggregation: False + Regional: + - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} + - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} + Regrid: + method: bilinear + type: system + Workflow: + Calibration: + method: SBC + Skill: + - {metric: fRPSS, probs: [1/3, 2/3]} + - {metric: BSS10} + - {metric: BSS90} + - {metric: EnsCorr} + - {metric: Bias} + Indicators: + index: FALSE + Output_format: S2S4E + +Run: + Loglevel: INFO + Terminal: TRUE -- GitLab From ab4dc47adc0b448aa5509aa6dbd5db997c8342be Mon Sep 17 00:00:00 2001 From: nperez Date: Fri, 27 Aug 2021 17:37:23 +0200 Subject: [PATCH 3/6] Add stops, check multiple Skills in the recipe, add tests --- OperationalCS.R | 3 +- modules/Calibration/Calibration.R | 10 +++-- modules/verifications.R | 18 ++++++--- recipes/tests/execute_tests.R | 42 +++++++++++++++++++ recipes/tests/seasonal_testWorkflow2.yml | 50 +++++++++++++++++++++++ recipes/tests/seasonal_testWorkflow3.yml | 50 +++++++++++++++++++++++ recipes/tests/seasonal_testWorkflow4.yml | 51 ++++++++++++++++++++++++ recipes/tests/seasonal_testWorkflow5.yml | 49 +++++++++++++++++++++++ recipes/tests/seasonal_testWorkflow6.yml | 51 ++++++++++++++++++++++++ recipes/tests/seasonal_testWorkflow7.yml | 51 ++++++++++++++++++++++++ tools/check_recipe.R | 48 ++++++++++++++++++---- 11 files changed, 405 insertions(+), 18 deletions(-) create mode 100644 recipes/tests/execute_tests.R create mode 100644 recipes/tests/seasonal_testWorkflow2.yml create mode 100644 recipes/tests/seasonal_testWorkflow3.yml create mode 100644 recipes/tests/seasonal_testWorkflow4.yml create mode 100644 recipes/tests/seasonal_testWorkflow5.yml create mode 100644 recipes/tests/seasonal_testWorkflow6.yml create mode 100644 recipes/tests/seasonal_testWorkflow7.yml diff --git a/OperationalCS.R b/OperationalCS.R index ffbb7ebf..c63c8c1f 100644 --- a/OperationalCS.R +++ b/OperationalCS.R @@ -27,6 +27,7 @@ check_conf(conf, file = args[1], logger) verifications <- check_recipe(recipe, file = args[2], conf, logger) # Go to verification code: -capture.output(source("modules/verifications.R"), file = log_file, type ='message', +capture.output(source("modules/verifications.R"), + file = log_file, type ='message', append = TRUE) diff --git a/modules/Calibration/Calibration.R b/modules/Calibration/Calibration.R index 8eb88250..b86ea959 100644 --- a/modules/Calibration/Calibration.R +++ b/modules/Calibration/Calibration.R @@ -7,14 +7,17 @@ if (!("obs" %in% ls()) || is.null(obs)) { error(logger, "There is no object 'obs' in the global environment or it is NULL") + stop("EXECUTION FAILED") } if (stream == "fcst" && (!("fcst" %in% ls()) || is.null(fcst))) { error(logger, "There is no object 'fcst' in the global environment or it is NULL") + stop("EXECUTION FAILED") } if (!("hcst" %in% ls()) || is.null(hcst)) { error(logger, "There is no object 'hcst' in the global environment or it is NULL") + stop("EXECUTION FAILED") } if (!("method" %in% ls()) || is.null(method)) { warn(logger, @@ -31,8 +34,9 @@ if (method %in% c('SBC', 'SimpleBiasCorrection')) { cal_fun <- "CSTools::QuantileMapping" } else { error(logger, "Unknown calibration method definde in the recipe.") + stop("EXECUTION FAILED") } -info(logger, paste("#-------------------------- ", - " running Calibration module ", - " it can call ", cal_fun )) +info(logger, paste("#-------------------------- ", "\n", + "running Calibration module", "\n", + "it can call", cal_fun )) diff --git a/modules/verifications.R b/modules/verifications.R index ec908625..40bbf5eb 100644 --- a/modules/verifications.R +++ b/modules/verifications.R @@ -72,14 +72,20 @@ for (indep in verifications$independent) { recipe$Analysis$Workflow <- tmp_modules ## 3) Call each module and pass arguments: for (mod in 1:length(modules)) { - info(logger, paste("Start running module", names(modules)[mod])) + # In case multiple calls to a module e.g.: Skill_1 --> Skill + if (any(strsplit(names(modules)[mod], "")[[1]] == "_")) { + module_name <- substr(names(modules)[mod], start = 1, + stop = which(strsplit(names(modules)[mod], "")[[1]] == "_") - 1) + } else { + module_name <- names(modules)[mod] + } + info(logger, paste("Start running module", module_name)) module_code <- file.path(conf$code_dir, "modules", - names(modules)[mod], - paste0(names(modules)[mod], ".R")) + module_name, + paste0(module_name, ".R")) # Define variables setup in the recipe for (param in names(recipe$Analysis$Workflow[[mod]])) { - print(param) - info(logger, paste("Variable *", param, "* set as ", + info(logger, paste("Variable *", param, "* set as", recipe$Analysis$Workflow[[mod]][[param]])) assign(as.character(param), recipe$Analysis$Workflow[[mod]][[param]]) @@ -88,7 +94,7 @@ for (indep in verifications$independent) { # TO DO: # Check the arguments of each module can be an option here: #... - info(logger, paste(mod, "run successfully.")) + info(logger, paste(module_name, "module run finished.")) } } } diff --git a/recipes/tests/execute_tests.R b/recipes/tests/execute_tests.R new file mode 100644 index 00000000..538e9415 --- /dev/null +++ b/recipes/tests/execute_tests.R @@ -0,0 +1,42 @@ +library(yaml) + +args <- NULL; +args[1] <- "recipes/conf_user.yml"; + +# Function to run tests: +# source_lines("/esarchive/scratch/nperez/git/startR/inst/doc/usecase/ex2_1_timedim.R", +# start = 4, end = 14) +source_lines <- function(file, start, end, ...) { + file.lines <- scan(file, what = character(), skip = start - 1, + nlines = end - start + 1, sep = '\n') + file.lines.collapsed <- paste(file.lines, collapse = '\n') + source(textConnection(file.lines.collapsed), ...) +} + +# ------------------------------------------ +# Section to check recipes that should work: +args[2] <- "recipes/tests/seasonal_testWorkflow1.yml" +source_lines("OperationalCS.R", start = 14, end = 31) +# Calibration method None --> raw data verification +args[2] <- "recipes/tests/seasonal_testWorkflow4.yml" +source_lines("OperationalCS.R", start = 14, end = 32) +# Calibration: None --> raw data verification +args[2] <- "recipes/tests/seasonal_testWorkflow5.yml" +source_lines("OperationalCS.R", start = 14, end = 32) +# Case Skill_1 and Skill_2 when multiple times needed +args[2] <- "recipes/tests/seasonal_testWorkflow7.yml" +source_lines("OperationalCS.R", start = 14, end = 32) + +# ------------------------------------------ +# Section to check recipes that should fail: +## This should fail because there is no Horizon: +args[2] <- "recipes/tests/seasonal_testWorkflow2.yml" +source_lines("OperationalCS.R", start = 14, end = 32) + +## This should fail because there are 2 Calibration options: +args[2] <- "recipes/tests/seasonal_testWorkflow3.yml" +source_lines("OperationalCS.R", start = 14, end = 32) + +## This fails because it is not allow repeating the name Skill +args[2] <- "recipes/tests/seasonal_testWorkflow6.yml" +source_lines("OperationalCS.R", start = 14, end = 32) diff --git a/recipes/tests/seasonal_testWorkflow2.yml b/recipes/tests/seasonal_testWorkflow2.yml new file mode 100644 index 00000000..e31e4e5a --- /dev/null +++ b/recipes/tests/seasonal_testWorkflow2.yml @@ -0,0 +1,50 @@ +Description: + Author: N.Pérez-Zanón + Info: This is a test to transform s2s4e data-analysis for SEAS5 + +Analysis: + Variables: + ECVs: + - {name: tas, freq: monthly_mean} + Indicators: + - None + Datasets: + System: + - name: system5c3s + Multimodel: False + Reference: + - {name: era5} + Time: + sdate: + fcst_year: 2021 + fcst_month: 07 + fcst_day: 01 + hcst_start: 2000 + hcst_end: 2002 + leadtimemin: 2 + leadtimemax: 4 + Region: + Global: TRUE + Aggregation: False + Regional: + - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} + - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} + Regrid: + method: bilinear + type: system + Workflow: + Calibration: + method: SBC + Skill: + - {metric: fRPSS, probs: [1/3, 2/3]} + - {metric: BSS10} + - {metric: BSS90} + - {metric: EnsCorr} + - {metric: Bias} + Indicators: + index: FALSE + Output_format: S2S4E + +Run: + Loglevel: INFO + Terminal: TRUE diff --git a/recipes/tests/seasonal_testWorkflow3.yml b/recipes/tests/seasonal_testWorkflow3.yml new file mode 100644 index 00000000..26754f23 --- /dev/null +++ b/recipes/tests/seasonal_testWorkflow3.yml @@ -0,0 +1,50 @@ +Description: + Author: N.Pérez-Zanón + Info: This is a test to transform s2s4e data-analysis for SEAS5 + +Analysis: + Horizon: Seasonal + Variables: + ECVs: + - {name: tas, freq: monthly_mean} + Indicators: + - None + Datasets: + System: + - name: system5c3s + Multimodel: False + Reference: + - {name: era5} + Time: + sdate: + fcst_year: 2021 + fcst_month: 07 + fcst_day: 01 + hcst_start: 2000 + hcst_end: 2002 + leadtimemin: 2 + leadtimemax: 4 + Region: + Global: TRUE + Aggregation: False + Regional: + - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} + - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} + Regrid: + method: bilinear + type: system + Workflow: + Calibration: + - {method: SBC} + - {method: VarianceInflation} + Skill: + - {metric: fRPSS, probs: [1/3, 2/3]} + - {metric: BSS10} + - {metric: BSS90} + - {metric: EnsCorr} + - {metric: Bias} + Output_format: S2S4E + +Run: + Loglevel: INFO + Terminal: TRUE diff --git a/recipes/tests/seasonal_testWorkflow4.yml b/recipes/tests/seasonal_testWorkflow4.yml new file mode 100644 index 00000000..2b7992ee --- /dev/null +++ b/recipes/tests/seasonal_testWorkflow4.yml @@ -0,0 +1,51 @@ +Description: + Author: N.Pérez-Zanón + Info: This is a test to transform s2s4e data-analysis for SEAS5 + +Analysis: + Horizon: Seasonal + Variables: + ECVs: + - {name: tas, freq: monthly_mean} + Indicators: + - None + Datasets: + System: + - name: system5c3s + Multimodel: False + Reference: + - {name: era5} + Time: + sdate: + fcst_year: 2021 + fcst_month: 07 + fcst_day: 01 + hcst_start: 2000 + hcst_end: 2002 + leadtimemin: 2 + leadtimemax: 4 + Region: + Global: TRUE + Aggregation: False + Regional: + - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} + - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} + Regrid: + method: bilinear + type: system + Workflow: + Calibration: + method: None + Skill: + - {metric: fRPSS, probs: [1/3, 2/3]} + - {metric: BSS10} + - {metric: BSS90} + - {metric: EnsCorr} + - {metric: Bias} + Indicators: + index: FALSE + Output_format: S2S4E + +Run: + Loglevel: INFO + Terminal: TRUE diff --git a/recipes/tests/seasonal_testWorkflow5.yml b/recipes/tests/seasonal_testWorkflow5.yml new file mode 100644 index 00000000..04700ed9 --- /dev/null +++ b/recipes/tests/seasonal_testWorkflow5.yml @@ -0,0 +1,49 @@ +Description: + Author: N.Pérez-Zanón + Info: This is a test to transform s2s4e data-analysis for SEAS5 + +Analysis: + Horizon: Seasonal + Variables: + ECVs: + - {name: tas, freq: monthly_mean} + Indicators: + - None + Datasets: + System: + - name: system5c3s + Multimodel: False + Reference: + - {name: era5} + Time: + sdate: + fcst_year: 2021 + fcst_month: 07 + fcst_day: 01 + hcst_start: 2000 + hcst_end: 2002 + leadtimemin: 2 + leadtimemax: 4 + Region: + Global: TRUE + Aggregation: False + Regional: + - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} + - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} + Regrid: + method: bilinear + type: system + Workflow: + Calibration: FALSE + Skill: + - {metric: fRPSS, probs: [1/3, 2/3]} + - {metric: BSS10} + - {metric: BSS90} + - {metric: EnsCorr} + - {metric: Bias} + Indicators: FALSE + Output_format: S2S4E + +Run: + Loglevel: INFO + Terminal: TRUE diff --git a/recipes/tests/seasonal_testWorkflow6.yml b/recipes/tests/seasonal_testWorkflow6.yml new file mode 100644 index 00000000..45033b6f --- /dev/null +++ b/recipes/tests/seasonal_testWorkflow6.yml @@ -0,0 +1,51 @@ +Description: + Author: N.Pérez-Zanón + Info: This is a test to transform s2s4e data-analysis for SEAS5 + +Analysis: + Horizon: Seasonal + Variables: + ECVs: + - {name: tas, freq: monthly_mean} + Indicators: + - None + Datasets: + System: + - name: system5c3s + Multimodel: False + Reference: + - {name: era5} + Time: + sdate: + fcst_year: 2021 + fcst_month: 07 + fcst_day: 01 + hcst_start: 2000 + hcst_end: 2002 + leadtimemin: 2 + leadtimemax: 4 + Region: + Global: TRUE + Aggregation: False + Regional: + - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} + - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} + Regrid: + method: bilinear + type: system + Workflow: + Skill: + - {metric: EnsCorr} + - {metric: Bias} + Calibration: + method: SBC + Skill: + - {metric: EnsCorr} + - {metric: Bias} + Indicators: + index: FALSE + Output_format: S2S4E + +Run: + Loglevel: INFO + Terminal: TRUE diff --git a/recipes/tests/seasonal_testWorkflow7.yml b/recipes/tests/seasonal_testWorkflow7.yml new file mode 100644 index 00000000..093f697c --- /dev/null +++ b/recipes/tests/seasonal_testWorkflow7.yml @@ -0,0 +1,51 @@ +Description: + Author: N.Pérez-Zanón + Info: This is a test to transform s2s4e data-analysis for SEAS5 + +Analysis: + Horizon: Seasonal + Variables: + ECVs: + - {name: tas, freq: monthly_mean} + Indicators: + - None + Datasets: + System: + - name: system5c3s + Multimodel: False + Reference: + - {name: era5} + Time: + sdate: + fcst_year: 2021 + fcst_month: 07 + fcst_day: 01 + hcst_start: 2000 + hcst_end: 2002 + leadtimemin: 2 + leadtimemax: 4 + Region: + Global: TRUE + Aggregation: False + Regional: + - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} + - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} + Regrid: + method: bilinear + type: system + Workflow: + Skill_1: + - {metric: EnsCorr} + - {metric: Bias} + Calibration: + method: SBC + Skill_2: + - {metric: EnsCorr} + - {metric: Bias} + Indicators: + index: FALSE + Output_format: S2S4E + +Run: + Loglevel: INFO + Terminal: TRUE diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 3fea5594..83a6d7f6 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -11,23 +11,31 @@ check_recipe <- function(recipe, file, conf, logger) { params <- c('Horizon', 'Time', 'Variables', 'Region', 'Regrid', 'Workflow', 'Datasets') if (!all(params %in% names(recipe$Analysis))) { + params <- paste(params, collapse = ' ') error(logger, paste("The element 'Analysis' in the recipe should contain these", - "elements:", paste(params, collapse = " "))) + "elements:", params)) + stop("EXECUTION FAILED") } horizons <- c('Subseasonal', 'Seasonal', 'Decadal') if (!any(horizons %in% recipe$Analysis$Horizon)) { + horizons <- paste(horizons, collapse = " ") error(logger, - "The element 'Horizon' in the recipe should be one of the followings:", - paste(horizons, collapse = " ")) + paste("The element 'Horizon' in the recipe", + "should be one of the followings:", + horizons)) + stop("EXECUTION FAILED") } # Check temporal settings and # count the number of verifications - time_settings <- c('sdate', 'leadtimemin', 'leadtimemax', 'hcst_start', 'hcst_end') + time_settings <- c('sdate', 'leadtimemin', 'leadtimemax', + 'hcst_start', 'hcst_end') if (!all(time_settings %in% names(recipe$Analysis$Time))) { + time_settings <- paste(time_settings, collapse = " ") error(logger, paste("The element 'Time' in the recipe should contain these elements:", - paste(time_settings, collapse = " "))) + time_settings)) + stop("EXECUTION FAILED") } if (is.null(recipe$Analysis$Time$sdate$fcst_year) || recipe$Analysis$Time$sdate$fcst_year == 'None') { @@ -63,17 +71,20 @@ check_recipe <- function(recipe, file, conf, logger) { if (length(recipe$Analysis$Regrid) != 2) { error(logger, "The 'Regrid' element should specified the 'method' and 'type'.") + stop("EXECUTION FAILED") } # more checks # ... # calculate number of workflows to create for each variable and if (length(recipe$Analysis$Horizon) > 1) { - error(logger, "Only 1 Horizon can be specified in the recipe") + error(logger, "Only 1 Horizon can be specified in the recipe") + stop("EXECUTION FAILED") } nvar <- length(recipe$Analysis$Variables) if (nvar > 2) { error(logger, "Only two type of Variables can be listed: ECVs and Indicators.") + stop("EXECUTION FAILED") } # remove NULL or None Indicators or ECVs from the recipe: if (!is.list(recipe$Analysis$Variables$Indicators)) { @@ -84,12 +95,30 @@ check_recipe <- function(recipe, file, conf, logger) { recipe$Analysis$Variables <- recipe$Analysis$Variables[ -which(names(recipe$Analysis$Variables) == 'ECVs')] } + # Only one Calibration method allowed: + if ((is.logical(recipe$Analysis$Workflow$Calibration[[1]]) && + recipe$Analysis$Workflow$Calibration[[1]] == FALSE) || + recipe$Analysis$Workflow$Calibration[[1]] == 'None' || + is.null(recipe$Analysis$Workflow$Calibration[[1]])) { + warn(logger, + "There is no Calibration method selected, raw data verification.") + recipe$Analysis$Workflow$Calibration[[1]] <- FALSE + } else { + # remove multiple calibration methods + if (is.null(names(recipe$Analysis$Workflow$Calibration))) { + error(logger, + "The 'Calibration' element should specified at least the 'method'.") + stop("EXECUTION FAILED") + } + } if (length(recipe$Analysis$Region) > 3) { error(logger, "The section 'Region' in the recipe should be checked.", "Global, Countries and Regional elements are expected.") + stop("EXECUTION FAILED") } if (!is.logical(recipe$Analysis$Region$Global)) { error(logger, "The element 'Global' in Region should be logical.") + stop("EXECUTION FAILED") } nregions <- 0 if (recipe$Analysis$Region$Global == TRUE) { @@ -103,9 +132,12 @@ check_recipe <- function(recipe, file, conf, logger) { limits <- c('latmin', 'latmax', 'lonmin', 'lonmax') for (i in 1:length(recipe$Analysis$Region$Regional)) { if (!all(limits %in% names(recipe$Analysis$Region$Regional[[i]]))) { + limits <- paste(limits, collapse = " ") error(logger, - "Each region defined in element 'Regional' should have 4 elements:", - paste(limits, collapse = " ")) + paste("Each region defined in element 'Regional'", + "should have 4 elements:", + limits)) + stop("EXECUTION FAILED") } # are numeric? class list mode list } -- GitLab From 27425091fc1843e5c1f56ba6846e0fe8278b5c36 Mon Sep 17 00:00:00 2001 From: nperez Date: Fri, 27 Aug 2021 18:02:58 +0200 Subject: [PATCH 4/6] params can be a vector --- OperationalCS.R | 1 + modules/Skill/Skill.R | 7 +++---- modules/verifications.R | 9 +++++++-- recipes/tests/execute_tests.R | 14 +++++++------- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/OperationalCS.R b/OperationalCS.R index c63c8c1f..bdb960c8 100644 --- a/OperationalCS.R +++ b/OperationalCS.R @@ -22,6 +22,7 @@ source("tools/libs.R") logger <- prepare_outputs(recipe = recipe, file = args[2], conf = conf) log_file <- logger$logname logger <- logger$logger + # Checks: check_conf(conf, file = args[1], logger) verifications <- check_recipe(recipe, file = args[2], conf, logger) diff --git a/modules/Skill/Skill.R b/modules/Skill/Skill.R index 4be37ccc..e7cad0cb 100644 --- a/modules/Skill/Skill.R +++ b/modules/Skill/Skill.R @@ -20,7 +20,7 @@ if (!("hcst" %in% ls()) || is.null(hcst)) { error(logger, "There is no object 'hcst' in the global environment or it is NULL") } -if (!("method" %in% ls()) || is.null(metric)) { +if (!("metric" %in% ls()) || is.null(metric)) { warn(logger, "Verification metric not found and it is set as 'EnsCorr'.") metric <- 'EnsCorr' @@ -38,7 +38,6 @@ if (metric %in% c('FRPSS', 'RPSS')) { error(logger, "Unknown verification metric defined in the recipe.") metric_fun <- 'NotFound' } -info(logger, paste("#-------------------------- ", - " running Skill module ", +info(logger, paste("#-------------------------- ", "\n", + " running Skill module ", "\n", " it can call ", metric_fun )) - diff --git a/modules/verifications.R b/modules/verifications.R index 40bbf5eb..d161c097 100644 --- a/modules/verifications.R +++ b/modules/verifications.R @@ -85,8 +85,13 @@ for (indep in verifications$independent) { paste0(module_name, ".R")) # Define variables setup in the recipe for (param in names(recipe$Analysis$Workflow[[mod]])) { - info(logger, paste("Variable *", param, "* set as", - recipe$Analysis$Workflow[[mod]][[param]])) + if (length(recipe$Analysis$Workflow[[mod]][[param]])) { + tmp <- paste(recipe$Analysis$Workflow[[mod]][[param]], + collapse = ",") + } else { + tmp <- recipe$Analysis$Workflow[[mod]][[param]] + } + info(logger, paste("Variable *", param, "* set as", tmp)) assign(as.character(param), recipe$Analysis$Workflow[[mod]][[param]]) } diff --git a/recipes/tests/execute_tests.R b/recipes/tests/execute_tests.R index 538e9415..0b35894c 100644 --- a/recipes/tests/execute_tests.R +++ b/recipes/tests/execute_tests.R @@ -16,27 +16,27 @@ source_lines <- function(file, start, end, ...) { # ------------------------------------------ # Section to check recipes that should work: args[2] <- "recipes/tests/seasonal_testWorkflow1.yml" -source_lines("OperationalCS.R", start = 14, end = 31) +source_lines("OperationalCS.R", start = 14, end = 50) # Calibration method None --> raw data verification args[2] <- "recipes/tests/seasonal_testWorkflow4.yml" -source_lines("OperationalCS.R", start = 14, end = 32) +source_lines("OperationalCS.R", start = 14, end = 50) # Calibration: None --> raw data verification args[2] <- "recipes/tests/seasonal_testWorkflow5.yml" -source_lines("OperationalCS.R", start = 14, end = 32) +source_lines("OperationalCS.R", start = 14, end = 50) # Case Skill_1 and Skill_2 when multiple times needed args[2] <- "recipes/tests/seasonal_testWorkflow7.yml" -source_lines("OperationalCS.R", start = 14, end = 32) +source_lines("OperationalCS.R", start = 14, end = 50) # ------------------------------------------ # Section to check recipes that should fail: ## This should fail because there is no Horizon: args[2] <- "recipes/tests/seasonal_testWorkflow2.yml" -source_lines("OperationalCS.R", start = 14, end = 32) +source_lines("OperationalCS.R", start = 14, end = 50) ## This should fail because there are 2 Calibration options: args[2] <- "recipes/tests/seasonal_testWorkflow3.yml" -source_lines("OperationalCS.R", start = 14, end = 32) +source_lines("OperationalCS.R", start = 14, end = 50) ## This fails because it is not allow repeating the name Skill args[2] <- "recipes/tests/seasonal_testWorkflow6.yml" -source_lines("OperationalCS.R", start = 14, end = 32) +source_lines("OperationalCS.R", start = 14, end = 50) -- GitLab From 1ef67e9120e343dd2d481f4c5c42b5123c17afae Mon Sep 17 00:00:00 2001 From: nperez Date: Fri, 27 Aug 2021 18:34:01 +0200 Subject: [PATCH 5/6] fixed case ECVs not specified --- recipes/tests/execute_tests.R | 4 +- recipes/tests/seasonal_testWorkflow8.yml | 49 ++++++++++++++++++++++++ tools/check_recipe.R | 6 ++- 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 recipes/tests/seasonal_testWorkflow8.yml diff --git a/recipes/tests/execute_tests.R b/recipes/tests/execute_tests.R index 0b35894c..584bc98a 100644 --- a/recipes/tests/execute_tests.R +++ b/recipes/tests/execute_tests.R @@ -26,7 +26,9 @@ source_lines("OperationalCS.R", start = 14, end = 50) # Case Skill_1 and Skill_2 when multiple times needed args[2] <- "recipes/tests/seasonal_testWorkflow7.yml" source_lines("OperationalCS.R", start = 14, end = 50) - +# Indicator +args[2] <- "recipes/tests/seasonal_testWorkflow8.yml" +source_lines("OperationalCS.R", start = 14, end = 50) # ------------------------------------------ # Section to check recipes that should fail: ## This should fail because there is no Horizon: diff --git a/recipes/tests/seasonal_testWorkflow8.yml b/recipes/tests/seasonal_testWorkflow8.yml new file mode 100644 index 00000000..b4e57216 --- /dev/null +++ b/recipes/tests/seasonal_testWorkflow8.yml @@ -0,0 +1,49 @@ +Description: + Author: N.Pérez-Zanón + Info: This is a test to transform s2s4e data-analysis for SEAS5 + +Analysis: + Horizon: Seasonal + Variables: + Indicators: + - {name: gdd} + Datasets: + System: + - name: system5c3s + Multimodel: False + Reference: + - {name: era5} + Time: + sdate: + fcst_year: 2021 + fcst_month: 07 + fcst_day: 01 + hcst_start: 2000 + hcst_end: 2002 + leadtimemin: 2 + leadtimemax: 4 + Region: + Global: TRUE + Aggregation: False + Regional: + - {latmin: -90, latmax: 90, lonmin: 0, lonmax: 360} + - {latmin: -10, latmax: 10, lonmin: 0, lonmax: 20} + Regrid: + method: bilinear + type: system + Workflow: + Calibration: + method: SBC + Skill: + - {metric: fRPSS, probs: [1/3, 2/3]} + - {metric: BSS10} + - {metric: BSS90} + - {metric: EnsCorr} + - {metric: Bias} + Indicators: + index: TRUE + Output_format: S2S4E + +Run: + Loglevel: INFO + Terminal: TRUE diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 83a6d7f6..2b7479d6 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -87,11 +87,13 @@ check_recipe <- function(recipe, file, conf, logger) { stop("EXECUTION FAILED") } # remove NULL or None Indicators or ECVs from the recipe: - if (!is.list(recipe$Analysis$Variables$Indicators)) { + if (!is.null(recipe$Analysis$Variables$Indicators) && + !is.list(recipe$Analysis$Variables$Indicators)) { recipe$Analysis$Variables <- recipe$Analysis$Variables[ -which(names(recipe$Analysis$Variables) == 'Indicators')] } - if (!is.list(recipe$Analysis$Variables$ECVs)) { + if (!is.null(recipe$Analysis$Variables$ECVs) && + !is.list(recipe$Analysis$Variables$ECVs)) { recipe$Analysis$Variables <- recipe$Analysis$Variables[ -which(names(recipe$Analysis$Variables) == 'ECVs')] } -- GitLab From 66d4f5ca132ac07e38fb98127d0a267131ba4335 Mon Sep 17 00:00:00 2001 From: nperez Date: Mon, 30 Aug 2021 10:42:38 +0200 Subject: [PATCH 6/6] avoid overwritte recipe elements of the Workflow in the loops --- modules/verifications.R | 14 ++++++++------ recipes/tests/execute_tests.R | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/verifications.R b/modules/verifications.R index d161c097..2451b7d7 100644 --- a/modules/verifications.R +++ b/modules/verifications.R @@ -69,7 +69,6 @@ for (indep in verifications$independent) { } } modules <- tmp_modules - recipe$Analysis$Workflow <- tmp_modules ## 3) Call each module and pass arguments: for (mod in 1:length(modules)) { # In case multiple calls to a module e.g.: Skill_1 --> Skill @@ -84,16 +83,16 @@ for (indep in verifications$independent) { module_name, paste0(module_name, ".R")) # Define variables setup in the recipe - for (param in names(recipe$Analysis$Workflow[[mod]])) { - if (length(recipe$Analysis$Workflow[[mod]][[param]])) { - tmp <- paste(recipe$Analysis$Workflow[[mod]][[param]], + for (param in names(modules[[mod]])) { + if (length(modules[[mod]][[param]])) { + tmp <- paste(modules[[mod]][[param]], collapse = ",") } else { - tmp <- recipe$Analysis$Workflow[[mod]][[param]] + tmp <- modules[[mod]][[param]] } info(logger, paste("Variable *", param, "* set as", tmp)) assign(as.character(param), - recipe$Analysis$Workflow[[mod]][[param]]) + modules[[mod]][[param]]) } source(module_code) # TO DO: @@ -101,8 +100,11 @@ for (indep in verifications$independent) { #... info(logger, paste(module_name, "module run finished.")) } + info(logger, paste(sdate, "start date finished.")) } + info(logger, paste(ref, "reference dataset finished.")) } + info(logger, paste(sys, "systemn finished.")) } } } diff --git a/recipes/tests/execute_tests.R b/recipes/tests/execute_tests.R index 584bc98a..44292f16 100644 --- a/recipes/tests/execute_tests.R +++ b/recipes/tests/execute_tests.R @@ -29,6 +29,7 @@ source_lines("OperationalCS.R", start = 14, end = 50) # Indicator args[2] <- "recipes/tests/seasonal_testWorkflow8.yml" source_lines("OperationalCS.R", start = 14, end = 50) + # ------------------------------------------ # Section to check recipes that should fail: ## This should fail because there is no Horizon: -- GitLab