From 513a2763d1c6d145172042578c6f4a597328989f Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 24 Apr 2024 12:06:29 +0200 Subject: [PATCH 1/8] formatting --- modules/Scorecards/execute_scorecards.R | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/Scorecards/execute_scorecards.R b/modules/Scorecards/execute_scorecards.R index e4dcd4ec..b79a9ab5 100644 --- a/modules/Scorecards/execute_scorecards.R +++ b/modules/Scorecards/execute_scorecards.R @@ -18,7 +18,6 @@ for (variable in 1:length(recipe$Analysis$Variables)) { scorecard_recipe$Analysis$Datasets$System <- as.vector(unlist(recipe$Analysis$Datasets$System)) - ## Include multimodel in systems if(isTRUE(scorecard_recipe$Analysis$Datasets$Multimodel$execute) || scorecard_recipe$Analysis$Datasets$Multimodel$execute == 'both' || @@ -26,7 +25,6 @@ for (variable in 1:length(recipe$Analysis$Variables)) { scorecard_recipe$Analysis$Datasets$System <- c(scorecard_recipe$Analysis$Datasets$System, 'Multimodel') } - scorecard_recipe$Analysis$Datasets$Reference <- as.vector(unlist(recipe$Analysis$Datasets$Reference)) scorecard_recipe$Analysis$Variables <- -- GitLab From d9e0e82690d18e1c3c17f22d92cf5050387bf3d9 Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 24 Apr 2024 12:06:59 +0200 Subject: [PATCH 2/8] Add error message if 'enscorr' is requested for Scorecards but statistics are not --- tools/check_recipe.R | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 42cb832c..04193bd3 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -648,8 +648,14 @@ check_recipe <- function(recipe) { } } # Scorecards + if (!is.null(recipe$Analysis$Workflow$Statistics)) { + statistics <- strsplit(recipe$Analysis$Workflow$Statistics, + ", | |,")[[1]] + } else { + statistics <- NULL + } if ("Scorecards" %in% names(recipe$Analysis$Workflow)) { - if(recipe$Analysis$Workflow$Scorecards$execute == TRUE){ + if (recipe$Analysis$Workflow$Scorecards$execute == TRUE) { if (is.null(recipe$Analysis$Workflow$Scorecards$metric)) { error(recipe$Run$logger, "Parameter 'metric' must be defined under 'Scorecards'.") @@ -674,13 +680,21 @@ check_recipe <- function(recipe) { requested_metrics <- c(requested_metrics, 'crps_syear') } } - if ('enscorr' %in% tolower(sc_metrics)) { - recipe$Analysis$Workflow$Statistics <- c('std', 'cov', 'n_eff') + if ('enscorr' %in% tolower(sc_metrics) && + !all(c('std', 'cov', 'n_eff') %in% statistics)) { + error(recipe$Run$logger, + paste("For 'enscorr' to be plotted in the Scorecards with", + "the 'score' aggregation, the Statistics module must", + "be called and the statistics 'std', 'cov' and", + "'n_eff' are required.")) + error_status <- TRUE } recipe$Analysis$Workflow$Skill$metric <- paste0(requested_metrics, collapse = " ") } if (tolower(recipe$Analysis$Output_format) != 'scorecards') { + warn(recipe$Run$logger, + "Scorecards requested: setting output format as 'Scorecards'") recipe$Analysis$Output_format <- 'scorecards' } if (!all(tolower(sc_metrics) %in% tolower(requested_metrics))) { -- GitLab From 674310a6b9501a764e631b655fbaab0b4ad4c1fc Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 24 Apr 2024 12:15:35 +0200 Subject: [PATCH 3/8] Update Scorecards use case to call Statistics module for enscorr --- use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md | 5 ++++- use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml | 3 +++ use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md index c77fb2b1..3a50759b 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md @@ -67,7 +67,7 @@ Feel free to also modify other aspects according to your particular needs. You c ## 3. The user-defined script -We need to have a script to define the modules to use and the steps of the workflow. Note that the script is for data loading and verification parts. The Scorecards module doesn't need to be included in this script. +We need to have a script to define the modules to use and the steps of the workflow. Note that the script is for data loading and verification parts. The Scorecards module doesn't need to be included in this script. The Statistics module is called in order to compute some statistics that are needed to correctly to the ensemble correlation aggregation for the Scorecards. The prepare_outputs() function is already incorporated into the launcher script (see the next section for details about launcher), so it does not need to be included in the user-defined script in this case. In its place, we will use the function read_atomic_recipe(). The recipe path is passed as an argument onto the R script. The beginning of our script should look like this: @@ -77,6 +77,7 @@ In its place, we will use the function read_atomic_recipe(). The recipe path is source("modules/Loading/Loading.R") source("modules/Anomalies/Anomalies.R") source("modules/Skill/Skill.R") +source("modules/Statistics/Statistics.R") source("modules/Saving/Saving.R") # Read recipe @@ -95,6 +96,8 @@ data <- Loading(recipe) data <- Anomalies(recipe, data) # Compute skill metrics skill_metrics <- Skill(recipe, data) +# Compute statistics +statistics <- Statistics(recipe, data) ``` Check the example script at [ex1_2-script.yml](use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R). You can execute it as-is or copy it and modify it according to your needs. diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml index 73f16311..9973e4bf 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml @@ -48,6 +48,9 @@ Analysis: metric: mean_bias EnsCorr rps rpss crps crpss EnsSprErr # list, don't split cross_validation: yes save: 'all' + Statistics: + metric: cov std n_eff + save: 'all' Probabilities: percentiles: [[1/3, 2/3]] save: 'none' diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R index a823438c..a39c1c39 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R @@ -10,6 +10,7 @@ source("modules/Loading/Loading.R") source("modules/Anomalies/Anomalies.R") source("modules/Skill/Skill.R") +source("modules/Statistics/Statistics.R") source("modules/Saving/Saving.R") # Read recipe @@ -23,4 +24,5 @@ data <- Loading(recipe) data <- Anomalies(recipe, data) # Compute skill metrics skill_metrics <- Skill(recipe, data) - +# Compute statistics +statistics <- Statistics(recipe, data) -- GitLab From ced118fb7e3555366f61ed41cd97d8c542728b86 Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 24 Apr 2024 12:28:41 +0200 Subject: [PATCH 4/8] Bugfix: correct call to recipe variable --- tools/check_recipe.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 04193bd3..026735e4 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -649,7 +649,7 @@ check_recipe <- function(recipe) { } # Scorecards if (!is.null(recipe$Analysis$Workflow$Statistics)) { - statistics <- strsplit(recipe$Analysis$Workflow$Statistics, + statistics <- strsplit(recipe$Analysis$Workflow$Statistics$metric, ", | |,")[[1]] } else { statistics <- NULL -- GitLab From 4c8450345519229bdd4cd2bcea51c4cdc452b0a0 Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 24 Apr 2024 12:52:59 +0200 Subject: [PATCH 5/8] Formatting: fix indentation --- modules/Scorecards/execute_scorecards.R | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/Scorecards/execute_scorecards.R b/modules/Scorecards/execute_scorecards.R index b79a9ab5..8311c6e9 100644 --- a/modules/Scorecards/execute_scorecards.R +++ b/modules/Scorecards/execute_scorecards.R @@ -14,22 +14,22 @@ recipe$Run$output_dir <- output_dir datasets <- recipe$Analysis$Datasets ## TODO: Improve dependency system? for (variable in 1:length(recipe$Analysis$Variables)) { - scorecard_recipe <- recipe + scorecard_recipe <- recipe - scorecard_recipe$Analysis$Datasets$System <- - as.vector(unlist(recipe$Analysis$Datasets$System)) - ## Include multimodel in systems - if(isTRUE(scorecard_recipe$Analysis$Datasets$Multimodel$execute) || - scorecard_recipe$Analysis$Datasets$Multimodel$execute == 'both' || - scorecard_recipe$Analysis$Datasets$Multimodel$execute == 'yes'){ - scorecard_recipe$Analysis$Datasets$System <- - c(scorecard_recipe$Analysis$Datasets$System, 'Multimodel') - } - scorecard_recipe$Analysis$Datasets$Reference <- - as.vector(unlist(recipe$Analysis$Datasets$Reference)) - scorecard_recipe$Analysis$Variables <- - recipe$Analysis$Variables[[variable]] - # Plot Scorecards - Scorecards(scorecard_recipe) + scorecard_recipe$Analysis$Datasets$System <- + as.vector(unlist(recipe$Analysis$Datasets$System)) + ## Include multimodel in systems + if(isTRUE(scorecard_recipe$Analysis$Datasets$Multimodel$execute) || + scorecard_recipe$Analysis$Datasets$Multimodel$execute == 'both' || + scorecard_recipe$Analysis$Datasets$Multimodel$execute == 'yes'){ + scorecard_recipe$Analysis$Datasets$System <- + c(scorecard_recipe$Analysis$Datasets$System, 'Multimodel') + } + scorecard_recipe$Analysis$Datasets$Reference <- + as.vector(unlist(recipe$Analysis$Datasets$Reference)) + scorecard_recipe$Analysis$Variables <- + recipe$Analysis$Variables[[variable]] + # Plot Scorecards + Scorecards(scorecard_recipe) } print("##### SCORECARDS SAVED TO THE OUTPUT DIRECTORY #####") -- GitLab From 9e3350d7ece04688ef69db907376cacdc133ba67 Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 10 Apr 2024 16:37:47 +0200 Subject: [PATCH 6/8] Copy corrected recipe instead of original --- tools/prepare_outputs.R | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R index c571a3b8..6c77e5cf 100644 --- a/tools/prepare_outputs.R +++ b/tools/prepare_outputs.R @@ -52,9 +52,6 @@ prepare_outputs <- function(recipe_file, } dir.create(file.path(output_dir, folder_name, 'logs', 'recipes'), recursive = T, showWarnings = F) - # Copy recipe to output folder - file.copy(recipe$recipe_path, file.path(output_dir, folder_name, 'logs', - 'recipes')) # Create log output file logfile <- file.path(output_dir, folder_name, 'logs', 'main.log') file.create(logfile) @@ -118,5 +115,12 @@ prepare_outputs <- function(recipe_file, } else { recipe <- check_recipe(recipe) } + # Create a copy of the recipe and remove the logger + recipe_copy <- recipe + recipe_copy$Run$logger <- NULL + # Copy recipe to output folder + write_yaml(recipe_copy, + file = file.path(output_dir, folder_name, 'logs', 'recipes', + paste0(recipe$name, ".yml"))) return(recipe) } -- GitLab From a3767cab20dab1bd8ed6a5bb2b548431050b029f Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 25 Apr 2024 10:53:40 +0200 Subject: [PATCH 7/8] Correct recipe --- use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml index 9973e4bf..ee7befc5 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml @@ -9,7 +9,8 @@ Analysis: Datasets: System: # multiple systems for single model, split if Multimodel = F - {name: ECMWF-SEAS5} - Multimodel: False # single option + Multimodel: + execute: False # single option Reference: - {name: ERA5} Time: -- GitLab From a505acdf87b52a7b36a382564b787bcb362c26cb Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 25 Apr 2024 10:54:18 +0200 Subject: [PATCH 8/8] Formatting and logic improvements --- modules/Scorecards/execute_scorecards.R | 22 ++++++++++++---------- tools/check_recipe.R | 3 +-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/modules/Scorecards/execute_scorecards.R b/modules/Scorecards/execute_scorecards.R index 8311c6e9..c10def7a 100644 --- a/modules/Scorecards/execute_scorecards.R +++ b/modules/Scorecards/execute_scorecards.R @@ -12,24 +12,26 @@ recipe$Run$output_dir <- output_dir ## Loop over variables datasets <- recipe$Analysis$Datasets -## TODO: Improve dependency system? + for (variable in 1:length(recipe$Analysis$Variables)) { + # Create a copy of the recipe for this variable scorecard_recipe <- recipe - + # Collect all system names scorecard_recipe$Analysis$Datasets$System <- - as.vector(unlist(recipe$Analysis$Datasets$System)) - ## Include multimodel in systems - if(isTRUE(scorecard_recipe$Analysis$Datasets$Multimodel$execute) || - scorecard_recipe$Analysis$Datasets$Multimodel$execute == 'both' || - scorecard_recipe$Analysis$Datasets$Multimodel$execute == 'yes'){ - scorecard_recipe$Analysis$Datasets$System <- - c(scorecard_recipe$Analysis$Datasets$System, 'Multimodel') + list(name = as.vector(unlist(recipe$Analysis$Datasets$System))) + # Include multimodel in systems + if (!isFALSE(recipe$Analysis$Datasets$Multimodel$execute)) { + scorecard_recipe$Analysis$Datasets$System$name <- + c(scorecard_recipe$Analysis$Datasets$System$name, 'Multimodel') } + # Collect all reference names scorecard_recipe$Analysis$Datasets$Reference <- - as.vector(unlist(recipe$Analysis$Datasets$Reference)) + list(name = as.vector(unlist(recipe$Analysis$Datasets$Reference))) + # Assign variables scorecard_recipe$Analysis$Variables <- recipe$Analysis$Variables[[variable]] # Plot Scorecards Scorecards(scorecard_recipe) } + print("##### SCORECARDS SAVED TO THE OUTPUT DIRECTORY #####") diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 026735e4..2d2e73b5 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -98,8 +98,7 @@ check_recipe <- function(recipe) { } # Check multimodel if (is.null(recipe$Analysis$Datasets$Multimodel) || - (is.logical(recipe$Analysis$Datasets$Multimodel) && - !(recipe$Analysis$Datasets$Multimodel))) { + isFALSE(recipe$Analysis$Datasets$Multimodel)) { recipe$Analysis$Datasets$Multimodel <- list(execute = FALSE) } if (tolower(recipe$Analysis$Datasets$Multimodel$execute) == 'false') { -- GitLab