diff --git a/modules/Scorecards/execute_scorecards.R b/modules/Scorecards/execute_scorecards.R index e4dcd4eca1055b179ecdaf2b407fa63bdf23e330..c10def7abcbe329508fcee0705d4cd62095ff3d9 100644 --- a/modules/Scorecards/execute_scorecards.R +++ b/modules/Scorecards/execute_scorecards.R @@ -12,26 +12,26 @@ recipe$Run$output_dir <- output_dir ## Loop over variables datasets <- recipe$Analysis$Datasets -## TODO: Improve dependency system? -for (variable in 1:length(recipe$Analysis$Variables)) { - scorecard_recipe <- recipe - - scorecard_recipe$Analysis$Datasets$System <- - as.vector(unlist(recipe$Analysis$Datasets$System)) - - ## Include multimodel in systems - if(isTRUE(scorecard_recipe$Analysis$Datasets$Multimodel$execute) || - scorecard_recipe$Analysis$Datasets$Multimodel$execute == 'both' || - scorecard_recipe$Analysis$Datasets$Multimodel$execute == 'yes'){ - scorecard_recipe$Analysis$Datasets$System <- - c(scorecard_recipe$Analysis$Datasets$System, 'Multimodel') - } - scorecard_recipe$Analysis$Datasets$Reference <- - as.vector(unlist(recipe$Analysis$Datasets$Reference)) - scorecard_recipe$Analysis$Variables <- - recipe$Analysis$Variables[[variable]] - # Plot Scorecards - Scorecards(scorecard_recipe) +for (variable in 1:length(recipe$Analysis$Variables)) { + # Create a copy of the recipe for this variable + scorecard_recipe <- recipe + # Collect all system names + scorecard_recipe$Analysis$Datasets$System <- + list(name = as.vector(unlist(recipe$Analysis$Datasets$System))) + # Include multimodel in systems + if (!isFALSE(recipe$Analysis$Datasets$Multimodel$execute)) { + scorecard_recipe$Analysis$Datasets$System$name <- + c(scorecard_recipe$Analysis$Datasets$System$name, 'Multimodel') + } + # Collect all reference names + scorecard_recipe$Analysis$Datasets$Reference <- + list(name = as.vector(unlist(recipe$Analysis$Datasets$Reference))) + # Assign variables + scorecard_recipe$Analysis$Variables <- + recipe$Analysis$Variables[[variable]] + # Plot Scorecards + Scorecards(scorecard_recipe) } + print("##### SCORECARDS SAVED TO THE OUTPUT DIRECTORY #####") diff --git a/tools/check_recipe.R b/tools/check_recipe.R index c0d2f9cdf16ca622762e4ad0bae814ba9daaed93..be38ae7187dc4e1b5b2fb6ec0a2b189f82ae1503 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -98,8 +98,7 @@ check_recipe <- function(recipe) { } # Check multimodel if (is.null(recipe$Analysis$Datasets$Multimodel) || - (is.logical(recipe$Analysis$Datasets$Multimodel) && - !(recipe$Analysis$Datasets$Multimodel))) { + isFALSE(recipe$Analysis$Datasets$Multimodel)) { recipe$Analysis$Datasets$Multimodel <- list(execute = FALSE) } if (tolower(recipe$Analysis$Datasets$Multimodel$execute) == 'false') { @@ -648,8 +647,14 @@ check_recipe <- function(recipe) { } } # Scorecards + if (!is.null(recipe$Analysis$Workflow$Statistics)) { + statistics <- strsplit(recipe$Analysis$Workflow$Statistics$metric, + ", | |,")[[1]] + } else { + statistics <- NULL + } if ("Scorecards" %in% names(recipe$Analysis$Workflow)) { - if(recipe$Analysis$Workflow$Scorecards$execute == TRUE){ + if (recipe$Analysis$Workflow$Scorecards$execute == TRUE) { if (is.null(recipe$Analysis$Workflow$Scorecards$metric)) { error(recipe$Run$logger, "Parameter 'metric' must be defined under 'Scorecards'.") @@ -674,13 +679,21 @@ check_recipe <- function(recipe) { requested_metrics <- c(requested_metrics, 'crps_syear') } } - if ('enscorr' %in% tolower(sc_metrics)) { - recipe$Analysis$Workflow$Statistics <- c('std', 'cov', 'n_eff') + if ('enscorr' %in% tolower(sc_metrics) && + !all(c('std', 'cov', 'n_eff') %in% statistics)) { + error(recipe$Run$logger, + paste("For 'enscorr' to be plotted in the Scorecards with", + "the 'score' aggregation, the Statistics module must", + "be called and the statistics 'std', 'cov' and", + "'n_eff' are required.")) + error_status <- TRUE } recipe$Analysis$Workflow$Skill$metric <- paste0(requested_metrics, collapse = " ") } if (tolower(recipe$Analysis$Output_format) != 'scorecards') { + warn(recipe$Run$logger, + "Scorecards requested: setting output format as 'Scorecards'") recipe$Analysis$Output_format <- 'scorecards' } if (!all(tolower(sc_metrics) %in% tolower(requested_metrics))) { diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R index c571a3b8c4af280d6a045075a03edc4e13c353c6..6c77e5cf7220ba20f9cdb0fe26277714f8b7af0e 100644 --- a/tools/prepare_outputs.R +++ b/tools/prepare_outputs.R @@ -52,9 +52,6 @@ prepare_outputs <- function(recipe_file, } dir.create(file.path(output_dir, folder_name, 'logs', 'recipes'), recursive = T, showWarnings = F) - # Copy recipe to output folder - file.copy(recipe$recipe_path, file.path(output_dir, folder_name, 'logs', - 'recipes')) # Create log output file logfile <- file.path(output_dir, folder_name, 'logs', 'main.log') file.create(logfile) @@ -118,5 +115,12 @@ prepare_outputs <- function(recipe_file, } else { recipe <- check_recipe(recipe) } + # Create a copy of the recipe and remove the logger + recipe_copy <- recipe + recipe_copy$Run$logger <- NULL + # Copy recipe to output folder + write_yaml(recipe_copy, + file = file.path(output_dir, folder_name, 'logs', 'recipes', + paste0(recipe$name, ".yml"))) return(recipe) } diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md index b98b8d4678a3ffccd6e2f07ac14aeaa1bb29447b..aab5b87568374ad04f75477a42a2eedd4d71ddac 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md @@ -67,7 +67,7 @@ Feel free to also modify other aspects according to your particular needs. You c ## 3. The user-defined script -We need to have a script to define the modules to use and the steps of the workflow. Note that the script is for data loading and verification parts. The Scorecards module doesn't need to be included in this script. +We need to have a script to define the modules to use and the steps of the workflow. Note that the script is for data loading and verification parts. The Scorecards module doesn't need to be included in this script. The Statistics module is called in order to compute some statistics that are needed to correctly to the ensemble correlation aggregation for the Scorecards. The prepare_outputs() function is already incorporated into the launcher script (see the next section for details about launcher), so it does not need to be included in the user-defined script in this case. In its place, we will use the function read_atomic_recipe(). The recipe path is passed as an argument onto the R script. Below is an example of what the script should look like. @@ -81,6 +81,7 @@ The beginning of the user-defined script should look like this: source("modules/Loading/Loading.R") source("modules/Anomalies/Anomalies.R") source("modules/Skill/Skill.R") +source("modules/Statistics/Statistics.R") source("modules/Saving/Saving.R") # Read recipe @@ -99,6 +100,8 @@ data <- Loading(recipe) data <- Anomalies(recipe, data) # Compute skill metrics skill_metrics <- Skill(recipe, data) +# Compute statistics +statistics <- Statistics(recipe, data) ``` Check the example script at [ex1_2-script.yml](use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R). You can execute it as-is or copy it and modify it according to your needs. diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml index 8e1204bf380c20159a9b22e5e97e1872649fbe07..68a8dc627dbfdfffd3772aab55c2b1c80ba12881 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml @@ -9,7 +9,8 @@ Analysis: Datasets: System: # multiple systems for single model, split if Multimodel = F - {name: ECMWF-SEAS5} - Multimodel: False # single option + Multimodel: + execute: False # single option Reference: - {name: ERA5} Time: @@ -48,6 +49,9 @@ Analysis: metric: mean_bias EnsCorr rps rpss crps crpss EnsSprErr # list, don't split cross_validation: yes save: 'all' + Statistics: + metric: cov std n_eff + save: 'all' Probabilities: percentiles: [[1/3, 2/3]] save: 'none' diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R index a823438c8af000fb2d707ab120e7f7f1837be89d..a39c1c39800d195dac59466d3af4e06c9caf02e2 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R @@ -10,6 +10,7 @@ source("modules/Loading/Loading.R") source("modules/Anomalies/Anomalies.R") source("modules/Skill/Skill.R") +source("modules/Statistics/Statistics.R") source("modules/Saving/Saving.R") # Read recipe @@ -23,4 +24,5 @@ data <- Loading(recipe) data <- Anomalies(recipe, data) # Compute skill metrics skill_metrics <- Skill(recipe, data) - +# Compute statistics +statistics <- Statistics(recipe, data)