From 71d5f98fb2860b604c35c2553a037c8ab76121a5 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Wed, 23 Nov 2022 14:53:58 +0100
Subject: [PATCH 01/57] Add vitigeoss dict to .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index d95b20cb..ce85ad32 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 out-logs/
+conf/vitigeoss-vars-dict.yml
-- 
GitLab


From 1f80bdf61685b8eea370a9f953e284100eae1441 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Wed, 23 Nov 2022 17:15:19 +0100
Subject: [PATCH 02/57] First review of recipe checks (WIP)

---
 tools/check_recipe.R | 174 +++++++++++++++++++++++--------------------
 1 file changed, 93 insertions(+), 81 deletions(-)

diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index 25536335..a25c5f80 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -1,65 +1,70 @@
-check_recipe <- function(recipe, logger) {
+check_recipe <- function(recipe) {
 
   # recipe: yaml recipe already read it
   # output: errors or the total number of workflow (vars x regions) to compute
 
-  info(logger, paste("Checking recipe", recipe$filename))
+  info(recipe$Run$logger, paste("Checking recipe", recipe$name))
 
   # ---------------------------------------------------------------------
   # ANALYSIS CHECKS
   # ---------------------------------------------------------------------
 
-  TIME_SETTINGS = c('sdate','leadtimemin','leadtimemax','hcst_start','hcst_end')
-  PARAMS = c('Horizon','Time','Variables','Region','Regrid','Workflow','Datasets')
-  HORIZONS <- c('Subseasonal','Seasonal','Decadal')
+  TIME_SETTINGS = c('sdate', 'ftime_min', 'ftime_max', 'hcst_start', 'hcst_end')
+  PARAMS = c('Horizon', 'Time', 'Variables', 'Region', 'Regrid', 'Workflow',
+	     'Datasets')
+  HORIZONS <- c('subseasonal', 'seasonal', 'decadal')
 
-  # create output dirs:
-  if (!any(names(recipe) %in% "Analysis")) {
-    error(logger, "The recipe should contain an element called 'Analysis'.")
+  # Check basic elements in recipe:Analysis:
+  if (!("Analysis" %in% names(recipe))) {
+    error(recipe$Run$logger,
+	  "The recipe must contain an element called 'Analysis'.")
   }
 
   if (!all(PARAMS %in% names(recipe$Analysis))) {
     error(logger, 
-          paste("The element 'Analysis' in the recipe should contain these",
-         "elements:", paste(PARAMS, collapse = " ")))
+          paste("The element 'Analysis' in the recipe must contain these",
+          "elements:", paste(PARAMS, collapse = ", ")))
   }
 
-  if (!any(HORIZONS %in% recipe$Analysis$Horizon)) {
+  if (!any(HORIZONS %in% tolower(recipe$Analysis$Horizon))) {
     error(logger,
-         "The element 'Horizon' in the recipe should be one of the followings:",
-         paste(HORIZONS, collapse = " "))
+          "The element 'Horizon' in the recipe must be one of the following:",
+	  paste(HORIZONS, collapse = ", "))
   }
   # Check temporal settings and
   # count the number of verifications
   if (!all(TIME_SETTINGS %in% names(recipe$Analysis$Time))) {
-    error(logger, 
-       paste("The element 'Time' in the recipe should contain these elements:",
+    error(logger,
+       paste("The element 'Time' in the recipe must contain these elements:",
              paste(TIME_SETTINGS, collapse = " ")))
   }
-  if (is.null(recipe$Analysis$Time$sdate$fcst_year) ||
-    recipe$Analysis$Time$sdate$fcst_year == 'None') {
+  ## TODO: Is this needed?
+  if (is.null(recipe$Analysis$Time$fcst_year) || 
+      recipe$Analysis$Time$fcst_year == 'None') {
     stream <- "hindcast"
-    recipe$Analysis$Time$sdate$fcst_year <- 'YYYY'
+    # recipe$Analysis$Time$fcst_year <- 'YYYY'
   } else {
     stream <- "fcst"
   }
-  if (length(recipe$Analysis$Time$sdate$fcst_day) > 1 &&
-      tolower(recipe$Analysis$Horizon) != "subseasonal") {
+  ## TODO: To be implemented in the future
+  # if (length(recipe$Analysis$Time$sdate$fcst_day) > 1 &&
+  #     tolower(recipe$Analysis$Horizon) != "subseasonal") {
+  #   warn(logger,
+  #        paste("Only subseasonal verification allows multiple forecast days."),
+  #              "Element fcst_day in recipe set as 1.")
+  #   recipe$Analysis$Time$sdate$fcst_day <- '01'
+  # }
+  ## TODO: Delete, this parameter was deprecated
+  # if (is.null(recipe$Analysis$Time$sdate$fcst_sday)) {
+  #   error(logger,
+  #         paste("The element 'fcst_sday' in the recipe should be defined."))
+  # }
+  if (is.null(recipe$Analysis$Time$fcst_syear)) {
     warn(logger,
-         paste("Only subseasonal verification allows multiple forecast days."),
-               "Element fcst_day in recipe set as 1.")
-    recipe$Analysis$Time$sdate$fcst_day <- '01'
+         paste("The element 'fcst_year' is not defined in the recipe.
+	       No forecast year will be used."))
   }
-  if (is.null(recipe$Analysis$Time$sdate$fcst_sday)) {
-    error(logger,
-          paste("The element 'fcst_sday' in the recipe should be defined."))
-  }
-  if (is.null(recipe$Analysis$Time$sdate$fcst_syear)) {
-    error(logger,
-          paste("The element 'fcst_syear' in the recipe should be defined."))
-  }
-
-  
+  ## TODO: Move this inside 'if'? 
   fcst.sdate <- NULL
   for (syear in recipe$Analysis$Time$sdate$fcst_syear) {
       for (sday in recipe$Analysis$Time$sdate$fcst_sday) {
@@ -72,22 +77,23 @@ check_recipe <- function(recipe, logger) {
   # Regrid checks:
   if (length(recipe$Analysis$Regrid) != 2) {
     error(logger,
-          "The 'Regrid' element should specified the 'method' and 'type'.")
+          "The 'Regrid' element should specify the 'method' and 'type'.")
     stop("EXECUTION FAILED")    
   }
-# more checks
+  # TODO: Add Workflow checks?
   # ...
   # calculate number of workflows to create for each variable and 
   if (length(recipe$Analysis$Horizon) > 1) {
-    error(logger, "Only 1 Horizon can be specified in the recipe")
-    stop("EXECUTION FAILED")
-  }
-  nvar <- length(recipe$Analysis$Variables)
-  if (nvar > 2) {
-    error(logger,
-          "Only two type of Variables can be listed: ECVs and Indicators.")
+    error(logger, "Only one single Horizon can be specified in the recipe")
     stop("EXECUTION FAILED")
   }
+  ## TODO: Refine this
+  # nvar <- length(recipe$Analysis$Variables)
+  # if (nvar > 2) {
+  #   error(logger,
+  #         "Only two type of Variables can be listed: ECVs and Indicators.")
+  #   stop("EXECUTION FAILED")
+  # }
   # remove NULL or None Indicators or ECVs from the recipe:
   if (!is.null(recipe$Analysis$Variables$Indicators) &&
       !is.list(recipe$Analysis$Variables$Indicators)) {
@@ -100,41 +106,44 @@ check_recipe <- function(recipe, logger) {
            -which(names(recipe$Analysis$Variables) == 'ECVs')]
   }
   # Only one Calibration method allowed:
+  ## TODO: Make sure this condition is correctly written
   if ((is.logical(recipe$Analysis$Workflow$Calibration[[1]]) && 
-                 recipe$Analysis$Workflow$Calibration[[1]] == FALSE) ||
-                 recipe$Analysis$Workflow$Calibration[[1]] == 'None' ||
-                 is.null(recipe$Analysis$Workflow$Calibration[[1]])) {
+                  recipe$Analysis$Workflow$Calibration[[1]] == FALSE) ||
+                  recipe$Analysis$Workflow$Calibration[[1]] == 'None' ||
+                  is.null(recipe$Analysis$Workflow$Calibration[[1]])) {
     warn(logger, 
          "There is no Calibration method selected, raw data verification.")
-    recipe$Analysis$Workflow$Calibration[[1]] <- FALSE
+    recipe$Analysis$Workflow$Calibration[[1]] <- 'raw'
   } else {
     # remove multiple calibration methods
     if (is.null(names(recipe$Analysis$Workflow$Calibration))) {
       error(logger,
-          "The 'Calibration' element should specified at least the 'method'.")
+            "The 'Calibration' element 'method' must be specified.")
       stop("EXECUTION FAILED")
     }
   }  
 
   if ("Region" %in% names(recipe$Analysis)) {
-    nregions <- length(recipe$Analysis$Region$Regional)
     limits <- c('latmin', 'latmax', 'lonmin', 'lonmax')
-    for (i in 1:length(recipe$Analysis$Region)) {
-      if (!all(limits %in% names(recipe$Analysis$Region[[i]]))) {
-        limits <- paste(limits, collapse = " ")
-        error(logger,
-           paste("Each region defined in element 'Regional'",
-                 "should have 4 elements:",
-                 limits))
-        stop("EXECUTION FAILED")
-      }
-      # are numeric? class list mode list 
+    if (!all(limits %in% names(recipe$Analysis$Region))) {
+      error(recipe$Run$logger, paste("There must be 4 elements in 'Region':",
+				     limits))
     }
+    ## TODO: Implement multiple regions
+    # nregions <- length(recipe$Analysis$Region$Regional)
+    # for (i in 1:length(recipe$Analysis$Region)) {
+    #   if (!all(limits %in% names(recipe$Analysis$Region[[i]]))) {
+    #     limits <- paste(limits, collapse = " ")
+    #     error(logger,
+    #        paste("Each region defined in element 'Regional'",
+    #              "should have 4 elements:",
+    #              limits))
+    #     stop("EXECUTION FAILED")
+    #   }
+    #   # are numeric? class list mode list 
+    # }
   } else {
-    error(logger,
-           paste("'Region'",
-                 "should be defined",
-                 limits))
+    error(recipe$Run$logger, "'Region' must be defined.")
         stop("EXECUTION FAILED")
   }
   
@@ -142,39 +151,42 @@ check_recipe <- function(recipe, logger) {
   # RUN CHECKS
   # ---------------------------------------------------------------------
 
-  RUN_FIELDS = c("Loglevel","Terminal","output_dir","code_dir")
-  LOG_LEVELS = c("INFO","DEBUG","WARNING","ERROR")
+  RUN_FIELDS = c("Loglevel", "Terminal", "output_dir", "code_dir")
+  LOG_LEVELS = c("INFO", "DEBUG", "WARN", "ERROR", "FATAL")
 
-  if (!any(names(recipe) %in% "Run")) {
-    error(logger, "The recipe should contain an element called 'Run'.")
+  if (!("Run" %in% names(recipe))) {
+    stop("The recipe must contain an element named 'Run'.")
   }
   if (!all(RUN_FIELDS %in% names(recipe$Run))) {
-    error(logger, paste0("Run should contain the fields: ", 
-                         paste(RUN_FIELDS,collapse=", "), "."))
+    error(recipe$Run$logger, paste("Recipe element 'Run' must contain",
+				   "all of the following fields:",
+                                   paste(RUN_FIELDS, collapse=", "), "."))
   }
   if (!is.character(recipe$Run$output_dir)) {
-    error(logger,
-          paste("The Run element 'output_dir' in", recipe$filename,"file ",
+    error(recipe$Run$logger,
+          paste("The Run element 'output_dir' in", recipe$name, "file ",
                 "should be a character string indicating the path ",
-                "where to save the outputs."))
+                "where the outputs should be saved."))
   }
   if (!is.character(recipe$Run$code_dir)) {
-    error(logger,
-          paste("The Run element 'code_dir' in", recipe$filename,"file ",
+    error(recipe$Run$logger,
+          paste("The Run element 'code_dir' in", recipe$name, "file ",
                 "should be a character string indicating the path ",
                 "where the code is."))
   }
   if (!is.logical(recipe$Run$Terminal)) {
-    error(logger,
-          paste("The Run element 'Terminal' in", recipe$filename,"file ",
-                "should be a boolean value indicating wether to print or not the log",
-                "in the terminal."))
+    error(recipe$Run$logger,
+          paste("The Run element 'Terminal' in", recipe$filename, "file ",
+                "should be a boolean value indicating whether or not to",
+                "print the logs in the terminal."))
   }
-  if (!is.character(recipe$Run$Loglevel) || !any(recipe$Run$Loglevel %in% LOG_LEVELS)) {
+  ## TODO: Review this case, since default value is allowed
+  if (!is.character(recipe$Run$Loglevel) || 
+      !any(recipe$Run$Loglevel %in% LOG_LEVELS)) {
     error(logger,
-          paste("The Run element 'Loglevel' in", recipe$filename,"file ",
-                "should be a character string indicating one of the levels available: ",
-                paste0(LOG_LEVELS,collapse='/')))
+          paste("The Run element 'Loglevel' in", recipe$name, "file ",
+                "should be a character string specifying one of the levels available: ",
+                paste0(LOG_LEVELS, collapse='/')))
   }
 
   # ---------------------------------------------------------------------
-- 
GitLab


From 3680e1cbca0c88df9d880a8e55a9b3fe69bb72e2 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Fri, 25 Nov 2022 10:20:18 +0100
Subject: [PATCH 03/57] Update check_recipe()

---
 .../testing_recipes/wrong_recipe_example.yml  |  44 +++++++
 tools/check_recipe.R                          | 124 +++++++++++-------
 2 files changed, 117 insertions(+), 51 deletions(-)
 create mode 100644 modules/Loading/testing_recipes/wrong_recipe_example.yml

diff --git a/modules/Loading/testing_recipes/wrong_recipe_example.yml b/modules/Loading/testing_recipes/wrong_recipe_example.yml
new file mode 100644
index 00000000..e5d59ad8
--- /dev/null
+++ b/modules/Loading/testing_recipes/wrong_recipe_example.yml
@@ -0,0 +1,44 @@
+Description:
+  Author: V. Agudetse
+  Info: Incomplete recipe with incorrect fields to test the recipe checker.
+
+Analysis:
+  Horizon: Seasoning
+  Variables:
+    name: tas
+    freq: monthly_mean
+  Petaflops:
+    System:
+      name: system7c3s
+    Multimodel: False
+    Reference:
+      name: era5
+  Time:
+    sdate: '1101'
+    fcst_year: '2020'
+    hcst_start: '1993'
+    hcst_end: '2016'
+    ftime_max: 6
+  Region:
+    latmax: 10
+    lonmin: 0
+    lonmax: 20
+  Regrid:
+    method: bilinear
+    type: to_system
+  Workflow:
+    Calibration:
+      method:
+    Skill:
+      metric: RPS RPSS
+    Probabilities:
+      percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]]
+    Indicators:
+      index: no
+  ncores: 7
+  remove_NAs: yes
+  Output_format: S2S4E
+Run:
+  Loglevel: INFO
+  Terminal: yes
+  output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/
diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index a25c5f80..dbbfd665 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -3,7 +3,7 @@ check_recipe <- function(recipe) {
   # recipe: yaml recipe already read it
   # output: errors or the total number of workflow (vars x regions) to compute
 
-  info(recipe$Run$logger, paste("Checking recipe", recipe$name))
+  info(recipe$Run$logger, paste("Checking recipe:", recipe$name))
 
   # ---------------------------------------------------------------------
   # ANALYSIS CHECKS
@@ -13,30 +13,36 @@ check_recipe <- function(recipe) {
   PARAMS = c('Horizon', 'Time', 'Variables', 'Region', 'Regrid', 'Workflow',
 	     'Datasets')
   HORIZONS <- c('subseasonal', 'seasonal', 'decadal')
+  # Define error status variable
+  error_status <- F
 
   # Check basic elements in recipe:Analysis:
   if (!("Analysis" %in% names(recipe))) {
     error(recipe$Run$logger,
 	  "The recipe must contain an element called 'Analysis'.")
+    error_status <- T
   }
 
   if (!all(PARAMS %in% names(recipe$Analysis))) {
-    error(logger, 
-          paste("The element 'Analysis' in the recipe must contain these",
-          "elements:", paste(PARAMS, collapse = ", ")))
+    error(recipe$Run$logger, 
+          paste0("The element 'Analysis' in the recipe must contain all of ",
+          "the following: ", paste(PARAMS, collapse = ", "), "."))
+    error_status <- T
   }
 
   if (!any(HORIZONS %in% tolower(recipe$Analysis$Horizon))) {
-    error(logger,
-          "The element 'Horizon' in the recipe must be one of the following:",
-	  paste(HORIZONS, collapse = ", "))
+    error(recipe$Run$logger,
+          paste0("The element 'Horizon' in the recipe must be one of the ",
+		"following: ", paste(HORIZONS, collapse = ", "), "."))
+    error_status <- T
   }
   # Check temporal settings and
   # count the number of verifications
   if (!all(TIME_SETTINGS %in% names(recipe$Analysis$Time))) {
-    error(logger,
-       paste("The element 'Time' in the recipe must contain these elements:",
-             paste(TIME_SETTINGS, collapse = " ")))
+    error(recipe$Run$logger,
+	  paste0("The element 'Time' in the recipe must contain all of the ",
+		 "following: ", paste(TIME_SETTINGS, collapse = ", "), "."))
+    error_status <- T
   }
   ## TODO: Is this needed?
   if (is.null(recipe$Analysis$Time$fcst_year) || 
@@ -49,48 +55,49 @@ check_recipe <- function(recipe) {
   ## TODO: To be implemented in the future
   # if (length(recipe$Analysis$Time$sdate$fcst_day) > 1 &&
   #     tolower(recipe$Analysis$Horizon) != "subseasonal") {
-  #   warn(logger,
+  #   warn(recipe$Run$logger,
   #        paste("Only subseasonal verification allows multiple forecast days."),
   #              "Element fcst_day in recipe set as 1.")
   #   recipe$Analysis$Time$sdate$fcst_day <- '01'
   # }
   ## TODO: Delete, this parameter was deprecated
   # if (is.null(recipe$Analysis$Time$sdate$fcst_sday)) {
-  #   error(logger,
+  #   error(recipe$Run$logger,
   #         paste("The element 'fcst_sday' in the recipe should be defined."))
   # }
-  if (is.null(recipe$Analysis$Time$fcst_syear)) {
-    warn(logger,
-         paste("The element 'fcst_year' is not defined in the recipe.
-	       No forecast year will be used."))
+  if (is.null(recipe$Analysis$Time$fcst_year)) {
+    warn(recipe$Run$logger,
+         paste("The element 'fcst_year' is not defined in the recipe.",
+	       "No forecast year will be used."))
   }
-  ## TODO: Move this inside 'if'? 
+  ## TODO: Adapt and move this inside 'if'? 
   fcst.sdate <- NULL
-  for (syear in recipe$Analysis$Time$sdate$fcst_syear) {
-      for (sday in recipe$Analysis$Time$sdate$fcst_sday) {
-        fcst.sdate <- c(fcst.sdate, 
-                             paste0(syear,
-                               sprintf("%04d", as.numeric(sday))))
-      }
+  for (syear in recipe$Analysis$Time$fcst_year) {
+    for (sday in recipe$Analysis$Time$sdate) {
+      fcst.sdate <- c(fcst.sdate, 
+                      paste0(syear,
+                             sprintf("%04d", as.numeric(sday))))
+    }
   }
   fcst.sdate <- list(stream = stream, fcst.sdate = fcst.sdate)   
   # Regrid checks:
   if (length(recipe$Analysis$Regrid) != 2) {
-    error(logger,
-          "The 'Regrid' element should specify the 'method' and 'type'.")
-    stop("EXECUTION FAILED")    
+    error(recipe$Run$logger,
+          "The 'Regrid' element must specify the 'method' and 'type'.")
+    error_status <- T
   }
   # TODO: Add Workflow checks?
   # ...
   # calculate number of workflows to create for each variable and 
   if (length(recipe$Analysis$Horizon) > 1) {
-    error(logger, "Only one single Horizon can be specified in the recipe")
-    stop("EXECUTION FAILED")
+    error(recipe$Run$logger,
+	  "Only one single Horizon can be specified in the recipe")
+    error_status <- T
   }
   ## TODO: Refine this
   # nvar <- length(recipe$Analysis$Variables)
   # if (nvar > 2) {
-  #   error(logger,
+  #   error(recipe$Run$logger,
   #         "Only two type of Variables can be listed: ECVs and Indicators.")
   #   stop("EXECUTION FAILED")
   # }
@@ -107,34 +114,36 @@ check_recipe <- function(recipe) {
   }
   # Only one Calibration method allowed:
   ## TODO: Make sure this condition is correctly written
-  if ((is.logical(recipe$Analysis$Workflow$Calibration[[1]]) && 
-                  recipe$Analysis$Workflow$Calibration[[1]] == FALSE) ||
-                  recipe$Analysis$Workflow$Calibration[[1]] == 'None' ||
-                  is.null(recipe$Analysis$Workflow$Calibration[[1]])) {
-    warn(logger, 
-         "There is no Calibration method selected, raw data verification.")
-    recipe$Analysis$Workflow$Calibration[[1]] <- 'raw'
+  if ((is.logical(recipe$Analysis$Workflow$Calibration$method) && 
+                  recipe$Analysis$Workflow$Calibration$method == FALSE) ||
+                  recipe$Analysis$Workflow$Calibration$method == 'None' ||
+                  is.null(recipe$Analysis$Workflow$Calibration$method)) {
+    warn(recipe$Run$logger,
+         "No Calibration method was specified, raw data verification.")
+    recipe$Analysis$Workflow$Calibration$method <- 'raw'
   } else {
     # remove multiple calibration methods
     if (is.null(names(recipe$Analysis$Workflow$Calibration))) {
-      error(logger,
+      error(recipe$Run$logger,
             "The 'Calibration' element 'method' must be specified.")
-      stop("EXECUTION FAILED")
+      error_status <- T
     }
-  }  
+  }
 
   if ("Region" %in% names(recipe$Analysis)) {
     limits <- c('latmin', 'latmax', 'lonmin', 'lonmax')
     if (!all(limits %in% names(recipe$Analysis$Region))) {
-      error(recipe$Run$logger, paste("There must be 4 elements in 'Region':",
-				     limits))
+      error(recipe$Run$logger,
+	    paste0("There must be 4 elements in 'Region': ",
+		   paste(limits, collapse = ", "), "."))
+      error_status <- T
     }
     ## TODO: Implement multiple regions
     # nregions <- length(recipe$Analysis$Region$Regional)
     # for (i in 1:length(recipe$Analysis$Region)) {
     #   if (!all(limits %in% names(recipe$Analysis$Region[[i]]))) {
     #     limits <- paste(limits, collapse = " ")
-    #     error(logger,
+    #     error(recipe$Run$logger,
     #        paste("Each region defined in element 'Regional'",
     #              "should have 4 elements:",
     #              limits))
@@ -144,7 +153,7 @@ check_recipe <- function(recipe) {
     # }
   } else {
     error(recipe$Run$logger, "'Region' must be defined.")
-        stop("EXECUTION FAILED")
+    error_status <- T
   }
   
   # ---------------------------------------------------------------------
@@ -161,24 +170,28 @@ check_recipe <- function(recipe) {
     error(recipe$Run$logger, paste("Recipe element 'Run' must contain",
 				   "all of the following fields:",
                                    paste(RUN_FIELDS, collapse=", "), "."))
+    error_status <- T
   }
   if (!is.character(recipe$Run$output_dir)) {
     error(recipe$Run$logger,
-          paste("The Run element 'output_dir' in", recipe$name, "file ",
-                "should be a character string indicating the path ",
-                "where the outputs should be saved."))
+          paste("The Run element 'output_dir' in", recipe$name, "file",
+                "should be a character string indicating the path where",
+                "the outputs should be saved."))
+    error_status <- T
   }
   if (!is.character(recipe$Run$code_dir)) {
     error(recipe$Run$logger,
           paste("The Run element 'code_dir' in", recipe$name, "file ",
                 "should be a character string indicating the path ",
                 "where the code is."))
+    error_status <- T
   }
   if (!is.logical(recipe$Run$Terminal)) {
     error(recipe$Run$logger,
-          paste("The Run element 'Terminal' in", recipe$filename, "file ",
+          paste("The Run element 'Terminal' in", recipe$name, "file ",
                 "should be a boolean value indicating whether or not to",
                 "print the logs in the terminal."))
+    error_status <- T
   }
   ## TODO: Review this case, since default value is allowed
   if (!is.character(recipe$Run$Loglevel) || 
@@ -187,6 +200,7 @@ check_recipe <- function(recipe) {
           paste("The Run element 'Loglevel' in", recipe$name, "file ",
                 "should be a character string specifying one of the levels available: ",
                 paste0(LOG_LEVELS, collapse='/')))
+    error_status <- T
   }
 
   # ---------------------------------------------------------------------
@@ -194,10 +208,18 @@ check_recipe <- function(recipe) {
   # ---------------------------------------------------------------------
   # Check workflow: need to define restrictions? 
   # e.g. only one calibration method
-  nverifications <- check_number_of_dependent_verifications(recipe)
-  info(logger, paste("Start Dates", paste(fcst.sdate, collapse = " ")))
-  info(logger, "Recipe checked succsessfully.")
-  return(append(nverifications, fcst.sdate))
+  ## TODO: Implement numbr of dependent verifications
+  #nverifications <- check_number_of_dependent_verifications(recipe)
+  info(recipe$Run$logger, paste("Start Dates:",
+				paste(fcst.sdate, collapse = " ")))
+
+  # Return error if any check has failed
+  if (error_status) {
+    stop("Recipe check failed. Please check the logs for a list of errors.")
+  } else {
+    info(recipe$Run$logger, "##### RECIPE CHECKS WERE SUCCESSFULL #####.")
+    # return(append(nverifications, fcst.sdate))
+  }
 }
 
 check_number_of_dependent_verifications <- function(recipe) {
-- 
GitLab


From cae758b0e71e818fc27f5b76acda85f5f01accc1 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Mon, 12 Dec 2022 15:25:19 +0100
Subject: [PATCH 04/57] Add example of main recipe

---
 recipes/recipe_example.yml | 58 ++++++++++++++++++++++++++++++++++++++
 tools/divide_recipe.R      |  6 ++--
 2 files changed, 62 insertions(+), 2 deletions(-)
 create mode 100644 recipes/recipe_example.yml

diff --git a/recipes/recipe_example.yml b/recipes/recipe_example.yml
new file mode 100644
index 00000000..27f74df9
--- /dev/null
+++ b/recipes/recipe_example.yml
@@ -0,0 +1,58 @@
+################################################################################ 
+## RECIPE DESCRIPTION  
+################################################################################ 
+
+Description:
+  Author: V. Agudetse
+  Info: Test for main recipe
+
+################################################################################
+## ANALYSIS CONFIGURATION
+################################################################################
+
+Analysis:
+  Horizon: Seasonal
+  Variables: # ECVs and Indicators?
+    - {name: tas, freq: monthly_mean}
+    - {name: prlr, freq: monthly_mean}
+  Datasets:
+    System: # multiple systems for single model, split if Multimodel = F
+      - {name: system7c3s}
+      - {name: system5c3s}
+    Multimodel: False # single option
+    Reference:
+      name: era5 # multiple references for single model?
+  Time:
+    sdate: '1101' # list, split
+    fcst_year: '2020' # list, don't split, handled internally
+    hcst_start: '1993' # single option
+    hcst_end: '2016' # single option
+    ftime_min: 1 # single option
+    ftime_max: 6 # singl eoption
+  Region: # multiple lists, split? Add region name if length(Region) > 1
+    - {name: "global", latmin: -90, latmax: 90, lonmin: 0, lonmax: 360}
+    - {name: "nino34", latmin: -5, latmax: 5, lonmin: -10, lonmax: 60}
+  Regrid:
+    method: bilinear # allow multiple methods?
+    type: to_system
+  Workflow:
+    Calibration:
+      method: mse_min # list, split?
+    Skill:
+      metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS # list, don't split
+    Probabilities:
+      percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]] # list, don't split
+    Indicators:
+      index: no # ? 
+  ncores: 7
+  remove_NAs: yes # bool, don't split
+  Output_format: S2S4E # string, don't split
+
+################################################################################
+## Run CONFIGURATION
+################################################################################
+Run:
+  Loglevel: INFO
+  Terminal: yes
+  output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/
+  code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/
diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R
index dafc8704..8f4ef35a 100644
--- a/tools/divide_recipe.R
+++ b/tools/divide_recipe.R
@@ -2,6 +2,8 @@
 # verifications: the output from check_recipe
 # folder: the name of the output folder for this run
 # logger: the log file obtain from prepare_outputs
+
+## TODO: Rethink strategy and adapt function accordingly
 divide_recipe <- function(recipe, verifications, folder, logger) {
   info(logger, "Spliting recipe in single verifications.")
   beta_recipe <- list(Description = append(recipe$Description, 
@@ -14,8 +16,8 @@ divide_recipe <- function(recipe, verifications, folder, logger) {
                                       Regrid = recipe$Analysis$Regrid,
                                       Workflow = recipe$Analysis$Workflow,
                                       Output_format = 
-                                              recipe$Analysis$Output_format),
-                       Run = recipe$Run)
+				        recipe$Analysis$Output_format),
+		      Run = recipe$Run)
   # duplicate recipe by Variables considering dep and indep:
   all_recipes <- list(beta_recipe)
   i <- 1 # to get track of the recipe number
-- 
GitLab


From fbb329af5930396261b8f8e4e28ff3cca2ddd106 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 13 Dec 2022 11:43:21 +0100
Subject: [PATCH 05/57] Formatting

---
 tools/check_recipe.R |  2 +-
 tools/data_summary.R | 11 +++++------
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index dbbfd665..316b26f7 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -26,7 +26,7 @@ check_recipe <- function(recipe) {
   if (!all(PARAMS %in% names(recipe$Analysis))) {
     error(recipe$Run$logger, 
           paste0("The element 'Analysis' in the recipe must contain all of ",
-          "the following: ", paste(PARAMS, collapse = ", "), "."))
+		 "the following: ", paste(PARAMS, collapse = ", "), "."))
     error_status <- T
   }
 
diff --git a/tools/data_summary.R b/tools/data_summary.R
index 34b6bd6e..cddd4f9e 100644
--- a/tools/data_summary.R
+++ b/tools/data_summary.R
@@ -8,15 +8,15 @@ data_summary <- function(data_cube, recipe) {
   # Get name, leadtime months and date range
   object_name <- deparse(substitute(data_cube))
   if (recipe$Analysis$Variables$freq == "monthly_mean") {
-    date_format <- '%b %Y'
+    date_format <- "%b %Y"
   } else if (recipe$Analysis$Variables$freq == "daily_mean") {
-    date_format <- '%b %d %Y'
+    date_format <- "%b %d %Y"
   }
-  months <- unique(format(as.Date(data_cube$Dates[[1]]), format = '%B'))
-  months <- paste(as.character(months), collapse=", ")
+  months <- unique(format(as.Date(data_cube$Dates[[1]]), format = "%B"))
+  months <- paste(as.character(months), collapse = ", ")
   sdate_min <- format(min(as.Date(data_cube$Dates[[1]])), format = date_format)
   sdate_max <- format(max(as.Date(data_cube$Dates[[1]])), format = date_format)
-  
+
   # Create log instance and sink output to logfile and terminal
   info(recipe$Run$logger, "DATA SUMMARY:")
   sink(recipe$Run$logfile, append = TRUE, split = TRUE)
@@ -29,4 +29,3 @@ data_summary <- function(data_cube, recipe) {
   print("---------------------------------------------")
   sink()
 }
-  
-- 
GitLab


From 8da9afb6c3de5ba472b7aaf22f69c70a4d7662d3 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 15 Dec 2022 15:09:52 +0100
Subject: [PATCH 06/57] Update wrong example and recipe checker

---
 .../testing_recipes/wrong_recipe_example.yml  |  2 +-
 tools/check_recipe.R                          | 28 +++++++++++--------
 2 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/modules/Loading/testing_recipes/wrong_recipe_example.yml b/modules/Loading/testing_recipes/wrong_recipe_example.yml
index e5d59ad8..12e2fc06 100644
--- a/modules/Loading/testing_recipes/wrong_recipe_example.yml
+++ b/modules/Loading/testing_recipes/wrong_recipe_example.yml
@@ -15,7 +15,7 @@ Analysis:
       name: era5
   Time:
     sdate: '1101'
-    fcst_year: '2020'
+    fcst_syear: '2020'
     hcst_start: '1993'
     hcst_end: '2016'
     ftime_max: 6
diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index 316b26f7..8f945b6c 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -139,16 +139,21 @@ check_recipe <- function(recipe) {
       error_status <- T
     }
     ## TODO: Implement multiple regions
-    # nregions <- length(recipe$Analysis$Region$Regional)
+    # nregions <- length(recipe$Analysis$Region)
     # for (i in 1:length(recipe$Analysis$Region)) {
     #   if (!all(limits %in% names(recipe$Analysis$Region[[i]]))) {
     #     limits <- paste(limits, collapse = " ")
     #     error(recipe$Run$logger,
-    #        paste("Each region defined in element 'Regional'",
-    #              "should have 4 elements:",
-    #              limits))
-    #     stop("EXECUTION FAILED")
+    #           paste0("Each region defined in element 'Region' ",
+    #                  "should have 4 elements: ",
+    #                  paste(limits, collapse = ", "), "."))
+    #     error_status <- T
     #   }
+    #   if (length(recipe$Analysis$Region) > 1) {
+    #     if (!("name" %in% names(recipe$Analysis$Region[[i]]))) {
+    #       error(recipe$Run$logger,
+    #             paste("If multiple regions are requested, each region must",
+    #                   "have a 'name'".)
     #   # are numeric? class list mode list 
     # }
   } else {
@@ -182,7 +187,7 @@ check_recipe <- function(recipe) {
   if (!is.character(recipe$Run$code_dir)) {
     error(recipe$Run$logger,
           paste("The Run element 'code_dir' in", recipe$name, "file ",
-                "should be a character string indicating the path ",
+                "should be a character string indicating the path",
                 "where the code is."))
     error_status <- T
   }
@@ -197,8 +202,8 @@ check_recipe <- function(recipe) {
   if (!is.character(recipe$Run$Loglevel) || 
       !any(recipe$Run$Loglevel %in% LOG_LEVELS)) {
     error(logger,
-          paste("The Run element 'Loglevel' in", recipe$name, "file ",
-                "should be a character string specifying one of the levels available: ",
+          paste("The Run element 'Loglevel' in", recipe$name, "file",
+                "should be a character string specifying one of the levels available:",
                 paste0(LOG_LEVELS, collapse='/')))
     error_status <- T
   }
@@ -208,16 +213,17 @@ check_recipe <- function(recipe) {
   # ---------------------------------------------------------------------
   # Check workflow: need to define restrictions? 
   # e.g. only one calibration method
-  ## TODO: Implement numbr of dependent verifications
+  ## TODO: Implement number of dependent verifications
   #nverifications <- check_number_of_dependent_verifications(recipe)
   info(recipe$Run$logger, paste("Start Dates:",
 				paste(fcst.sdate, collapse = " ")))
 
   # Return error if any check has failed
   if (error_status) {
-    stop("Recipe check failed. Please check the logs for a list of errors.")
+    error(recipe$Run$logger, "RECIPE CHECK FAILED.")
+    stop("The recipe contains some errors. The full list is in the logs.")
   } else {
-    info(recipe$Run$logger, "##### RECIPE CHECKS WERE SUCCESSFULL #####.")
+    info(recipe$Run$logger, "##### RECIPE CHECK SUCCESSFULL #####")
     # return(append(nverifications, fcst.sdate))
   }
 }
-- 
GitLab


From d3193f9aaf3a4dba672b277f36befd40ce6e022e Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 20 Dec 2022 16:07:51 +0100
Subject: [PATCH 07/57] update checker

---
 tools/check_recipe.R | 104 ++++++++++++++++++++++++++++++-------------
 1 file changed, 73 insertions(+), 31 deletions(-)

diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index 8f945b6c..a70a5b90 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -46,12 +46,13 @@ check_recipe <- function(recipe) {
   }
   ## TODO: Is this needed?
   if (is.null(recipe$Analysis$Time$fcst_year) || 
-      recipe$Analysis$Time$fcst_year == 'None') {
+      tolower(recipe$Analysis$Time$fcst_year) == 'none') {
     stream <- "hindcast"
     # recipe$Analysis$Time$fcst_year <- 'YYYY'
   } else {
     stream <- "fcst"
   }
+
   ## TODO: To be implemented in the future
   # if (length(recipe$Analysis$Time$sdate$fcst_day) > 1 &&
   #     tolower(recipe$Analysis$Horizon) != "subseasonal") {
@@ -65,6 +66,7 @@ check_recipe <- function(recipe) {
   #   error(recipe$Run$logger,
   #         paste("The element 'fcst_sday' in the recipe should be defined."))
   # }
+
   if (is.null(recipe$Analysis$Time$fcst_year)) {
     warn(recipe$Run$logger,
          paste("The element 'fcst_year' is not defined in the recipe.",
@@ -88,12 +90,13 @@ check_recipe <- function(recipe) {
   }
   # TODO: Add Workflow checks?
   # ...
-  # calculate number of workflows to create for each variable and 
+  # calculate number of workflows to create for each variable and
   if (length(recipe$Analysis$Horizon) > 1) {
     error(recipe$Run$logger,
 	  "Only one single Horizon can be specified in the recipe")
     error_status <- T
   }
+
   ## TODO: Refine this
   # nvar <- length(recipe$Analysis$Variables)
   # if (nvar > 2) {
@@ -101,6 +104,7 @@ check_recipe <- function(recipe) {
   #         "Only two type of Variables can be listed: ECVs and Indicators.")
   #   stop("EXECUTION FAILED")
   # }
+
   # remove NULL or None Indicators or ECVs from the recipe:
   if (!is.null(recipe$Analysis$Variables$Indicators) &&
       !is.list(recipe$Analysis$Variables$Indicators)) {
@@ -112,32 +116,15 @@ check_recipe <- function(recipe) {
     recipe$Analysis$Variables <- recipe$Analysis$Variables[
            -which(names(recipe$Analysis$Variables) == 'ECVs')]
   }
-  # Only one Calibration method allowed:
-  ## TODO: Make sure this condition is correctly written
-  if ((is.logical(recipe$Analysis$Workflow$Calibration$method) && 
-                  recipe$Analysis$Workflow$Calibration$method == FALSE) ||
-                  recipe$Analysis$Workflow$Calibration$method == 'None' ||
-                  is.null(recipe$Analysis$Workflow$Calibration$method)) {
-    warn(recipe$Run$logger,
-         "No Calibration method was specified, raw data verification.")
-    recipe$Analysis$Workflow$Calibration$method <- 'raw'
-  } else {
-    # remove multiple calibration methods
-    if (is.null(names(recipe$Analysis$Workflow$Calibration))) {
-      error(recipe$Run$logger,
-            "The 'Calibration' element 'method' must be specified.")
-      error_status <- T
-    }
-  }
 
-  if ("Region" %in% names(recipe$Analysis)) {
-    limits <- c('latmin', 'latmax', 'lonmin', 'lonmax')
-    if (!all(limits %in% names(recipe$Analysis$Region))) {
-      error(recipe$Run$logger,
-	    paste0("There must be 4 elements in 'Region': ",
-		   paste(limits, collapse = ", "), "."))
-      error_status <- T
-    }
+  # Region checks:
+  LIMITS <- c('latmin', 'latmax', 'lonmin', 'lonmax')
+  if (!all(LIMITS %in% names(recipe$Analysis$Region))) {
+    error(recipe$Run$logger,
+          paste0("There must be 4 elements in 'Region': ",
+		 paste(LIMITS, collapse = ", "), "."))
+    error_status <- T
+  }
     ## TODO: Implement multiple regions
     # nregions <- length(recipe$Analysis$Region)
     # for (i in 1:length(recipe$Analysis$Region)) {
@@ -156,11 +143,66 @@ check_recipe <- function(recipe) {
     #                   "have a 'name'".)
     #   # are numeric? class list mode list 
     # }
+  # ---------------------------------------------------------------------
+  # WORKFLOW CHECKS
+  # ---------------------------------------------------------------------
+
+  # Only one Calibration method allowed:
+  if ((is.logical(recipe$Analysis$Workflow$Calibration$method) &&
+       recipe$Analysis$Workflow$Calibration$method == FALSE) ||
+      tolower(recipe$Analysis$Workflow$Calibration$method) == 'none' ||
+      is.null(recipe$Analysis$Workflow$Calibration$method)) {
+    warn(recipe$Run$logger,
+         "No Calibration method was specified, raw data verification.")
+    recipe$Analysis$Workflow$Calibration$method <- 'raw'
   } else {
-    error(recipe$Run$logger, "'Region' must be defined.")
+    if (is.null(recipe$Analysis$Workflow$Calibration$method)) {
+      error(recipe$Run$logger,
+            "The 'Calibration' element 'method' must be specified.")
+      error_status <- T
+    }
+  }
+  # Anomalies
+  if ("Anomalies" %in% names(recipe$Analysis$Workflow)) {
+    if (!is.null(recipe$Analysis$Workflow$Anomalies$compute)) {
+      error(recipe$Run$logger,
+	    "Parameter 'compute' must be defined under 'Anomalies'.")
+      error_status <- T
+    } else if (!(is.logical(recipe$Analysis$Workflows$Anomalies$compute))) {
+      error(recipe$Run$logger,
+	    paste("Parameter 'Anomalies:compute' must be a logical value",
+		  "(True/False or yes/no)."))
+      error_status <- T
+    } else if ((recipe$Analysis$Workflows$Anomalies$compute) &&
+	       (!is.logical(recipe$Analysis$Workflow$Anomalies))) {
+      error(recipe$Run$logger,
+	    paste("If anomaly computation is requested, parameter",
+		  "'cross_validation' must be defined under 'Anomalies',
+		  and it must be a logical value (True/False or yes/no)."))
+      error_status <- T
+    }
+  }
+  # Skill
+  if (("Skill" %in% names(recipe$Analysis$Workflow)) &&
+      (is.null(recipe$Analysis$Workflow$Skill$metric))) {
+    error(recipe$Run$logger,
+	  "Parameter 'metric' must be defined under 'Skill'.")
     error_status <- T
   }
-  
+  # Probabilities
+  if ("Probabilities" %in% names(recipe$Analysis$Workflow)) {
+    if (is.null(recipe$Analysis$Workflow$Probabilities$percentiles)) {
+      error(recipe$Run$logger,
+	    "Parameter 'percentiles' must be defined under 'Skill'.")
+      error_status <- T
+    } else if (!is.list(recipe$Analysis$Workflow$Probabilities$percentiles)) {
+      error(recipe$Run$logger,
+	    paste("Parameter 'Probabilities:percentiles' expects a list.",
+		  "See documentation in the wiki for examples."))
+      error_status <- T
+    }
+  }
+
   # ---------------------------------------------------------------------
   # RUN CHECKS
   # ---------------------------------------------------------------------
@@ -215,8 +257,8 @@ check_recipe <- function(recipe) {
   # e.g. only one calibration method
   ## TODO: Implement number of dependent verifications
   #nverifications <- check_number_of_dependent_verifications(recipe)
-  info(recipe$Run$logger, paste("Start Dates:",
-				paste(fcst.sdate, collapse = " ")))
+  # info(recipe$Run$logger, paste("Start Dates:",
+  #  				  paste(fcst.sdate, collapse = " ")))
 
   # Return error if any check has failed
   if (error_status) {
-- 
GitLab


From a20f175b79fe2910ce25851d4aa7276b537fc087 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 20 Dec 2022 16:08:15 +0100
Subject: [PATCH 08/57] Add recipe checker to prepare_outputs

---
 modules/test_seasonal.R |  2 +-
 tools/prepare_outputs.R | 22 ++++++++++++----------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/modules/test_seasonal.R b/modules/test_seasonal.R
index 854e293f..c74fd1b1 100644
--- a/modules/test_seasonal.R
+++ b/modules/test_seasonal.R
@@ -6,7 +6,7 @@ source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
 recipe_file <- "modules/Loading/testing_recipes/recipe_seasonal-tests.yml"
-recipe <- prepare_outputs(recipe_file)
+recipe <- prepare_outputs(recipe_file, enable_checks = F)
 ## archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 
 # Load datasets
diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R
index da5d2562..d6eb970e 100644
--- a/tools/prepare_outputs.R
+++ b/tools/prepare_outputs.R
@@ -20,10 +20,11 @@
 #'
 #'@export
 
-prepare_outputs <- function(recipe_file) {
+prepare_outputs <- function(recipe_file,
+			    enable_checks = TRUE) {
 
-# recipe: the content of the readed recipe
-# file: the recipe file name
+# recipe_file: path to recipe YAML file
+# enable_checker: Whether or not to run the recipe checker
 
   recipe <- read_yaml(recipe_file)
   recipe$recipe_path <- recipe_file
@@ -33,32 +34,30 @@ prepare_outputs <- function(recipe_file) {
   # Create output folders:
   folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name)), "_",
                         gsub(" ", "", gsub(":", "", gsub("-", "", Sys.time()))))
-
   print("Saving all outputs to:")
   print(output_dir)
   print(folder_name)
-
   dir.create(file.path(output_dir, folder_name, 'outputs'), recursive = TRUE)
   dir.create(file.path(output_dir, folder_name, 'logs'))
   dir.create(file.path(output_dir, folder_name, 'logs', 'recipes'))
-
+  # Copy recipe to output folder
   file.copy(recipe$recipe_path, file.path(output_dir, folder_name, 'logs',
 				          'recipes'))
-
+  # Create log output file
   logfile <- file.path(output_dir, folder_name, 'logs', 'log.txt')
   file.create(logfile)
 
-  # Set default behaviour of log output file:
+  # Set default behaviour of logger
   if (is.null(recipe$Run)) {
     recipe$Run <- list(Loglevel = 'INFO', Terminal = TRUE)
   }
   if (is.null(recipe$Run$Loglevel)) {
     recipe$Run$Loglevel <- 'INFO'
   }
-
   if (!is.logical(recipe$Run$Terminal)) {
     recipe$Run$Terminal <- TRUE
   }
+  # logger set-up
   if (recipe$Run$Terminal) {
     logger <- log4r::logger(threshold = recipe$Run$Loglevel,  
                             appenders = list(console_appender(layout = default_log_layout()),
@@ -69,10 +68,13 @@ prepare_outputs <- function(recipe_file) {
                             appenders = list(file_appende(logfile, append = TRUE,             
                                              layout = default_log_layout())))
   }
-
   recipe$Run$output_dir <- file.path(output_dir, folder_name)
   recipe$Run$logger <- logger
   recipe$Run$logfile <- logfile
+  # Run recipe checker
+  if (enable_checks) {
+    check_recipe(recipe)
+  }
 
   return(recipe)
 }
-- 
GitLab


From ba419683e13cc1d14217b0116edabbb22bf317e2 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 20 Dec 2022 16:18:58 +0100
Subject: [PATCH 09/57] Print full path to recipe; fix bug in Anomalies check

---
 tools/check_recipe.R | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index a70a5b90..c17a7cd9 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -3,7 +3,7 @@ check_recipe <- function(recipe) {
   # recipe: yaml recipe already read it
   # output: errors or the total number of workflow (vars x regions) to compute
 
-  info(recipe$Run$logger, paste("Checking recipe:", recipe$name))
+  info(recipe$Run$logger, paste("Checking recipe:", recipe$path))
 
   # ---------------------------------------------------------------------
   # ANALYSIS CHECKS
@@ -164,7 +164,7 @@ check_recipe <- function(recipe) {
   }
   # Anomalies
   if ("Anomalies" %in% names(recipe$Analysis$Workflow)) {
-    if (!is.null(recipe$Analysis$Workflow$Anomalies$compute)) {
+    if (is.null(recipe$Analysis$Workflow$Anomalies$compute)) {
       error(recipe$Run$logger,
 	    "Parameter 'compute' must be defined under 'Anomalies'.")
       error_status <- T
-- 
GitLab


From 89ed146257cbded2fb8e7cc52cad5c54a0bc07d5 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 20 Dec 2022 16:19:48 +0100
Subject: [PATCH 10/57] Add TODO

---
 tools/check_recipe.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index c17a7cd9..2381309b 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -1,7 +1,7 @@
 check_recipe <- function(recipe) {
 
   # recipe: yaml recipe already read it
-  # output: errors or the total number of workflow (vars x regions) to compute
+  ## TODO: Adapt to decadal case
 
   info(recipe$Run$logger, paste("Checking recipe:", recipe$path))
 
-- 
GitLab


From 508683900950925f1bb894a447f49d2cdc34e841 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 20 Dec 2022 16:22:02 +0100
Subject: [PATCH 11/57] Fix bug caused by typo

---
 tools/check_recipe.R | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index 2381309b..153c4a18 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -168,12 +168,12 @@ check_recipe <- function(recipe) {
       error(recipe$Run$logger,
 	    "Parameter 'compute' must be defined under 'Anomalies'.")
       error_status <- T
-    } else if (!(is.logical(recipe$Analysis$Workflows$Anomalies$compute))) {
+    } else if (!(is.logical(recipe$Analysis$Workflow$Anomalies$compute))) {
       error(recipe$Run$logger,
 	    paste("Parameter 'Anomalies:compute' must be a logical value",
 		  "(True/False or yes/no)."))
       error_status <- T
-    } else if ((recipe$Analysis$Workflows$Anomalies$compute) &&
+    } else if ((recipe$Analysis$Workflow$Anomalies$compute) &&
 	       (!is.logical(recipe$Analysis$Workflow$Anomalies))) {
       error(recipe$Run$logger,
 	    paste("If anomaly computation is requested, parameter",
-- 
GitLab


From eb36caeeb545ce7811d2f8d2fb239f59b6f71a0a Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 20 Dec 2022 16:35:37 +0100
Subject: [PATCH 12/57] Fix pipeline

---
 tools/check_recipe.R | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index 153c4a18..fcd30ef3 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -3,16 +3,18 @@ check_recipe <- function(recipe) {
   # recipe: yaml recipe already read it
   ## TODO: Adapt to decadal case
 
-  info(recipe$Run$logger, paste("Checking recipe:", recipe$path))
+  info(recipe$Run$logger, paste("Checking recipe:", recipe$recipe_path))
 
   # ---------------------------------------------------------------------
   # ANALYSIS CHECKS
   # ---------------------------------------------------------------------
 
-  TIME_SETTINGS = c('sdate', 'ftime_min', 'ftime_max', 'hcst_start', 'hcst_end')
-  PARAMS = c('Horizon', 'Time', 'Variables', 'Region', 'Regrid', 'Workflow',
-	     'Datasets')
-  HORIZONS <- c('subseasonal', 'seasonal', 'decadal')
+  TIME_SETTINGS_SEASONAL <- c("sdate", "ftime_min", "ftime_max", "hcst_start",
+			      "hcst_end")
+  TIME_SETTINGS_DECADAL <- c("ftime_min", "ftime_max", "hcst_start", "hcst_end") 
+  PARAMS <- c("Horizon", "Time", "Variables", "Region", "Regrid", "Workflow",
+	      "Datasets")
+  HORIZONS <- c("subseasonal", "seasonal", "decadal")
   # Define error status variable
   error_status <- F
 
@@ -38,11 +40,22 @@ check_recipe <- function(recipe) {
   }
   # Check temporal settings and
   # count the number of verifications
-  if (!all(TIME_SETTINGS %in% names(recipe$Analysis$Time))) {
-    error(recipe$Run$logger,
-	  paste0("The element 'Time' in the recipe must contain all of the ",
-		 "following: ", paste(TIME_SETTINGS, collapse = ", "), "."))
-    error_status <- T
+  if (tolower(recipe$Analysis$Horizon) == "seasonal") {
+    if (!all(TIME_SETTINGS_SEASONAL %in% names(recipe$Analysis$Time))) {
+      error(recipe$Run$logger,
+	    paste0("The element 'Time' in the recipe must contain all of the ",
+		   "following: ", paste(TIME_SETTINGS_SEASONAL,
+					collapse = ", "), "."))
+      error_status <- T
+    }
+  } else if (tolower(recipe$Analysis$Horizon) == "decadal") {
+    if (!all(TIME_SETTINGS_DECADAL %in% names(recipe$Analysis$Time))) {
+      error(recipe$Run$logger,
+	    paste0("The element 'Time' in the recipe must contain all of the ",
+		   "following: ", paste(TIME_SETTINGS_DECADAL,
+					collapse = ", "), "."))
+      error_status <- T
+    }
   }
   ## TODO: Is this needed?
   if (is.null(recipe$Analysis$Time$fcst_year) || 
-- 
GitLab


From 7a985badf5d0cb06eec6504d56f20c1100ca1d57 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Fri, 23 Dec 2022 15:01:01 +0100
Subject: [PATCH 13/57] Add more detailed time section checks

---
 tools/check_recipe.R | 39 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index fcd30ef3..5fca3597 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -38,8 +38,7 @@ check_recipe <- function(recipe) {
 		"following: ", paste(HORIZONS, collapse = ", "), "."))
     error_status <- T
   }
-  # Check temporal settings and
-  # count the number of verifications
+  # Check time settings
   if (tolower(recipe$Analysis$Horizon) == "seasonal") {
     if (!all(TIME_SETTINGS_SEASONAL %in% names(recipe$Analysis$Time))) {
       error(recipe$Run$logger,
@@ -57,6 +56,42 @@ check_recipe <- function(recipe) {
       error_status <- T
     }
   }
+  # Check ftime_min and ftime_max
+  if ((!(recipe$Analysis$Time$ftime_min > 0)) ||
+      (!is.integer(recipe$Analysis$Time$ftime_min))) {
+    error(recipe$Run$logger,
+	  "The element 'ftime_min' must be an integer larger than 0.")
+    error_status <- T
+  }
+  if ((!(recipe$Analysis$Time$ftime_max > 0)) ||
+      (!is.integer(recipe$Analysis$Time$ftime_max))) {
+    error(recipe$Run$logger,
+	  "The element 'ftime_max' must be an integer larger than 0.")
+    error_status <- T
+  }
+  if (recipe$Analysis$Time$ftime_max < recipe$Analysis$Time$ftime_min) {
+    error(recipe$Run$logger,
+	  "'ftime_max' cannot be smaller than 'ftime_min'.")
+    error_status <- T
+  }
+  # Check consistency of hindcast years
+  if (!(as.numeric(recipe$Analysis$Time$hcst_start) %% 1 == 0) ||
+      (!(recipe$Analysis$Time$hcst_start > 0))) {
+    error(recipe$Run$logger,
+	  "The element 'hcst_start' must be a valid year.")
+    error_status <- T
+  }
+  if (!(as.numeric(recipe$Analysis$Time$hcst_end) %% 1 == 0) ||
+      (!(recipe$Analysis$Time$hcst_end > 0))) {
+    error(recipe$Run$logger,
+	  "The element 'hcst_end' must be a valid year.")
+    error_status <- T
+  }
+  if (recipe$Analysis$Time$hcst_end < recipe$Analysis$Time$hcst_start) {
+    error(recipe$Run$logger,
+	  "'hcst_end' cannot be smaller than 'hcst_start'.")
+    error_status <- T
+  }
   ## TODO: Is this needed?
   if (is.null(recipe$Analysis$Time$fcst_year) || 
       tolower(recipe$Analysis$Time$fcst_year) == 'none') {
-- 
GitLab


From d7ea2794b6df6e1492888546679aeba8758f63e3 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 29 Dec 2022 15:11:47 +0100
Subject: [PATCH 14/57] Update script to divide recipe

---
 ...ample.yml => recipe_splitting_example.yml} | 16 +--
 tools/divide_recipe.R                         | 98 ++++++++++---------
 2 files changed, 61 insertions(+), 53 deletions(-)
 rename recipes/{recipe_example.yml => recipe_splitting_example.yml} (86%)

diff --git a/recipes/recipe_example.yml b/recipes/recipe_splitting_example.yml
similarity index 86%
rename from recipes/recipe_example.yml
rename to recipes/recipe_splitting_example.yml
index 27f74df9..e62611ab 100644
--- a/recipes/recipe_example.yml
+++ b/recipes/recipe_splitting_example.yml
@@ -4,7 +4,7 @@
 
 Description:
   Author: V. Agudetse
-  Info: Test for main recipe
+  Info: Test for recipe splitting
 
 ################################################################################
 ## ANALYSIS CONFIGURATION
@@ -21,23 +21,25 @@ Analysis:
       - {name: system5c3s}
     Multimodel: False # single option
     Reference:
-      name: era5 # multiple references for single model?
+      - {name: era5} # multiple references for single model?
   Time:
-    sdate: '1101' # list, split
+    sdate: # list, split
+      - '1101'
+      - '1201'
     fcst_year: '2020' # list, don't split, handled internally
     hcst_start: '1993' # single option
     hcst_end: '2016' # single option
     ftime_min: 1 # single option
-    ftime_max: 6 # singl eoption
+    ftime_max: 6 # single option
   Region: # multiple lists, split? Add region name if length(Region) > 1
-    - {name: "global", latmin: -90, latmax: 90, lonmin: 0, lonmax: 360}
+    - {name: "global", latmin: -90, latmax: 90, lonmin: 0, lonmax: 359.9}
     - {name: "nino34", latmin: -5, latmax: 5, lonmin: -10, lonmax: 60}
   Regrid:
-    method: bilinear # allow multiple methods?
+    method: bilinear ## TODO: allow multiple methods?
     type: to_system
   Workflow:
     Calibration:
-      method: mse_min # list, split?
+      method: mse_min ## TODO: list, split?
     Skill:
       metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS # list, don't split
     Probabilities:
diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R
index 8f4ef35a..fd4efc6e 100644
--- a/tools/divide_recipe.R
+++ b/tools/divide_recipe.R
@@ -1,11 +1,8 @@
-# recipe: the content of the recipe
-# verifications: the output from check_recipe
-# folder: the name of the output folder for this run
-# logger: the log file obtain from prepare_outputs
+# recipe: the recipe as returned by prepare_outputs()
+divide_recipe <- function(recipe) {
 
-## TODO: Rethink strategy and adapt function accordingly
-divide_recipe <- function(recipe, verifications, folder, logger) {
-  info(logger, "Spliting recipe in single verifications.")
+  ## TODO: Implement dependent vs independent verifications?
+  info(recipe$Run$logger, "Spliting recipe in single verifications.")
   beta_recipe <- list(Description = append(recipe$Description, 
                                            "split version"),
                       Analysis = list(Horizon = recipe$Analysis$Horizon,
@@ -17,37 +14,37 @@ divide_recipe <- function(recipe, verifications, folder, logger) {
                                       Workflow = recipe$Analysis$Workflow,
                                       Output_format = 
 				        recipe$Analysis$Output_format),
-		      Run = recipe$Run)
-  # duplicate recipe by Variables considering dep and indep:
-  all_recipes <- list(beta_recipe)
-  i <- 1 # to get track of the recipe number
-  for (indep in verifications$independent) {
-    all_recipes[[i]]$Analysis$Variables <- indep
-    i = i + 1
-    all_recipes <- append(all_recipes, list(beta_recipe))
-  }
-  for (dep in verifications$dependent) {
-    all_recipes[[i]]$Analysis$Variables <- dep
-    i = i + 1
-    all_recipes <- append(all_recipes, list(beta_recipe))
+		      Run = recipe$Run[c("Loglevel", "output_dir", "Terminal",
+					 "code_dir", "logfile")])
+
+  # duplicate recipe by independent variables:
+  all_recipes <- rep(list(beta_recipe), length(recipe$Analysis$Variables))
+  for (var in 1:length(recipe$Analysis$Variables)) {
+    all_recipes[[var]]$Analysis$Variables <- recipe$Analysis$Variables[[var]]
   }
-  all_recipes <- all_recipes[-length(all_recipes)]
+  # for (dep in verifications$dependent) {
+  #   all_recipes[[i]]$Analysis$Variables <- dep
+  #   i = i + 1
+  #   all_recipes <- append(all_recipes, list(beta_recipe))
+  # }
+  # all_recipes <- all_recipes[-length(all_recipes)] # wth does this do
+  
   # duplicate recipe by Datasets:
   # check Systems
   if (recipe$Analysis$Datasets$Multimodel) {
     for (reci in 1:length(all_recipes)) {
-      all_recipes[[reci]]$Analysis$Datasets <- list(
-                            System = recipe$Analysis$Datasets$System,
-                            Multimodel = recipe$Analysis$Datasets$Multimodel,
-                            Reference = NULL)
+      all_recipes[[reci]]$Analysis$Datasets <- 
+	list(System = recipe$Analysis$Datasets$System,
+	     Multimodel = recipe$Analysis$Datasets$Multimodel,
+	     Reference = NULL)
     }
   } else {
     for (sys in 1:length(recipe$Analysis$Datasets$System)) {
       for (reci in 1:length(all_recipes)) {
-        all_recipes[[reci]]$Analysis$Datasets <- list(
-                            System = recipe$Analysis$Datasets$System[[sys]],
-                            Multimodel = recipe$Analysis$Datasets$Multimodel,
-                            Reference = NULL)
+        all_recipes[[reci]]$Analysis$Datasets <- 
+	  list(System = recipe$Analysis$Datasets$System[[sys]],
+	       Multimodel = recipe$Analysis$Datasets$Multimodel,
+	       Reference = NULL)
       }
       if (sys == 1) {
         recipes <- all_recipes
@@ -74,28 +71,28 @@ divide_recipe <- function(recipe, verifications, folder, logger) {
   # Duplicate recipe by Region
   recipes <- list()
   for (reg in 1:length(recipe$Analysis$Region)) {
-    if (length(recipe$Analysis$Region[[reg]]) == 4) { ##TODO: THIS SHOULD BE ONLY CHECK IN THE RECIPE CHECKER?
+    # if (length(recipe$Analysis$Region[[reg]]) == 4) { ##TODO: THIS SHOULD BE ONLY CHECK IN THE RECIPE CHECKER?
       for (reci in 1:length(all_recipes)) {
-        all_recipes[[reci]]$Analysis$Region <-
-                              recipe$Analysis$Region[[reg]]
+        all_recipes[[reci]]$Analysis$Region <- recipe$Analysis$Region[[reg]]
       }
       recipes <- append(recipes, all_recipes)
-    }
+    # }
   }
   all_recipes <- recipes
   rm(list = 'recipes')
+  # Duplicate recipe by start date
   if (tolower(recipe$Analysis$Horizon) == 'seasonal') {
-    for (sday in 1:length(recipe$Analysis$Time$sdate$fcst_sday)) {
+    for (sdate in 1:length(recipe$Analysis$Time$sdate)) {
       for (reci in 1:length(all_recipes)) {
-      all_recipes[[reci]]$Analysis$Time <- list(sdate = list(
-                      fcst_syear = recipe$Analysis$Time$sdate$fcst_syear,
-                      fcst_sday = recipe$Analysis$Time$sdate$fcst_sday[[sday]]),
-                      hcst_start = recipe$Analysis$Time$hcst_start,
-                      hcst_end = recipe$Analysis$Time$hcst_end,
-                      leadtimemin = recipe$Analysis$Time$leadtimemin,
-                      leadtimemax = recipe$Analysis$Time$leadtimemax)
+      all_recipes[[reci]]$Analysis$Time <-
+	list(sdate = recipe$Analysis$Time$sdate[[sdate]],
+	     fcst_year = recipe$Analysis$Time$fcst_year,
+	     hcst_start = recipe$Analysis$Time$hcst_start,
+	     hcst_end = recipe$Analysis$Time$hcst_end,
+	     ftime_min = recipe$Analysis$Time$ftime_min,
+	     ftime_max = recipe$Analysis$Time$ftime_max)
       }
-      if (sday == 1) {
+      if (sdate == 1) {
         recipes <- all_recipes
       } else {
         recipes <- append(recipes, all_recipes)
@@ -104,12 +101,21 @@ divide_recipe <- function(recipe, verifications, folder, logger) {
     all_recipes <- recipes
     rm(list = 'recipes')
   } # Rest of horizons
-  # Finally, save all recipes in saparated yaml files
+  # Save all recipes in separate YAML files
+  ## TODO: Re-add recipe$Run$logger
   for (reci in 1:length(all_recipes)) {
+    if (reci < 10) {
+      recipe_number <- paste0("0", reci)
+    } else {
+      recipe_number <- reci
+    }
     write_yaml(all_recipes[[reci]], 
-               paste0(folder, "/logs/recipes/recipe_", reci, ".yml"))
+               paste0(recipe$Run$output_dir, "/logs/recipes/recipe_",
+		      recipe_number, ".yml"))
+    all_recipes[[reci]]$Run$logger <- recipe$Run$logger
   }
-  text <- paste0("See folder ",folder,"/logs/recipes/ to see the individual recipes.")
-  info(logger, text)
+  text <- paste0("See recipe$Run$output_dir ", recipe$Run$output_dir,
+		 "/logs/recipes/ to see the individual recipes.")
+  info(recipe$Run$logger, text)
   return(all_recipes)
 }
-- 
GitLab


From 449258ad087a30faa1ae236eb381bd4e711dd481 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 29 Dec 2022 15:14:25 +0100
Subject: [PATCH 15/57] Move old test recipes to new directory

---
 recipes/tests/{ => old_tests}/execute_tests.R            | 0
 recipes/tests/{ => old_tests}/seasonal_testWorkflow1.yml | 0
 recipes/tests/{ => old_tests}/seasonal_testWorkflow2.yml | 0
 recipes/tests/{ => old_tests}/seasonal_testWorkflow3.yml | 0
 recipes/tests/{ => old_tests}/seasonal_testWorkflow4.yml | 0
 recipes/tests/{ => old_tests}/seasonal_testWorkflow5.yml | 0
 recipes/tests/{ => old_tests}/seasonal_testWorkflow6.yml | 0
 recipes/tests/{ => old_tests}/seasonal_testWorkflow7.yml | 0
 recipes/tests/{ => old_tests}/seasonal_testWorkflow8.yml | 0
 9 files changed, 0 insertions(+), 0 deletions(-)
 rename recipes/tests/{ => old_tests}/execute_tests.R (100%)
 rename recipes/tests/{ => old_tests}/seasonal_testWorkflow1.yml (100%)
 rename recipes/tests/{ => old_tests}/seasonal_testWorkflow2.yml (100%)
 rename recipes/tests/{ => old_tests}/seasonal_testWorkflow3.yml (100%)
 rename recipes/tests/{ => old_tests}/seasonal_testWorkflow4.yml (100%)
 rename recipes/tests/{ => old_tests}/seasonal_testWorkflow5.yml (100%)
 rename recipes/tests/{ => old_tests}/seasonal_testWorkflow6.yml (100%)
 rename recipes/tests/{ => old_tests}/seasonal_testWorkflow7.yml (100%)
 rename recipes/tests/{ => old_tests}/seasonal_testWorkflow8.yml (100%)

diff --git a/recipes/tests/execute_tests.R b/recipes/tests/old_tests/execute_tests.R
similarity index 100%
rename from recipes/tests/execute_tests.R
rename to recipes/tests/old_tests/execute_tests.R
diff --git a/recipes/tests/seasonal_testWorkflow1.yml b/recipes/tests/old_tests/seasonal_testWorkflow1.yml
similarity index 100%
rename from recipes/tests/seasonal_testWorkflow1.yml
rename to recipes/tests/old_tests/seasonal_testWorkflow1.yml
diff --git a/recipes/tests/seasonal_testWorkflow2.yml b/recipes/tests/old_tests/seasonal_testWorkflow2.yml
similarity index 100%
rename from recipes/tests/seasonal_testWorkflow2.yml
rename to recipes/tests/old_tests/seasonal_testWorkflow2.yml
diff --git a/recipes/tests/seasonal_testWorkflow3.yml b/recipes/tests/old_tests/seasonal_testWorkflow3.yml
similarity index 100%
rename from recipes/tests/seasonal_testWorkflow3.yml
rename to recipes/tests/old_tests/seasonal_testWorkflow3.yml
diff --git a/recipes/tests/seasonal_testWorkflow4.yml b/recipes/tests/old_tests/seasonal_testWorkflow4.yml
similarity index 100%
rename from recipes/tests/seasonal_testWorkflow4.yml
rename to recipes/tests/old_tests/seasonal_testWorkflow4.yml
diff --git a/recipes/tests/seasonal_testWorkflow5.yml b/recipes/tests/old_tests/seasonal_testWorkflow5.yml
similarity index 100%
rename from recipes/tests/seasonal_testWorkflow5.yml
rename to recipes/tests/old_tests/seasonal_testWorkflow5.yml
diff --git a/recipes/tests/seasonal_testWorkflow6.yml b/recipes/tests/old_tests/seasonal_testWorkflow6.yml
similarity index 100%
rename from recipes/tests/seasonal_testWorkflow6.yml
rename to recipes/tests/old_tests/seasonal_testWorkflow6.yml
diff --git a/recipes/tests/seasonal_testWorkflow7.yml b/recipes/tests/old_tests/seasonal_testWorkflow7.yml
similarity index 100%
rename from recipes/tests/seasonal_testWorkflow7.yml
rename to recipes/tests/old_tests/seasonal_testWorkflow7.yml
diff --git a/recipes/tests/seasonal_testWorkflow8.yml b/recipes/tests/old_tests/seasonal_testWorkflow8.yml
similarity index 100%
rename from recipes/tests/seasonal_testWorkflow8.yml
rename to recipes/tests/old_tests/seasonal_testWorkflow8.yml
-- 
GitLab


From db4d66fb1edb19085f882ed545df2b66740de349 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 29 Dec 2022 15:18:56 +0100
Subject: [PATCH 16/57] Change enable_checks param to disable_checks, default
 FALSE

---
 tools/prepare_outputs.R | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R
index d6eb970e..56ab56c6 100644
--- a/tools/prepare_outputs.R
+++ b/tools/prepare_outputs.R
@@ -1,7 +1,7 @@
 #'Read recipe YAML file and create and store logfile info
 #'
 #'The purpose of this function is to read the recipe configuration for Auto-S2S
-#'workflows and create logfiles stores in an the output directory specified in 
+#'workflows and create logfiles stored in an the output directory specified in 
 #'the recipe. It returns an object of class logger that stores information on 
 #'the recipe configuration and errors.
 #'
@@ -21,10 +21,10 @@
 #'@export
 
 prepare_outputs <- function(recipe_file,
-			    enable_checks = TRUE) {
+			    disable_checks = FALSE) {
 
 # recipe_file: path to recipe YAML file
-# enable_checker: Whether or not to run the recipe checker
+# disable_checks: If TRUE, does not perform checks on recipe
 
   recipe <- read_yaml(recipe_file)
   recipe$recipe_path <- recipe_file
@@ -72,9 +72,8 @@ prepare_outputs <- function(recipe_file,
   recipe$Run$logger <- logger
   recipe$Run$logfile <- logfile
   # Run recipe checker
-  if (enable_checks) {
+  if !(disable_checks) {
     check_recipe(recipe)
   }
-
   return(recipe)
 }
-- 
GitLab


From 9187ec11aa61e6ad87d0f257ffb3e593328e832f Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 29 Dec 2022 16:36:28 +0100
Subject: [PATCH 17/57] New recipe; test multiple verifications run serially;
 fix bug in prepare_outputs()

---
 modules/test_seasonal.R                       |  3 +-
 .../tests/recipe_seasonal_two-variables.yml   | 60 +++++++++++++++++++
 tools/divide_recipe.R                         |  7 ++-
 tools/prepare_outputs.R                       |  2 +-
 4 files changed, 68 insertions(+), 4 deletions(-)
 create mode 100644 recipes/tests/recipe_seasonal_two-variables.yml

diff --git a/modules/test_seasonal.R b/modules/test_seasonal.R
index c74fd1b1..d776609d 100644
--- a/modules/test_seasonal.R
+++ b/modules/test_seasonal.R
@@ -5,8 +5,9 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
-recipe_file <- "modules/Loading/testing_recipes/recipe_seasonal-tests.yml"
+recipe_file <- "recipes/recipe_splitting_example.yml"
 recipe <- prepare_outputs(recipe_file, enable_checks = F)
+atomic_recipes <- divide_recipe(recipe)
 ## archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 
 # Load datasets
diff --git a/recipes/tests/recipe_seasonal_two-variables.yml b/recipes/tests/recipe_seasonal_two-variables.yml
new file mode 100644
index 00000000..89406ece
--- /dev/null
+++ b/recipes/tests/recipe_seasonal_two-variables.yml
@@ -0,0 +1,60 @@
+################################################################################ 
+## RECIPE DESCRIPTION  
+################################################################################ 
+
+Description:
+  Author: V. Agudetse
+  Info: Test Independent verification of two variables
+
+################################################################################
+## ANALYSIS CONFIGURATION
+################################################################################
+
+Analysis:
+  Horizon: Seasonal
+  Variables: # ECVs and Indicators?
+    - {name: tas, freq: monthly_mean}
+    - {name: prlr, freq: monthly_mean}
+  Datasets:
+    System: # multiple systems for single model, split if Multimodel = F
+      - {name: system5c3s}
+    Multimodel: False # single option
+    Reference:
+      - {name: era5} # multiple references for single model?
+  Time:
+    sdate: # list, split
+      - '0101'
+    fcst_year: '2020' # list, don't split, handled internally
+    hcst_start: '2000' # single option
+    hcst_end: '2016' # single option
+    ftime_min: 1 # single option
+    ftime_max: 3 # single option
+  Region: # multiple lists, split? Add region name if length(Region) > 1
+    - {name: "nino34", latmin: -5, latmax: 5, lonmin: -10, lonmax: 60}
+  Regrid:
+    method: bilinear ## TODO: allow multiple methods?
+    type: to_system
+  Workflow:
+    Anomalies:
+      compute: yes
+      cross_validation: yes
+    Calibration:
+      method: mse_min ## TODO: list, split?
+    Skill:
+      metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS # list, don't split
+    Probabilities:
+      percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]] # list, don't split
+    Indicators:
+      index: no # ? 
+  ncores: 7
+  remove_NAs: yes # bool, don't split
+  Output_format: S2S4E # string, don't split
+
+################################################################################
+## Run CONFIGURATION
+################################################################################
+Run:
+  Loglevel: INFO
+  Terminal: yes
+  output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/
+  code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/
diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R
index fd4efc6e..6cc8ce4a 100644
--- a/tools/divide_recipe.R
+++ b/tools/divide_recipe.R
@@ -114,8 +114,11 @@ divide_recipe <- function(recipe) {
 		      recipe_number, ".yml"))
     all_recipes[[reci]]$Run$logger <- recipe$Run$logger
   }
-  text <- paste0("See recipe$Run$output_dir ", recipe$Run$output_dir,
-		 "/logs/recipes/ to see the individual recipes.")
+  info(recipe$Run$logger,
+       paste("The main recipe has been devided into", length(all_recipes),
+	     "atomic recipes."))
+  text <- paste0("See output directory ", recipe$Run$output_dir,
+		 "/logs/recipes/ to see all the individual atomic recipes.")
   info(recipe$Run$logger, text)
   return(all_recipes)
 }
diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R
index 56ab56c6..72fc5433 100644
--- a/tools/prepare_outputs.R
+++ b/tools/prepare_outputs.R
@@ -72,7 +72,7 @@ prepare_outputs <- function(recipe_file,
   recipe$Run$logger <- logger
   recipe$Run$logfile <- logfile
   # Run recipe checker
-  if !(disable_checks) {
+  if (!disable_checks) {
     check_recipe(recipe)
   }
   return(recipe)
-- 
GitLab


From 8b30132f06d7cce5b4202d68400ee36b30e382a5 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 29 Dec 2022 17:01:50 +0100
Subject: [PATCH 18/57] Add info to atomic recipe, fix typo

---
 tools/divide_recipe.R | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R
index 6cc8ce4a..18962f93 100644
--- a/tools/divide_recipe.R
+++ b/tools/divide_recipe.R
@@ -4,7 +4,9 @@ divide_recipe <- function(recipe) {
   ## TODO: Implement dependent vs independent verifications?
   info(recipe$Run$logger, "Spliting recipe in single verifications.")
   beta_recipe <- list(Description = append(recipe$Description, 
-                                           "split version"),
+                                           list(Origin = paste("Atomic recipe,",
+							       "split from:", 
+							       recipe$name))),
                       Analysis = list(Horizon = recipe$Analysis$Horizon,
                                       Variables = NULL,
                                       Datasets = NULL,
@@ -115,7 +117,7 @@ divide_recipe <- function(recipe) {
     all_recipes[[reci]]$Run$logger <- recipe$Run$logger
   }
   info(recipe$Run$logger,
-       paste("The main recipe has been devided into", length(all_recipes),
+       paste("The main recipe has been divided into", length(all_recipes),
 	     "atomic recipes."))
   text <- paste0("See output directory ", recipe$Run$output_dir,
 		 "/logs/recipes/ to see all the individual atomic recipes.")
-- 
GitLab


From 8a589bfe32da3bf2e5ddeb3662d96294f4988481 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Fri, 30 Dec 2022 10:16:56 +0100
Subject: [PATCH 19/57] Simple example using a for loop

---
 modules/test_seasonal.R | 36 +++++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/modules/test_seasonal.R b/modules/test_seasonal.R
index d776609d..921e8f96 100644
--- a/modules/test_seasonal.R
+++ b/modules/test_seasonal.R
@@ -5,23 +5,25 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
-recipe_file <- "recipes/recipe_splitting_example.yml"
-recipe <- prepare_outputs(recipe_file, enable_checks = F)
+recipe_file <- "recipes/tests/recipe_seasonal_two-variables.yml"
+recipe <- prepare_outputs(recipe_file, disable_checks = T)
 atomic_recipes <- divide_recipe(recipe)
 ## archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 
-# Load datasets
-data <- load_datasets(recipe)
-# Calibrate datasets
-calibrated_data <- calibrate_datasets(recipe, data)
-# Compute anomalies
-calibrated_data <- compute_anomalies(recipe, calibrated_data)
-# Compute skill metrics 
-skill_metrics <- compute_skill_metrics(recipe, calibrated_data)
-# Compute percentiles and probability bins
-probabilities <- compute_probabilities(recipe, calibrated_data)
-# Export all data to netCDF
-save_data(recipe, calibrated_data, skill_metrics, probabilities)
-# Plot data
-plot_data(recipe, calibrated_data, skill_metrics, probabilities,
-	  significance = T)
+for (atomic_recipe in atomic_recipes) {
+  # Load datasets
+  data <- load_datasets(atomic_recipe)
+  # Calibrate datasets
+  calibrated_data <- calibrate_datasets(atomic_recipe, data)
+  # Compute anomalies
+  calibrated_data <- compute_anomalies(atomic_recipe, calibrated_data)
+  # Compute skill metrics 
+  skill_metrics <- compute_skill_metrics(atomic_recipe, calibrated_data)
+  # Compute percentiles and probability bins
+  probabilities <- compute_probabilities(atomic_recipe, calibrated_data)
+  # Export all data to netCDF
+  save_data(atomic_recipe, calibrated_data, skill_metrics, probabilities)
+  # Plot data
+  plot_data(atomic_recipe, calibrated_data, skill_metrics, probabilities,
+	    significance = T)
+}
-- 
GitLab


From 200d3eefd7b197e11701da2cc92e83da726a198d Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Fri, 30 Dec 2022 10:18:34 +0100
Subject: [PATCH 20/57] Add a TODO

---
 modules/Saving/paths2save.R | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/Saving/paths2save.R b/modules/Saving/paths2save.R
index 2d6353fe..2d5a0a4e 100644
--- a/modules/Saving/paths2save.R
+++ b/modules/Saving/paths2save.R
@@ -1,4 +1,6 @@
 ## TODO: Separate by time aggregation
+## TODO: Build a default path that accounts for:
+## variable, system, reference, start date and region name
 
 get_filename <- function(dir, recipe, var, date, agg, file.type) {
   # This function builds the path of the output file based on directory,
-- 
GitLab


From 2473ba5d0f791e71de913f92331229bd59ca6fe4 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Wed, 11 Jan 2023 15:53:03 +0100
Subject: [PATCH 21/57] Adapt check_recipe() and prepare_outputs() to main
 recipe, improve generation of atomic recipes

---
 tools/check_recipe.R       | 30 ++++++++++++++------
 tools/divide_recipe.R      | 23 +++++++--------
 tools/prepare_outputs.R    | 30 ++++++++++++--------
 tools/read_atomic_recipe.R | 57 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 106 insertions(+), 34 deletions(-)
 create mode 100644 tools/read_atomic_recipe.R

diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index 5fca3597..3371b61f 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -1,8 +1,8 @@
 check_recipe <- function(recipe) {
-
   # recipe: yaml recipe already read it
+  ## TODO: Change input to recipe_file??
+  ## TODO: Adapt to main recipe, generate 'recipe_check.log' file
   ## TODO: Adapt to decadal case
-
   info(recipe$Run$logger, paste("Checking recipe:", recipe$recipe_path))
 
   # ---------------------------------------------------------------------
@@ -167,11 +167,22 @@ check_recipe <- function(recipe) {
 
   # Region checks:
   LIMITS <- c('latmin', 'latmax', 'lonmin', 'lonmax')
-  if (!all(LIMITS %in% names(recipe$Analysis$Region))) {
-    error(recipe$Run$logger,
-          paste0("There must be 4 elements in 'Region': ",
-		 paste(LIMITS, collapse = ", "), "."))
-    error_status <- T
+  for (region in recipe$Analysis$Region) {
+    if (!all(LIMITS %in% names(region))) {
+      error(recipe$Run$logger,
+            paste0("There must be 4 elements in 'Region': ",
+		   paste(LIMITS, collapse = ", "), "."))
+      error_status <- T
+    }
+  }
+  if (length(recipe$Analysis$Region) > 1) {
+    for (region in recipe$Analysis$Region) {
+      if (is.null(region$name)) {
+	error(recipe$Run$logger,
+	      paste("If more than one region has been defined, every region",
+		    "must have a unique name."))
+      }
+    }
   }
     ## TODO: Implement multiple regions
     # nregions <- length(recipe$Analysis$Region)
@@ -222,7 +233,7 @@ check_recipe <- function(recipe) {
 		  "(True/False or yes/no)."))
       error_status <- T
     } else if ((recipe$Analysis$Workflow$Anomalies$compute) &&
-	       (!is.logical(recipe$Analysis$Workflow$Anomalies))) {
+	       (!is.logical(recipe$Analysis$Workflow$Anomalies$cross_validation))) {
       error(recipe$Run$logger,
 	    paste("If anomaly computation is requested, parameter",
 		  "'cross_validation' must be defined under 'Anomalies',
@@ -311,7 +322,8 @@ check_recipe <- function(recipe) {
   # Return error if any check has failed
   if (error_status) {
     error(recipe$Run$logger, "RECIPE CHECK FAILED.")
-    stop("The recipe contains some errors. The full list is in the logs.")
+    stop("The recipe contains some errors. Find the full list in the",
+	 "startup.log file.")
   } else {
     info(recipe$Run$logger, "##### RECIPE CHECK SUCCESSFULL #####")
     # return(append(nverifications, fcst.sdate))
diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R
index 18962f93..f2ab44eb 100644
--- a/tools/divide_recipe.R
+++ b/tools/divide_recipe.R
@@ -2,7 +2,7 @@
 divide_recipe <- function(recipe) {
 
   ## TODO: Implement dependent vs independent verifications?
-  info(recipe$Run$logger, "Spliting recipe in single verifications.")
+  info(recipe$Run$logger, "Splitting recipe in single verifications.")
   beta_recipe <- list(Description = append(recipe$Description, 
                                            list(Origin = paste("Atomic recipe,",
 							       "split from:", 
@@ -18,7 +18,6 @@ divide_recipe <- function(recipe) {
 				        recipe$Analysis$Output_format),
 		      Run = recipe$Run[c("Loglevel", "output_dir", "Terminal",
 					 "code_dir", "logfile")])
-
   # duplicate recipe by independent variables:
   all_recipes <- rep(list(beta_recipe), length(recipe$Analysis$Variables))
   for (var in 1:length(recipe$Analysis$Variables)) {
@@ -73,12 +72,10 @@ divide_recipe <- function(recipe) {
   # Duplicate recipe by Region
   recipes <- list()
   for (reg in 1:length(recipe$Analysis$Region)) {
-    # if (length(recipe$Analysis$Region[[reg]]) == 4) { ##TODO: THIS SHOULD BE ONLY CHECK IN THE RECIPE CHECKER?
-      for (reci in 1:length(all_recipes)) {
-        all_recipes[[reci]]$Analysis$Region <- recipe$Analysis$Region[[reg]]
-      }
-      recipes <- append(recipes, all_recipes)
-    # }
+    for (reci in 1:length(all_recipes)) {
+      all_recipes[[reci]]$Analysis$Region <- recipe$Analysis$Region[[reg]]
+    }
+    recipes <- append(recipes, all_recipes)
   }
   all_recipes <- recipes
   rm(list = 'recipes')
@@ -104,7 +101,6 @@ divide_recipe <- function(recipe) {
     rm(list = 'recipes')
   } # Rest of horizons
   # Save all recipes in separate YAML files
-  ## TODO: Re-add recipe$Run$logger
   for (reci in 1:length(all_recipes)) {
     if (reci < 10) {
       recipe_number <- paste0("0", reci)
@@ -112,15 +108,16 @@ divide_recipe <- function(recipe) {
       recipe_number <- reci
     }
     write_yaml(all_recipes[[reci]], 
-               paste0(recipe$Run$output_dir, "/logs/recipes/recipe_",
+               paste0(recipe$Run$output_dir, "/logs/recipes/atomic_recipe_",
 		      recipe_number, ".yml"))
-    all_recipes[[reci]]$Run$logger <- recipe$Run$logger
+    # all_recipes[[reci]]$Run$logger <- recipe$Run$logger
   }
   info(recipe$Run$logger,
        paste("The main recipe has been divided into", length(all_recipes),
 	     "atomic recipes."))
-  text <- paste0("See output directory ", recipe$Run$output_dir,
+  text <- paste0("Check output directory ", recipe$Run$output_dir,
 		 "/logs/recipes/ to see all the individual atomic recipes.")
   info(recipe$Run$logger, text)
-  return(all_recipes)
+  ## TODO: Change returns?
+  return(recipe$Run$output_dir)
 }
diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R
index 72fc5433..79f6b44d 100644
--- a/tools/prepare_outputs.R
+++ b/tools/prepare_outputs.R
@@ -21,30 +21,35 @@
 #'@export
 
 prepare_outputs <- function(recipe_file,
-			    disable_checks = FALSE) {
-
-# recipe_file: path to recipe YAML file
-# disable_checks: If TRUE, does not perform checks on recipe
-
+			    disable_checks = FALSE,
+			    disable_uniqueID = FALSE) {
+  # recipe_file: path to recipe YAML file
+  # disable_checks: If TRUE, does not perform checks on recipe
+  # disable_uniqueID: If TRUE, does not add a unique ID to output dir
   recipe <- read_yaml(recipe_file)
   recipe$recipe_path <- recipe_file
   recipe$name <- tools::file_path_sans_ext(basename(recipe_file))
   
   output_dir = recipe$Run$output_dir
-  # Create output folders:
-  folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name)), "_",
-                        gsub(" ", "", gsub(":", "", gsub("-", "", Sys.time()))))
+  # Create output folders
+  if (disable_uniqueID) {
+    folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name)))
+  } else {
+    folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name)), "_",
+                          gsub(" ", "", gsub(":", "", gsub("-", "",
+							   Sys.time()))))
+  }
   print("Saving all outputs to:")
-  print(output_dir)
-  print(folder_name)
+  print(paste0(output_dir, folder_name))
   dir.create(file.path(output_dir, folder_name, 'outputs'), recursive = TRUE)
   dir.create(file.path(output_dir, folder_name, 'logs'))
   dir.create(file.path(output_dir, folder_name, 'logs', 'recipes'))
+  ## TODO: Move this part to main recipe 
   # Copy recipe to output folder
   file.copy(recipe$recipe_path, file.path(output_dir, folder_name, 'logs',
 				          'recipes'))
   # Create log output file
-  logfile <- file.path(output_dir, folder_name, 'logs', 'log.txt')
+  logfile <- file.path(output_dir, folder_name, 'logs', 'startup.log')
   file.create(logfile)
 
   # Set default behaviour of logger
@@ -65,9 +70,10 @@ prepare_outputs <- function(recipe_file,
                                                            layout = default_log_layout())))
   } else {
     logger <- log4r::logger(threshold = recipe$Run$Loglevel,
-                            appenders = list(file_appende(logfile, append = TRUE,             
+                            appenders = list(file_appender(logfile, append = TRUE,             
                                              layout = default_log_layout())))
   }
+  ## TODO: Move output_dir assignation 
   recipe$Run$output_dir <- file.path(output_dir, folder_name)
   recipe$Run$logger <- logger
   recipe$Run$logfile <- logfile
diff --git a/tools/read_atomic_recipe.R b/tools/read_atomic_recipe.R
new file mode 100644
index 00000000..1eadb707
--- /dev/null
+++ b/tools/read_atomic_recipe.R
@@ -0,0 +1,57 @@
+#'Read recipe YAML file and create and store logfile info
+#'
+#'The purpose of this function is to read the atomic recipe generated by 
+#'divide_recipe() and create its individual logfile in the output directory
+#'specified in the recipe. It returns the recipe as a list with an object of
+#'class logger added to it, that stores information on the workflow execution
+#'and errors.
+#'
+#'@param recipe_file path to a YAML file with Auto-S2S configuration recipe
+#'
+#'@return list contaning recipe with logger, log file name and log dir name
+#'
+#'@import log4r
+#'@import yaml
+#'
+#'@examples
+#'setwd("/esarchive/scratch/vagudets/repos/auto-s2s/")
+#'library(yaml)
+#'recipe <- prepare_outputs("modules/data_load/recipe_1.yml")
+#'info(recipe$Run$logger, "This is an info message")
+#'
+#'@export
+
+read_atomic_recipe <- function(recipe_file) {
+  # recipe_file: path to recipe YAML file
+  recipe <- read_yaml(recipe_file)
+  recipe$recipe_path <- recipe_file
+  recipe$name <- tools::file_path_sans_ext(basename(recipe_file))
+  # Create log file for atomic recipe
+  logfile <- file.path(recipe$Run$output_dir, 'logs',
+		       paste0(recipe$name, '.log'))
+  file.create(logfile)
+  # Set default behaviour of logger
+  if (is.null(recipe$Run)) {
+    recipe$Run <- list(Loglevel = 'INFO', Terminal = TRUE)
+  }
+  if (is.null(recipe$Run$Loglevel)) {
+    recipe$Run$Loglevel <- 'INFO'
+  }
+  if (!is.logical(recipe$Run$Terminal)) {
+    recipe$Run$Terminal <- TRUE
+  }
+  # logger set-up
+  if (recipe$Run$Terminal) {
+    logger <- log4r::logger(threshold = recipe$Run$Loglevel,  
+                            appenders = list(console_appender(layout = default_log_layout()),
+                                             file_appender(logfile, append = TRUE,   
+                                                           layout = default_log_layout())))
+  } else {
+    logger <- log4r::logger(threshold = recipe$Run$Loglevel,
+                            appenders = list(file_appender(logfile, append = TRUE,             
+                                             layout = default_log_layout())))
+  }
+  recipe$Run$logger <- logger
+  recipe$Run$logfile <- logfile
+  return(recipe)
+}
-- 
GitLab


From ddd75337e16e4f642d8d2f4cbd54d5fb2c5ed2d2 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Wed, 11 Jan 2023 15:54:56 +0100
Subject: [PATCH 22/57] Add read_atomic_recipe() to libs

---
 tools/libs.R | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/libs.R b/tools/libs.R
index a0767f76..e3c3c277 100644
--- a/tools/libs.R
+++ b/tools/libs.R
@@ -16,7 +16,7 @@ library(RColorBrewer)
 library(grDevices)
 #
 #  library(parallel)
-#  library(pryr) # To check mem usage.
+library(pryr) # To check mem usage.
 #setwd("/esarchive/scratch/nperez/git/S2S4E-backend-BSC/data-analysis/")
 #  source('export_2_nc.R')
 #  source('S2S/s2s.filefmt.R')
@@ -32,4 +32,5 @@ source("tools/check_recipe.R")
 source("tools/prepare_outputs.R")
 source("tools/divide_recipe.R")
 source("tools/data_summary.R")
+source("tools/read_atomic_recipe.R")
 # source("tools/add_dims.R") # Not sure if necessary yet
-- 
GitLab


From f5f58b933ad9a671479450110f371d1f40b529e3 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 12 Jan 2023 09:44:03 +0100
Subject: [PATCH 23/57] Add ncores and remove_NAs

---
 tools/divide_recipe.R | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R
index f2ab44eb..6b154f1d 100644
--- a/tools/divide_recipe.R
+++ b/tools/divide_recipe.R
@@ -14,6 +14,8 @@ divide_recipe <- function(recipe) {
                                       Region = NULL,
                                       Regrid = recipe$Analysis$Regrid,
                                       Workflow = recipe$Analysis$Workflow,
+				      ncores = recipe$Analysis$ncores,
+				      remove_NAs = recipe$Analysis$remove_NAs,
                                       Output_format = 
 				        recipe$Analysis$Output_format),
 		      Run = recipe$Run[c("Loglevel", "output_dir", "Terminal",
-- 
GitLab


From 4ef0412cb8c32927a9e092829b86654da147c477 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 12 Jan 2023 11:14:07 +0100
Subject: [PATCH 24/57] Fix 'expected unary operator' message

---
 MODULES | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/MODULES b/MODULES
index 0a01a979..f23a459d 100644
--- a/MODULES
+++ b/MODULES
@@ -3,7 +3,7 @@
   # WARNING: CDO HAS TO BE ON VERSION 1.9.4 
   # (If not, conflicts with  weekly means computation could appear)
 
-if [ $BSC_MACHINE == "power" ]; then
+if [[ $BSC_MACHINE == "power" ]]; then
 
   module unuse /apps/modules/modulefiles/applications
   module use /gpfs/projects/bsc32/software/rhel/7.4/ppc64le/POWER9/modules/all/
@@ -11,7 +11,7 @@ if [ $BSC_MACHINE == "power" ]; then
   module load CDO/1.9.4-foss-2018b 
   module load R/3.6.1-foss-2018b 
 
-elif [ $BSC_MACHINE == "nord3v2" ]; then
+elif [[ $BSC_MACHINE == "nord3v2" ]]; then
 
   module use /gpfs/projects/bsc32/software/suselinux/11/modules/all
   module unuse /apps/modules/modulefiles/applications /apps/modules/modulefiles/compilers /apps/modules/modulefiles/tools /apps/modules/modulefiles/libraries /apps/modules/modulefiles/environment
-- 
GitLab


From 6fb775171db03cf7b6bb3f78a0267a0ac8b012b3 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 12 Jan 2023 11:14:17 +0100
Subject: [PATCH 25/57] Add a TODO

---
 modules/Loading/Loading.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R
index 66a53451..21a9bd7c 100644
--- a/modules/Loading/Loading.R
+++ b/modules/Loading/Loading.R
@@ -324,6 +324,7 @@ load_datasets <- function(recipe) {
   }
 
   # Convert prlr from m/s to mm/day
+  ## TODO: Revise this
   ## TODO: Make a unit conversion function?
   if (variable == "prlr") {
     # Verify that the units are m/s and the same in obs and hcst
-- 
GitLab


From 3d97761563843a89fad4ad9e0e715c2386d5a4b5 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 12 Jan 2023 11:14:42 +0100
Subject: [PATCH 26/57] Restore single verification wf

---
 modules/test_seasonal.R | 34 ++++++++++++++++------------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/modules/test_seasonal.R b/modules/test_seasonal.R
index 921e8f96..1a723478 100644
--- a/modules/test_seasonal.R
+++ b/modules/test_seasonal.R
@@ -6,24 +6,22 @@ source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
 recipe_file <- "recipes/tests/recipe_seasonal_two-variables.yml"
-recipe <- prepare_outputs(recipe_file, disable_checks = T)
+recipe <- prepare_outputs(recipe_file)
 atomic_recipes <- divide_recipe(recipe)
 ## archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 
-for (atomic_recipe in atomic_recipes) {
-  # Load datasets
-  data <- load_datasets(atomic_recipe)
-  # Calibrate datasets
-  calibrated_data <- calibrate_datasets(atomic_recipe, data)
-  # Compute anomalies
-  calibrated_data <- compute_anomalies(atomic_recipe, calibrated_data)
-  # Compute skill metrics 
-  skill_metrics <- compute_skill_metrics(atomic_recipe, calibrated_data)
-  # Compute percentiles and probability bins
-  probabilities <- compute_probabilities(atomic_recipe, calibrated_data)
-  # Export all data to netCDF
-  save_data(atomic_recipe, calibrated_data, skill_metrics, probabilities)
-  # Plot data
-  plot_data(atomic_recipe, calibrated_data, skill_metrics, probabilities,
-	    significance = T)
-}
+# Load datasets
+data <- load_datasets(recipe)
+# Calibrate datasets
+data <- calibrate_datasets(recipe, data)
+# Compute anomalies
+data <- compute_anomalies(recipe, data)
+# Compute skill metrics 
+skill_metrics <- compute_skill_metrics(recipe, data)
+# Compute percentiles and probability bins
+probabilities <- compute_probabilities(recipe, data)
+# Export all data to netCDF
+save_data(recipe, data, skill_metrics, probabilities)
+# Plot data
+plot_data(recipe, data, skill_metrics, probabilities,
+	  significance = T)
-- 
GitLab


From b738aa2eda2b1f4cd37a9481444c8f3bb6926dab Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 12 Jan 2023 11:15:05 +0100
Subject: [PATCH 27/57] tests for multiple verifications and parallelization

---
 modules/test_parallel_workflow.R              | 25 +++++++++++++++++++
 modules/test_split.R                          |  8 ++++++
 modules/test_split_and_run.sh                 | 17 +++++++++++++
 .../tests/recipe_seasonal_two-variables.yml   |  4 +--
 4 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100644 modules/test_parallel_workflow.R
 create mode 100644 modules/test_split.R
 create mode 100644 modules/test_split_and_run.sh

diff --git a/modules/test_parallel_workflow.R b/modules/test_parallel_workflow.R
new file mode 100644
index 00000000..30c9cb91
--- /dev/null
+++ b/modules/test_parallel_workflow.R
@@ -0,0 +1,25 @@
+source("modules/Loading/Loading.R")
+source("modules/Calibration/Calibration.R")
+source("modules/Anomalies/Anomalies.R")
+source("modules/Skill/Skill.R")
+source("modules/Saving/Saving.R")
+source("modules/Visualization/Visualization.R")
+
+args = commandArgs(trailingOnly = TRUE)
+recipe_file <- args[1]
+recipe <- read_atomic_recipe(recipe_file)
+# Load datasets
+data <- load_datasets(recipe)
+# Calibrate datasets
+data <- calibrate_datasets(recipe, data)
+# Compute anomalies
+data <- compute_anomalies(recipe, data)
+# Compute skill metrics 
+skill_metrics <- compute_skill_metrics(recipe, data)
+# Compute percentiles and probability bins
+probabilities <- compute_probabilities(recipe, data)
+# Export all data to netCDF
+save_data(recipe, data, skill_metrics, probabilities)
+# Plot data
+plot_data(recipe, data, skill_metrics, probabilities,
+	  significance = T)
diff --git a/modules/test_split.R b/modules/test_split.R
new file mode 100644
index 00000000..e66a5422
--- /dev/null
+++ b/modules/test_split.R
@@ -0,0 +1,8 @@
+source("tools/libs.R")
+
+# Define recipe file path
+recipe_file <- "recipes/tests/recipe_seasonal_two-variables.yml"
+# Check recipe and prepare output directories
+recipe <- prepare_outputs(recipe_file)
+# Split recipe into atomic recipes
+divide_recipe(recipe)
diff --git a/modules/test_split_and_run.sh b/modules/test_split_and_run.sh
new file mode 100644
index 00000000..f1ca1f90
--- /dev/null
+++ b/modules/test_split_and_run.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+source MODULES
+
+tmpfile=$(mktemp /tmp/test_split.XXXXXX)
+Rscript modules/test_split.R | tee $tmpfile
+outdir=$( tail -n 1 $tmpfile | sed 's/"//g' )
+outdir=($outdir)
+rm $tmpfile
+
+## These will have to be separate scripts to work with autosubmit
+## Create a tmpfile inside the projdir with a consistent name??
+## '??' would be a chunk
+## TODO: Add sbatch
+for recipe in ${outdir[1]}/logs/recipes/atomic_recipe_??.yml; do
+  Rscript modules/test_parallel_workflow.R ${recipe}
+done
diff --git a/recipes/tests/recipe_seasonal_two-variables.yml b/recipes/tests/recipe_seasonal_two-variables.yml
index 89406ece..af262e4d 100644
--- a/recipes/tests/recipe_seasonal_two-variables.yml
+++ b/recipes/tests/recipe_seasonal_two-variables.yml
@@ -30,7 +30,7 @@ Analysis:
     ftime_min: 1 # single option
     ftime_max: 3 # single option
   Region: # multiple lists, split? Add region name if length(Region) > 1
-    - {name: "nino34", latmin: -5, latmax: 5, lonmin: -10, lonmax: 60}
+    - {name: "tropics", latmin: -5, latmax: 5, lonmin: -10, lonmax: 10}
   Regrid:
     method: bilinear ## TODO: allow multiple methods?
     type: to_system
@@ -39,7 +39,7 @@ Analysis:
       compute: yes
       cross_validation: yes
     Calibration:
-      method: mse_min ## TODO: list, split?
+      method: raw ## TODO: list, split?
     Skill:
       metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS # list, don't split
     Probabilities:
-- 
GitLab


From 485a40062089c9c3a50d8bd76ba99ffc334dd143 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 12 Jan 2023 12:29:22 +0100
Subject: [PATCH 28/57] Fix unary error

---
 MODULES | 1 -
 1 file changed, 1 deletion(-)

diff --git a/MODULES b/MODULES
index f23a459d..ff2a2a34 100644
--- a/MODULES
+++ b/MODULES
@@ -16,7 +16,6 @@ elif [[ $BSC_MACHINE == "nord3v2" ]]; then
   module use /gpfs/projects/bsc32/software/suselinux/11/modules/all
   module unuse /apps/modules/modulefiles/applications /apps/modules/modulefiles/compilers /apps/modules/modulefiles/tools /apps/modules/modulefiles/libraries /apps/modules/modulefiles/environment
 
-
   module load CDO/1.9.8-foss-2019b
   module load R/4.1.2-foss-2019b
   module load OpenMPI/4.0.5-GCC-8.3.0-nord3-v2
-- 
GitLab


From e60038ed6c800019815656ef4601b2b9cd65f9c6 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 17 Jan 2023 08:28:01 +0100
Subject: [PATCH 29/57] Test sbatch

---
 modules/test_parallel_workflow.sh | 13 +++++++++++++
 modules/test_split_and_run.sh     |  4 +++-
 2 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 modules/test_parallel_workflow.sh

diff --git a/modules/test_parallel_workflow.sh b/modules/test_parallel_workflow.sh
new file mode 100644
index 00000000..4a5dc512
--- /dev/null
+++ b/modules/test_parallel_workflow.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+#SBATCH -n 1
+#SBATCH -t 23:59:59
+#SBATCH -J med
+#SBATCH -e /esarchive/scratch/vagudets/tmp/auto-s2s-logs/run-%j.err
+#SBATCH -o /esarchive/scratch/vagudets/tmp/auto-s2s-logs/run-%j.out
+
+atomic_recipe=$1
+
+source MODULES
+
+Rscript modules/test_parallel_workflow.R ${atomic_recipe}
diff --git a/modules/test_split_and_run.sh b/modules/test_split_and_run.sh
index f1ca1f90..97b9a9f7 100644
--- a/modules/test_split_and_run.sh
+++ b/modules/test_split_and_run.sh
@@ -2,6 +2,8 @@
 
 source MODULES
 
+set -vx
+
 tmpfile=$(mktemp /tmp/test_split.XXXXXX)
 Rscript modules/test_split.R | tee $tmpfile
 outdir=$( tail -n 1 $tmpfile | sed 's/"//g' )
@@ -13,5 +15,5 @@ rm $tmpfile
 ## '??' would be a chunk
 ## TODO: Add sbatch
 for recipe in ${outdir[1]}/logs/recipes/atomic_recipe_??.yml; do
-  Rscript modules/test_parallel_workflow.R ${recipe}
+  sbatch modules/test_parallel_workflow.sh ${recipe}
 done
-- 
GitLab


From 4b14eb4f4ee0be2e222426f371cfe3a3e17e74b8 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Fri, 3 Feb 2023 10:57:31 +0100
Subject: [PATCH 30/57] Rearrange, clean and generalize scripts

---
 launch.sh                         | 30 ++++++++++++++++++++++++++++++
 modules/{test_split.R => split.R} |  7 ++++---
 modules/test_parallel_workflow.sh | 13 -------------
 modules/test_split_and_run.sh     | 19 -------------------
 run_parallel_workflow.sh          | 14 ++++++++++++++
 5 files changed, 48 insertions(+), 35 deletions(-)
 create mode 100644 launch.sh
 rename modules/{test_split.R => split.R} (52%)
 delete mode 100644 modules/test_parallel_workflow.sh
 delete mode 100644 modules/test_split_and_run.sh
 create mode 100644 run_parallel_workflow.sh

diff --git a/launch.sh b/launch.sh
new file mode 100644
index 00000000..543a203a
--- /dev/null
+++ b/launch.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+source MODULES
+
+# set -vx
+
+# Define recipe and script
+recipe=recipes/tests/recipe_seasonal_two-variables.yml
+script=modules/test_parallel_workflow.R
+
+# Define tmp file to store necessary information
+tmpfile=$(mktemp /tmp/test_split.XXXXXX)
+
+# Create outdir and split recipes
+Rscript modules/split.R ${recipe} | tee $tmpfile
+outdir=$( tail -n 1 $tmpfile | sed 's/"//g' )
+outdir=($outdir)
+
+rm $tmpfile
+
+# Create directory for slurm output
+mkdir -p /esarchive/scratch/$(whoami)/tmp/auto-s2s-logs
+
+## These will have to be separate scripts to work with autosubmit
+## Create a tmpfile inside the projdir with a consistent name??
+## '??' would be a chunk
+# Launch one job per atomic recipe
+for atomic_recipe in ${outdir[1]}/logs/recipes/atomic_recipe_??.yml; do
+  sbatch run_parallel_workflow.sh ${script} ${atomic_recipe}
+done
diff --git a/modules/test_split.R b/modules/split.R
similarity index 52%
rename from modules/test_split.R
rename to modules/split.R
index e66a5422..58391808 100644
--- a/modules/test_split.R
+++ b/modules/split.R
@@ -1,7 +1,8 @@
-source("tools/libs.R")
+suppressMessages(source("tools/libs.R"))
 
-# Define recipe file path
-recipe_file <- "recipes/tests/recipe_seasonal_two-variables.yml"
+# Retrieve recipe file path
+args = commandArgs(trailingOnly = TRUE)
+recipe_file <- args[1]
 # Check recipe and prepare output directories
 recipe <- prepare_outputs(recipe_file)
 # Split recipe into atomic recipes
diff --git a/modules/test_parallel_workflow.sh b/modules/test_parallel_workflow.sh
deleted file mode 100644
index 4a5dc512..00000000
--- a/modules/test_parallel_workflow.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-
-#SBATCH -n 1
-#SBATCH -t 23:59:59
-#SBATCH -J med
-#SBATCH -e /esarchive/scratch/vagudets/tmp/auto-s2s-logs/run-%j.err
-#SBATCH -o /esarchive/scratch/vagudets/tmp/auto-s2s-logs/run-%j.out
-
-atomic_recipe=$1
-
-source MODULES
-
-Rscript modules/test_parallel_workflow.R ${atomic_recipe}
diff --git a/modules/test_split_and_run.sh b/modules/test_split_and_run.sh
deleted file mode 100644
index 97b9a9f7..00000000
--- a/modules/test_split_and_run.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-source MODULES
-
-set -vx
-
-tmpfile=$(mktemp /tmp/test_split.XXXXXX)
-Rscript modules/test_split.R | tee $tmpfile
-outdir=$( tail -n 1 $tmpfile | sed 's/"//g' )
-outdir=($outdir)
-rm $tmpfile
-
-## These will have to be separate scripts to work with autosubmit
-## Create a tmpfile inside the projdir with a consistent name??
-## '??' would be a chunk
-## TODO: Add sbatch
-for recipe in ${outdir[1]}/logs/recipes/atomic_recipe_??.yml; do
-  sbatch modules/test_parallel_workflow.sh ${recipe}
-done
diff --git a/run_parallel_workflow.sh b/run_parallel_workflow.sh
new file mode 100644
index 00000000..dd325b8d
--- /dev/null
+++ b/run_parallel_workflow.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#SBATCH -n 1
+#SBATCH -t 23:59:59
+#SBATCH -J med
+#SBATCH -e /esarchive/scratch/%u/tmp/auto-s2s-logs/run-%j.err
+#SBATCH -o /esarchive/scratch/%u/tmp/auto-s2s-logs/run-%j.out
+
+script=$1
+atomic_recipe=$2
+
+source MODULES
+
+Rscript ${script} ${atomic_recipe}
-- 
GitLab


From d39ca067839b207af3025b1731b57f2e54b681b2 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Fri, 3 Feb 2023 12:09:25 +0100
Subject: [PATCH 31/57] Add new example recipe

---
 launch.sh                                 |  2 +-
 recipes/recipe_splitting_example.yml      |  2 +-
 recipes/tests/recipe_seasonal_example.yml | 62 +++++++++++++++++++++++
 3 files changed, 64 insertions(+), 2 deletions(-)
 create mode 100644 recipes/tests/recipe_seasonal_example.yml

diff --git a/launch.sh b/launch.sh
index 543a203a..89bc75ce 100644
--- a/launch.sh
+++ b/launch.sh
@@ -5,7 +5,7 @@ source MODULES
 # set -vx
 
 # Define recipe and script
-recipe=recipes/tests/recipe_seasonal_two-variables.yml
+recipe=recipes/tests/recipe_seasonal_example.yml
 script=modules/test_parallel_workflow.R
 
 # Define tmp file to store necessary information
diff --git a/recipes/recipe_splitting_example.yml b/recipes/recipe_splitting_example.yml
index e62611ab..5759a23b 100644
--- a/recipes/recipe_splitting_example.yml
+++ b/recipes/recipe_splitting_example.yml
@@ -48,7 +48,7 @@ Analysis:
       index: no # ? 
   ncores: 7
   remove_NAs: yes # bool, don't split
-  Output_format: S2S4E # string, don't split
+  Output_format: Scorecards # string, don't split
 
 ################################################################################
 ## Run CONFIGURATION
diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml
new file mode 100644
index 00000000..7a836b2c
--- /dev/null
+++ b/recipes/tests/recipe_seasonal_example.yml
@@ -0,0 +1,62 @@
+################################################################################ 
+## RECIPE DESCRIPTION  
+################################################################################ 
+
+Description:
+  Author: V. Agudetse
+  Info: Test Independent verification of two variables, two sdates, two systems
+
+################################################################################
+## ANALYSIS CONFIGURATION
+################################################################################
+
+Analysis:
+  Horizon: Seasonal
+  Variables: # ECVs and Indicators?
+    - {name: tas, freq: monthly_mean}
+    - {name: prlr, freq: monthly_mean}
+  Datasets:
+    System: # multiple systems for single model, split if Multimodel = F
+      - {name: system5c3s}
+      - {name: system7c3s}
+    Multimodel: False # single option
+    Reference:
+      - {name: era5} # multiple references for single model?
+  Time:
+    sdate: # list, split
+      - '0101'
+      - '0201'
+    fcst_year: # list, don't split, handled internally
+    hcst_start: '2000' # single option
+    hcst_end: '2016' # single option
+    ftime_min: 1 # single option
+    ftime_max: 3 # single option
+  Region: # multiple lists, Add region name if there is more than 1 region
+    - {latmin: -5, latmax: 5, lonmin: -10, lonmax: 10}
+  Regrid:
+    method: bilinear ## TODO: allow multiple methods?
+    type: to_system
+  Workflow:
+    Anomalies:
+      compute: yes
+      cross_validation: yes
+    Calibration:
+      method: raw ## TODO: list, split?
+    Skill:
+      metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS # list, don't split
+    Probabilities:
+      percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]] # list, don't split
+    Indicators:
+      index: no # ? 
+  ncores: 7
+  remove_NAs: yes # bool, don't split
+  Output_format: Scorecards # string, don't split
+
+################################################################################
+## Run CONFIGURATION
+################################################################################
+Run:
+  Loglevel: INFO
+  Terminal: yes
+  output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/
+  code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/
-- 
GitLab


From 4c82c76ffc24eb292b2abd0bb0b692bd432402f0 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Fri, 10 Feb 2023 17:00:34 +0100
Subject: [PATCH 32/57] Create function to generate autosubmit experiment
 configuration (WIP)

---
 autosubmit/conf/autosubmit.conf           | 42 ++++++++++++
 autosubmit/conf/expdef.conf               | 78 +++++++++++++++++++++++
 autosubmit/conf/jobs.conf                 |  9 +++
 autosubmit/conf/platforms.conf            | 20 ++++++
 autosubmit/conf/proj.conf                 |  4 ++
 recipes/tests/recipe_seasonal_example.yml | 21 ++++++
 tools/write_autosubmit_conf.R             | 51 +++++++++++++++
 7 files changed, 225 insertions(+)
 create mode 100644 autosubmit/conf/autosubmit.conf
 create mode 100644 autosubmit/conf/expdef.conf
 create mode 100644 autosubmit/conf/jobs.conf
 create mode 100644 autosubmit/conf/platforms.conf
 create mode 100644 autosubmit/conf/proj.conf
 create mode 100644 tools/write_autosubmit_conf.R

diff --git a/autosubmit/conf/autosubmit.conf b/autosubmit/conf/autosubmit.conf
new file mode 100644
index 00000000..fd69390a
--- /dev/null
+++ b/autosubmit/conf/autosubmit.conf
@@ -0,0 +1,42 @@
+[config]
+# Experiment identifier
+# No need to change
+EXPID =
+# No need to change.
+# Autosubmit version identifier
+AUTOSUBMIT_VERSION = "3.14.0" 
+# Default maximum number of jobs to be waiting in any platform
+# Default = 3
+MAXWAITINGJOBS = 20
+# Default maximum number of jobs to be running at the same time at any platform
+# Default = 6
+TOTALJOBS = 20
+# Time (seconds) between connections to the HPC queue scheduler to poll already submitted jobs status
+# Default = 10
+SAFETYSLEEPTIME = 10
+# Number of retrials if a job fails. Can be override at job level
+# Default = 0
+RETRIALS = 0
+
+[mail]
+# Enable mail notifications
+# Default = False
+NOTIFICATIONS =  
+# Mail address where notifications will be received
+TO = 
+
+[communications]
+# Communications library used to connect with platforms: paramiko or saga.
+# Default = paramiko
+API = paramiko
+
+[storage]
+# Defines the way of storing the progress of the experiment. The available options are:
+# A PICKLE file (pkl) or an SQLite database (db). Default = pkl
+TYPE = pkl
+# Defines if the remote logs will be copied to the local platform. Default = True.
+COPY_REMOTE_LOGS = False
+
+[migrate]
+# Changes experiment files owner.
+TO_USER =
diff --git a/autosubmit/conf/expdef.conf b/autosubmit/conf/expdef.conf
new file mode 100644
index 00000000..6ea998d9
--- /dev/null
+++ b/autosubmit/conf/expdef.conf
@@ -0,0 +1,78 @@
+[DEFAULT]
+# Experiment identifier
+# No need to change
+EXPID =
+# HPC name.
+# No need to change
+HPCARCH = nord3v2 
+
+[experiment]
+# Supply the list of start dates. Available formats: YYYYMMDD YYYYMMDDhh YYYYMMDDhhmm
+# You can also use an abbreviated syntax for multiple dates with common parts: 200001[01 15] <=> 20000101 20000115
+# 200001[01-04] <=> 20000101 20000102 20000103 20000104
+# DATELIST = 19600101 19650101 19700101
+# DATELIST = 1960[0101 0201 0301]
+# DATELIST = 19[60-65]
+DATELIST = 20201015
+# Supply the list of members. Format fcX
+# You can also use an abreviated syntax for multiple members: fc[0 1 2] <=> fc0 fc1 fc2
+# fc[0-2] <=> fc0 fc1 fc2
+# MEMBERS = fc0 fc1 fc2 fc3 fc4
+#MEMBERS = fc4
+MEMBERS = fc[0-6]
+
+
+# Chunk size unit. STRING = hour, day, month, year
+CHUNKSIZEUNIT = month
+# Chunk size. NUMERIC = 4, 6, 12
+CHUNKSIZE = 1
+# Total number of chunks in experiment. NUMERIC = 30, 15, 10
+NUMCHUNKS = 
+# Initial chunk of the experiment. Optional. DEFAULT = 1
+CHUNKINI =
+# Calendar used. LIST: standard, noleap
+CALENDAR = standard
+
+[project]
+# Select project type. STRING = git, svn, local, none
+# If PROJECT_TYPE is set to none, Autosubmit self-contained dummy templates will be used
+PROJECT_TYPE = local
+# Destination folder name for project. type = STRING, default = leave empty,
+PROJECT_DESTINATION = auto-s2s
+
+# If PROJECT_TYPE is not git, no need to change
+[git]
+# Repository URL  STRING = 'https://github.com/torvalds/linux.git'
+PROJECT_ORIGIN = https://earth.bsc.es/gitlab/es/auto-s2s.git 
+# Select branch or tag, STRING, default = 'master', help = {'master' (default), 'develop', 'v3.1b', ...}
+PROJECT_BRANCH = master
+# type = STRING, default = leave empty, help = if model branch is a TAG leave empty
+PROJECT_COMMIT =
+
+# If PROJECT_TYPE is not svn, no need to change
+[svn]
+# type = STRING, help = 'https://svn.ec-earth.org/ecearth3'
+PROJECT_URL =
+# Select revision number. NUMERIC = 1778
+PROJECT_REVISION =
+
+# If PROJECT_TYPE is not local, no need to change
+[local]
+# type = STRING, help = /foo/bar/ecearth 
+PROJECT_PATH = /esarchive/scratch/vagudets/repos/auto-s2s/
+
+# If PROJECT_TYPE is none, no need to change
+[project_files]
+# Where is PROJECT CONFIGURATION file location relative to project root path
+FILE_PROJECT_CONF = 
+# Where is JOBS CONFIGURATION file location relative to project root path
+FILE_JOBS_CONF =
+# Default job scripts type in the project. type = STRING, default = bash, supported = 'bash', 'python' or 'r'
+JOB_SCRIPTS_TYPE =
+
+[rerun]
+# Is a rerun or not? [Default: Do set FALSE]. BOOLEAN = TRUE, FALSE
+RERUN = FALSE
+# If RERUN = TRUE then supply the list of chunks to rerun
+# LIST = [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]
+CHUNKLIST =
diff --git a/autosubmit/conf/jobs.conf b/autosubmit/conf/jobs.conf
new file mode 100644
index 00000000..e2d7c56e
--- /dev/null
+++ b/autosubmit/conf/jobs.conf
@@ -0,0 +1,9 @@
+[verification]
+FILE = autosubmit/auto-verification.sh
+RUNNING = chunk
+WALLCLOCK = 
+NOTIFY_ON = 
+PLATFORM = nord3v2
+MEMORY = 
+PROCESSORS = 
+
diff --git a/autosubmit/conf/platforms.conf b/autosubmit/conf/platforms.conf
new file mode 100644
index 00000000..33825817
--- /dev/null
+++ b/autosubmit/conf/platforms.conf
@@ -0,0 +1,20 @@
+[power9]
+TYPE = slurm
+HOST = plogin1.bsc.es
+PROJECT = bsc32
+USER = bsc32036
+QUEUE = bsc_es
+SCRATCH_DIR = /gpfs/scratch
+ADD_PROJECT_TO_HOST = false
+
+[nord3v2]
+TYPE = slurm
+HOST = nord4.bsc.es
+PROJECT = bsc32
+ADD_PROJECT_TO_HOST = False
+USER = 
+SCRATCH_DIR = /gpfs/scratch
+PROCESSORS_PER_NODE = 16
+SERIAL_QUEUE = debug
+QUEUE = bsc_es
+CUSTOM_DIRECTIVES = ["#SBATCH --exclusive"]
diff --git a/autosubmit/conf/proj.conf b/autosubmit/conf/proj.conf
new file mode 100644
index 00000000..7eacf6c4
--- /dev/null
+++ b/autosubmit/conf/proj.conf
@@ -0,0 +1,4 @@
+[common]
+
+MODULES = "MODULES"
+OUTDIR = 
diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml
index 7a836b2c..10eb0f90 100644
--- a/recipes/tests/recipe_seasonal_example.yml
+++ b/recipes/tests/recipe_seasonal_example.yml
@@ -60,3 +60,24 @@ Run:
   Terminal: yes
   output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/
   code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/
+  autosubmit: yes
+  # fill only if using autosubmit
+  auto_conf:
+    expid: # if left empty, a new experiment will be created
+    wallclock: '04:00' # hh:mm
+    memory_per_job: '120000'
+    processors_per_job: 8
+    platform: nord3v2 # make this constant?
+    email_notifications: yes
+    notify_completed: no
+    notify_failed: yes
+    email_address: victoria.agudetse@bsc.es
+ 
+################################################################################
+### Autosubmit CONFIGURATION
+#################################################################################
+Autosubmit:
+  use_autosubmit: yes
+  # fill only if using autosubmit
+  #
+  
diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R
new file mode 100644
index 00000000..40614985
--- /dev/null
+++ b/tools/write_autosubmit_conf.R
@@ -0,0 +1,51 @@
+function write_autosubmit_conf(recipe, outdir) {
+  # Function to write autosubmit configuration from an Auto-S2S recipe
+  template_dir <- "autosubmit/conf/"
+  ## TODO: create folder for auto experiment
+  conf_dest_dir <- paste0("/esarchive/autosubmit/", expid "/conf/")
+
+  for (file in list.files(template_dir)) {
+    # Define destionation directory and file name accordin to expid
+    conf_type <- strsplit(file, split = "[.]")[[1]][1]
+    extension <- strsplit(file, split = "[.]")[[1]][2]
+    dest_file <- paste0(conf_type, "_", expid, ".", extension)
+    dest_dir <- paste0("/esarchive/autosubmit/", expid, "/")
+    # Read configuration file
+    conf <- read.config(file = paste0(template_dir, file))
+    if (conf_type == "autosubmit") {
+      ## Section 1: autosubmit.conf
+      ## expid, email notifications and address
+      conf$config$EXPID <- expid
+      conf$mail$NOTIFICATIONS < recipe$Run$auto_conf$email_notifications
+      conf$mail$TO <- recipe$Run$auto_conf$email_address
+    } else if (conf_type == "expdef") {
+      ## Section 2: expdef
+      ## expid, hpcarch?, numchunks, project_type, project?, project_destination = auto-s2s
+      conf$DEFAULT$EXPID <- expid
+      # conf$experiment$NUMCHUNKS <- 
+      conf$local$PROJECT_PATH <- recipe$Run$code_dir
+    } else if (conf_type == "jobs") {
+      ## Section 3: jobs
+      ## wallclock, notify_on?, platform?, processors, retrials (none?)
+      conf$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock
+      if (recipe$Run$auto_conf$notify_completed) {
+        conf$verification$NOTIFY_ON <- paste(conf$verification$NOTIFY_ON,
+					     "COMPLETED")
+      }
+      if (recipe$Run$auto_conf$notify_failed) {
+	conf$verification$NOTIFY_ON <- paste(conf$verification$NOTIFY_ON,
+					     "FAILED")
+      }
+      conf$verification$MEMORY <- recipe$Run$auto_conf$memory
+      conf$verification$PROCESSORS <- recipe$Run$auto_conf$processors
+    } else if (conf_type = platform) {
+      ## Section 4: platform configuration
+      ## nord3v2 configuration... platform name? user, processors_per_node
+      # conf$nord3v2$USER <- 
+    } (conf_type = proj) {
+      ## Section 5: proj
+      ## modules? Info that goes on script, e.g. output directory
+      conf$common$OUTDIR <- outdir
+    }
+    write.config(conf, paste0(dest_dir, dest_file), "ini")
+}
-- 
GitLab


From 7318b4604c9865ad3d74c223b54356c3354287c2 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Mon, 13 Feb 2023 16:02:37 +0100
Subject: [PATCH 33/57] First version of tests to create a configuration for
 autosubmit and run it

---
 auto-verification.sh                      | 18 ++++++++++
 autosubmit/conf/autosubmit.conf           |  6 ++--
 autosubmit/conf/expdef.conf               |  6 ++--
 autosubmit/conf/jobs.conf                 |  3 +-
 autosubmit/conf/platforms.conf            |  9 -----
 modules/split.R                           | 12 +++++--
 recipes/tests/recipe_seasonal_example.yml | 25 +++++--------
 tools/divide_recipe.R                     |  3 +-
 tools/libs.R                              |  2 ++
 tools/write_autosubmit_conf.R             | 43 +++++++++++++++--------
 10 files changed, 73 insertions(+), 54 deletions(-)
 create mode 100644 auto-verification.sh

diff --git a/auto-verification.sh b/auto-verification.sh
new file mode 100644
index 00000000..c2328a72
--- /dev/null
+++ b/auto-verification.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+############ AUTOSUBMIT INPUTS ############
+proj_dir=%PROJDIR%
+outdir=%OUTDIR%
+CHUNK=%CHUNK%
+###############################
+
+## TODO: How to define the script
+cd $proj_dir
+
+script=modules/test_parallel_workflow.R
+atomic_recipe_number=$(printf "%02d" $CHUNK)
+atomic_recipe=${outdir}/logs/recipes/atomic_recipe_${atomic_recipe_number}.yml
+
+source MODULES
+
+Rscript ${script} ${atomic_recipe}
diff --git a/autosubmit/conf/autosubmit.conf b/autosubmit/conf/autosubmit.conf
index fd69390a..12723d14 100644
--- a/autosubmit/conf/autosubmit.conf
+++ b/autosubmit/conf/autosubmit.conf
@@ -4,13 +4,13 @@
 EXPID =
 # No need to change.
 # Autosubmit version identifier
-AUTOSUBMIT_VERSION = "3.14.0" 
+AUTOSUBMIT_VERSION = 3.14.0
 # Default maximum number of jobs to be waiting in any platform
 # Default = 3
-MAXWAITINGJOBS = 20
+MAXWAITINGJOBS = 16
 # Default maximum number of jobs to be running at the same time at any platform
 # Default = 6
-TOTALJOBS = 20
+TOTALJOBS = 16
 # Time (seconds) between connections to the HPC queue scheduler to poll already submitted jobs status
 # Default = 10
 SAFETYSLEEPTIME = 10
diff --git a/autosubmit/conf/expdef.conf b/autosubmit/conf/expdef.conf
index 6ea998d9..8cae214c 100644
--- a/autosubmit/conf/expdef.conf
+++ b/autosubmit/conf/expdef.conf
@@ -13,15 +13,13 @@ HPCARCH = nord3v2
 # DATELIST = 19600101 19650101 19700101
 # DATELIST = 1960[0101 0201 0301]
 # DATELIST = 19[60-65]
-DATELIST = 20201015
+DATELIST = 
 # Supply the list of members. Format fcX
 # You can also use an abreviated syntax for multiple members: fc[0 1 2] <=> fc0 fc1 fc2
 # fc[0-2] <=> fc0 fc1 fc2
 # MEMBERS = fc0 fc1 fc2 fc3 fc4
 #MEMBERS = fc4
-MEMBERS = fc[0-6]
-
-
+MEMBERS =
 # Chunk size unit. STRING = hour, day, month, year
 CHUNKSIZEUNIT = month
 # Chunk size. NUMERIC = 4, 6, 12
diff --git a/autosubmit/conf/jobs.conf b/autosubmit/conf/jobs.conf
index e2d7c56e..b4369031 100644
--- a/autosubmit/conf/jobs.conf
+++ b/autosubmit/conf/jobs.conf
@@ -1,9 +1,8 @@
 [verification]
-FILE = autosubmit/auto-verification.sh
+FILE = auto-verification.sh
 RUNNING = chunk
 WALLCLOCK = 
 NOTIFY_ON = 
 PLATFORM = nord3v2
-MEMORY = 
 PROCESSORS = 
 
diff --git a/autosubmit/conf/platforms.conf b/autosubmit/conf/platforms.conf
index 33825817..0f6819d0 100644
--- a/autosubmit/conf/platforms.conf
+++ b/autosubmit/conf/platforms.conf
@@ -1,12 +1,3 @@
-[power9]
-TYPE = slurm
-HOST = plogin1.bsc.es
-PROJECT = bsc32
-USER = bsc32036
-QUEUE = bsc_es
-SCRATCH_DIR = /gpfs/scratch
-ADD_PROJECT_TO_HOST = false
-
 [nord3v2]
 TYPE = slurm
 HOST = nord4.bsc.es
diff --git a/modules/split.R b/modules/split.R
index 58391808..ca6adca1 100644
--- a/modules/split.R
+++ b/modules/split.R
@@ -1,9 +1,15 @@
 suppressMessages(source("tools/libs.R"))
 
 # Retrieve recipe file path
-args = commandArgs(trailingOnly = TRUE)
-recipe_file <- args[1]
+# args = commandArgs(trailingOnly = TRUE)
+# recipe_file <- args[1]
+recipe_file <- "recipes/tests/recipe_seasonal_example.yml"
 # Check recipe and prepare output directories
 recipe <- prepare_outputs(recipe_file)
 # Split recipe into atomic recipes
-divide_recipe(recipe)
+## TODO: Add autosubmit yes/no to the parameters?
+run_parameters <- divide_recipe(recipe)
+if (recipe$Run$autosubmit) {
+  write_autosubmit_conf(recipe, run_parameters$n_atomic_recipes)
+}
+run_parameters$outdir
diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml
index 10eb0f90..f9d6a799 100644
--- a/recipes/tests/recipe_seasonal_example.yml
+++ b/recipes/tests/recipe_seasonal_example.yml
@@ -63,21 +63,12 @@ Run:
   autosubmit: yes
   # fill only if using autosubmit
   auto_conf:
-    expid: # if left empty, a new experiment will be created
-    wallclock: '04:00' # hh:mm
-    memory_per_job: '120000'
-    processors_per_job: 8
+    expid: a5no ## if left empty, create new exp?
+    hpc_user: bsc32762 # your hpc username
+    wallclock: 04:00 # hh:mm
+    processors_per_job: 8 # use ncores parameter?
     platform: nord3v2 # make this constant?
-    email_notifications: yes
-    notify_completed: no
-    notify_failed: yes
-    email_address: victoria.agudetse@bsc.es
- 
-################################################################################
-### Autosubmit CONFIGURATION
-#################################################################################
-Autosubmit:
-  use_autosubmit: yes
-  # fill only if using autosubmit
-  #
-  
+    email_notifications: yes # enable/disable email notifications
+    email_address: victoria.agudetse@bsc.es # email address for notifications
+    notify_completed: yes # notify me by email when a job finishes
+    notify_failed: yes # notify me by email when a job fails
diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R
index 2119eee7..d2a263bc 100644
--- a/tools/divide_recipe.R
+++ b/tools/divide_recipe.R
@@ -121,5 +121,6 @@ divide_recipe <- function(recipe) {
 		 "/logs/recipes/ to see all the individual atomic recipes.")
   info(recipe$Run$logger, text)
   ## TODO: Change returns?
-  return(recipe$Run$output_dir)
+  return(list(n_atomic_recipes = length(all_recipes),
+	      outdir = recipe$Run$output_dir))
 }
diff --git a/tools/libs.R b/tools/libs.R
index e3c3c277..826a1f19 100644
--- a/tools/libs.R
+++ b/tools/libs.R
@@ -14,6 +14,7 @@ library(lubridate)
 library(PCICt)
 library(RColorBrewer)
 library(grDevices)
+library(configr)
 #
 #  library(parallel)
 library(pryr) # To check mem usage.
@@ -33,4 +34,5 @@ source("tools/prepare_outputs.R")
 source("tools/divide_recipe.R")
 source("tools/data_summary.R")
 source("tools/read_atomic_recipe.R")
+source("tools/write_autosubmit_conf.R")
 # source("tools/add_dims.R") # Not sure if necessary yet
diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R
index 40614985..f6ad9318 100644
--- a/tools/write_autosubmit_conf.R
+++ b/tools/write_autosubmit_conf.R
@@ -1,9 +1,9 @@
-function write_autosubmit_conf(recipe, outdir) {
+write_autosubmit_conf <- function(recipe, nchunks) {
   # Function to write autosubmit configuration from an Auto-S2S recipe
+  expid <- recipe$Run$auto_conf$expid
   template_dir <- "autosubmit/conf/"
-  ## TODO: create folder for auto experiment
-  conf_dest_dir <- paste0("/esarchive/autosubmit/", expid "/conf/")
-
+  conf_dest_dir <- paste0("/esarchive/autosubmit/", expid, "/conf/")
+  # Modify the configuration files according to the info in the recipe
   for (file in list.files(template_dir)) {
     # Define destionation directory and file name accordin to expid
     conf_type <- strsplit(file, split = "[.]")[[1]][1]
@@ -16,17 +16,21 @@ function write_autosubmit_conf(recipe, outdir) {
       ## Section 1: autosubmit.conf
       ## expid, email notifications and address
       conf$config$EXPID <- expid
-      conf$mail$NOTIFICATIONS < recipe$Run$auto_conf$email_notifications
+      if (recipe$Run$auto_conf$email_notifications) {
+	conf$mail$NOTIFICATIONS <- "True"
+      } else {
+	conf$mail$NOTIFICATIONS <- "False"
+      }
       conf$mail$TO <- recipe$Run$auto_conf$email_address
     } else if (conf_type == "expdef") {
       ## Section 2: expdef
-      ## expid, hpcarch?, numchunks, project_type, project?, project_destination = auto-s2s
+      ## expid, numchunks, project_type?, project_destination = auto-s2s
       conf$DEFAULT$EXPID <- expid
-      # conf$experiment$NUMCHUNKS <- 
+      conf$experiment$NUMCHUNKS <- nchunks
       conf$local$PROJECT_PATH <- recipe$Run$code_dir
     } else if (conf_type == "jobs") {
       ## Section 3: jobs
-      ## wallclock, notify_on?, platform?, processors, retrials (none?)
+      ## wallclock, notify_on, platform?, processors
       conf$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock
       if (recipe$Run$auto_conf$notify_completed) {
         conf$verification$NOTIFY_ON <- paste(conf$verification$NOTIFY_ON,
@@ -36,16 +40,25 @@ function write_autosubmit_conf(recipe, outdir) {
 	conf$verification$NOTIFY_ON <- paste(conf$verification$NOTIFY_ON,
 					     "FAILED")
       }
-      conf$verification$MEMORY <- recipe$Run$auto_conf$memory
-      conf$verification$PROCESSORS <- recipe$Run$auto_conf$processors
-    } else if (conf_type = platform) {
+      conf$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores?
+    } else if (conf_type == "platforms") {
       ## Section 4: platform configuration
       ## nord3v2 configuration... platform name? user, processors_per_node
-      # conf$nord3v2$USER <- 
-    } (conf_type = proj) {
+      conf$nord3v2$USER <- recipe$Run$auto_conf$hpc_user
+    } else if (conf_type == "proj") {
       ## Section 5: proj
       ## modules? Info that goes on script, e.g. output directory
-      conf$common$OUTDIR <- outdir
+      conf$common$OUTDIR <- recipe$Run$output_dir
     }
-    write.config(conf, paste0(dest_dir, dest_file), "ini")
+    # Write config file inside autosubmit dir
+    write.config(conf, paste0(conf_dest_dir, dest_file), write.type = "ini")
+    Sys.chmod(paste0(conf_dest_dir, dest_file), mode = "755", use_umask = F)
+  }
+  info(recipe$Run$logger,
+       paste("##### AUTOSUBMIT CONFIGURATION WRITTEN FOR", expid, "#####"))
+  print(paste("Please SSH into bscesautosubmit01 or bscesautosubmit02 and run",
+	      "the following commands:"))
+  print("module load autosubmit/3.14.0-foss-2015a-Python-2.7.9")
+  print(paste("autosubmit create", expid))
+  print(paste("nohup autosubmit run", expid, "& disown"))
 }
-- 
GitLab


From 4259c3e6ef4b0285e8cf7e98ea2d188b5aee3c95 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Mon, 13 Feb 2023 16:40:14 +0100
Subject: [PATCH 34/57] Add DATELIST and MEMBER

---
 autosubmit/conf/expdef.conf   | 4 ++--
 tools/write_autosubmit_conf.R | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/autosubmit/conf/expdef.conf b/autosubmit/conf/expdef.conf
index 8cae214c..d3a3370a 100644
--- a/autosubmit/conf/expdef.conf
+++ b/autosubmit/conf/expdef.conf
@@ -13,13 +13,13 @@ HPCARCH = nord3v2
 # DATELIST = 19600101 19650101 19700101
 # DATELIST = 1960[0101 0201 0301]
 # DATELIST = 19[60-65]
-DATELIST = 
+DATELIST =  
 # Supply the list of members. Format fcX
 # You can also use an abreviated syntax for multiple members: fc[0 1 2] <=> fc0 fc1 fc2
 # fc[0-2] <=> fc0 fc1 fc2
 # MEMBERS = fc0 fc1 fc2 fc3 fc4
 #MEMBERS = fc4
-MEMBERS =
+MEMBERS = fc0
 # Chunk size unit. STRING = hour, day, month, year
 CHUNKSIZEUNIT = month
 # Chunk size. NUMERIC = 4, 6, 12
diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R
index f6ad9318..bc99a713 100644
--- a/tools/write_autosubmit_conf.R
+++ b/tools/write_autosubmit_conf.R
@@ -26,6 +26,7 @@ write_autosubmit_conf <- function(recipe, nchunks) {
       ## Section 2: expdef
       ## expid, numchunks, project_type?, project_destination = auto-s2s
       conf$DEFAULT$EXPID <- expid
+      conf$experiment$DATELIST <- format(Sys.Date(), "%Y%m%d")
       conf$experiment$NUMCHUNKS <- nchunks
       conf$local$PROJECT_PATH <- recipe$Run$code_dir
     } else if (conf_type == "jobs") {
-- 
GitLab


From 4b46ded058ad3cd78e20c6a8e2a121c294946a92 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 14 Feb 2023 17:03:25 +0100
Subject: [PATCH 35/57] Change 'archive' to 'esarchive' in YAML config; remove
 code_dir absolute path

---
 conf/archive.yml                      | 4 +---
 conf/archive_decadal.yml              | 2 +-
 modules/Loading/Loading.R             | 3 ++-
 modules/Loading/Loading_decadal.R     | 3 ++-
 modules/Saving/Saving.R               | 8 ++++----
 modules/Visualization/Visualization.R | 8 ++++----
 6 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/conf/archive.yml b/conf/archive.yml
index 0251beaf..52f08f1d 100644
--- a/conf/archive.yml
+++ b/conf/archive.yml
@@ -1,6 +1,4 @@
-
-
-archive:
+esarchive:
     src: "/esarchive/"
     System:
         system5c3s:
diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml
index 2b74bff8..91637024 100644
--- a/conf/archive_decadal.yml
+++ b/conf/archive_decadal.yml
@@ -1,4 +1,4 @@
-archive:
+esarchive:
     src: "/esarchive/" 
     System:
 # ---- 
diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R
index 40af39dd..7db2a3e9 100644
--- a/modules/Loading/Loading.R
+++ b/modules/Loading/Loading.R
@@ -48,7 +48,8 @@ load_datasets <- function(recipe) {
   ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name
   
   # get esarchive datasets dict:
-  archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
+  ## TODO: Adapt to 'filesystem' option in recipe
+  archive <- read_yaml("conf/archive.yml")$esarchive
   exp_descrip <- archive$System[[exp.name]]
   
   freq.hcst <- unlist(exp_descrip[[store.freq]][variable])
diff --git a/modules/Loading/Loading_decadal.R b/modules/Loading/Loading_decadal.R
index 0a95d9a4..43f2276f 100644
--- a/modules/Loading/Loading_decadal.R
+++ b/modules/Loading/Loading_decadal.R
@@ -21,7 +21,8 @@ source("tools/tmp/as.s2dv_cube.R")
 
 load_datasets <- function(recipe) {
 
-  archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
+  ## 
+  archive <- read_yaml(paste0("conf/archive_decadal.yml"))$esarchive
 
   # Print Start() info or not
   DEBUG <- FALSE
diff --git a/modules/Saving/Saving.R b/modules/Saving/Saving.R
index 28b5e552..608c1faf 100644
--- a/modules/Saving/Saving.R
+++ b/modules/Saving/Saving.R
@@ -28,11 +28,11 @@ save_data <- function(recipe, data,
   }
   if (is.null(archive)) {
     if (tolower(recipe$Analysis$Horizon) == "seasonal") {
-      archive <- read_yaml(paste0(recipe$Run$code_dir,
-	                          "conf/archive.yml"))$archive
+      archive <- 
+	read_yaml(paste0("conf/archive.yml"))[[recipe$Run$filesystem]]
     } else if (tolower(recipe$Analysis$Horizon) == "decadal") {
-      archive <- read_yaml(paste0(recipe$Run$code_dir,
-				  "conf/archive_decadal.yml"))$archive
+      archive <- 
+	read_yaml(paste0("conf/archive_decadal.yml"))[[recipe$Run$filesystem]]
     }
   }
   dict <- read_yaml("conf/variable-dictionary.yml")
diff --git a/modules/Visualization/Visualization.R b/modules/Visualization/Visualization.R
index a8664569..839060ac 100644
--- a/modules/Visualization/Visualization.R
+++ b/modules/Visualization/Visualization.R
@@ -34,11 +34,11 @@ plot_data <- function(recipe,
 
   if (is.null(archive)) {
     if (tolower(recipe$Analysis$Horizon) == "seasonal") {
-      archive <- read_yaml(paste0(recipe$Run$code_dir,
-				  "conf/archive.yml"))$archive
+      archive <- 
+	read_yaml(paste0("conf/archive.yml"))[[recipe$Run$filesystem]]
     } else if (tolower(recipe$Analysis$Horizon) == "decadal") {
-      archive <- read_yaml(paste0(recipe$Run$code_dir,
-				  "conf/archive_decadal.yml"))$archive
+      archive <- 
+	read_yaml(paste0("conf/archive_decadal.yml"))[[recipe$Run$filesystem]]
     }
   }
 
-- 
GitLab


From 6b27c413efb30cf6945614f856064b401b560113 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 14 Feb 2023 17:11:42 +0100
Subject: [PATCH 36/57] Include autosubmit recipe checks; generalize autosubmit
 code for use outside of BSC environment

---
 .../{conf => conf_esarchive}/autosubmit.conf  |  2 +-
 .../{conf => conf_esarchive}/expdef.conf      |  0
 autosubmit/{conf => conf_esarchive}/jobs.conf |  0
 .../{conf => conf_esarchive}/platforms.conf   |  0
 autosubmit/{conf => conf_esarchive}/proj.conf |  0
 conf/autosubmit.yml                           |  5 ++
 recipes/tests/recipe_seasonal_example.yml     |  3 +-
 tests/testthat/test-seasonal_daily.R          |  2 +-
 tests/testthat/test-seasonal_monthly.R        |  2 +-
 tools/check_recipe.R                          | 54 +++++++++++++++++++
 tools/divide_recipe.R                         |  2 +-
 tools/prepare_outputs.R                       |  3 ++
 tools/write_autosubmit_conf.R                 | 53 +++++++++++-------
 13 files changed, 103 insertions(+), 23 deletions(-)
 rename autosubmit/{conf => conf_esarchive}/autosubmit.conf (97%)
 rename autosubmit/{conf => conf_esarchive}/expdef.conf (100%)
 rename autosubmit/{conf => conf_esarchive}/jobs.conf (100%)
 rename autosubmit/{conf => conf_esarchive}/platforms.conf (100%)
 rename autosubmit/{conf => conf_esarchive}/proj.conf (100%)
 create mode 100644 conf/autosubmit.yml

diff --git a/autosubmit/conf/autosubmit.conf b/autosubmit/conf_esarchive/autosubmit.conf
similarity index 97%
rename from autosubmit/conf/autosubmit.conf
rename to autosubmit/conf_esarchive/autosubmit.conf
index 12723d14..685876a1 100644
--- a/autosubmit/conf/autosubmit.conf
+++ b/autosubmit/conf_esarchive/autosubmit.conf
@@ -35,7 +35,7 @@ API = paramiko
 # A PICKLE file (pkl) or an SQLite database (db). Default = pkl
 TYPE = pkl
 # Defines if the remote logs will be copied to the local platform. Default = True.
-COPY_REMOTE_LOGS = False
+COPY_REMOTE_LOGS = True
 
 [migrate]
 # Changes experiment files owner.
diff --git a/autosubmit/conf/expdef.conf b/autosubmit/conf_esarchive/expdef.conf
similarity index 100%
rename from autosubmit/conf/expdef.conf
rename to autosubmit/conf_esarchive/expdef.conf
diff --git a/autosubmit/conf/jobs.conf b/autosubmit/conf_esarchive/jobs.conf
similarity index 100%
rename from autosubmit/conf/jobs.conf
rename to autosubmit/conf_esarchive/jobs.conf
diff --git a/autosubmit/conf/platforms.conf b/autosubmit/conf_esarchive/platforms.conf
similarity index 100%
rename from autosubmit/conf/platforms.conf
rename to autosubmit/conf_esarchive/platforms.conf
diff --git a/autosubmit/conf/proj.conf b/autosubmit/conf_esarchive/proj.conf
similarity index 100%
rename from autosubmit/conf/proj.conf
rename to autosubmit/conf_esarchive/proj.conf
diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml
new file mode 100644
index 00000000..3f852776
--- /dev/null
+++ b/conf/autosubmit.yml
@@ -0,0 +1,5 @@
+esarchive:
+  platform: nord3v2
+  module_version: autosubmit/3.14.0-foss-2015a-Python-2.7.9
+  experiment_dir: /esarchive/autosubmit/
+  userID: bsc32
diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml
index f9d6a799..8b40d21a 100644
--- a/recipes/tests/recipe_seasonal_example.yml
+++ b/recipes/tests/recipe_seasonal_example.yml
@@ -58,6 +58,7 @@ Analysis:
 Run:
   Loglevel: INFO
   Terminal: yes
+  filesystem: esarchive
   output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/
   code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/
   autosubmit: yes
@@ -70,5 +71,5 @@ Run:
     platform: nord3v2 # make this constant?
     email_notifications: yes # enable/disable email notifications
     email_address: victoria.agudetse@bsc.es # email address for notifications
-    notify_completed: yes # notify me by email when a job finishes
+    notify_completed: no # notify me by email when a job finishes
     notify_failed: yes # notify me by email when a job fails
diff --git a/tests/testthat/test-seasonal_daily.R b/tests/testthat/test-seasonal_daily.R
index ddcca22f..53a235e7 100644
--- a/tests/testthat/test-seasonal_daily.R
+++ b/tests/testthat/test-seasonal_daily.R
@@ -6,7 +6,7 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 
 recipe_file <- "tests/recipes/recipe-seasonal_daily_1.yml"
-recipe <- prepare_outputs(recipe_file)
+recipe <- prepare_outputs(recipe_file, disable_checks = T)
 # Load datasets
 suppressWarnings({invisible(capture.output(
 data <- load_datasets(recipe)
diff --git a/tests/testthat/test-seasonal_monthly.R b/tests/testthat/test-seasonal_monthly.R
index de03bf73..4ea7f9ab 100644
--- a/tests/testthat/test-seasonal_monthly.R
+++ b/tests/testthat/test-seasonal_monthly.R
@@ -7,7 +7,7 @@ source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
 recipe_file <- "tests/recipes/recipe-seasonal_monthly_1.yml"
-recipe <- prepare_outputs(recipe_file)
+recipe <- prepare_outputs(recipe_file, disable_checks = T)
 archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 
 # Load datasets
diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index 67ba4d0d..521db407 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -310,6 +310,60 @@ check_recipe <- function(recipe) {
     error_status <- T
   }
 
+  # ---------------------------------------------------------------------
+  # AUTOSUBMIT CHECKS
+  # ---------------------------------------------------------------------
+
+  AUTO_PARAMS <- c("expid", "hpc_user", "wallclock", "processors_per_job",
+		   "platform", "email_notifications", "email_address",
+		   "notify_completed", "notify_failed")
+
+  if (recipe$Run$autosubmit) {
+    # Read autosubmit info for the specific filesystem (e.g. esarchive)
+    auto_specs <- read_yaml("conf/autosubmit.yml")[[recipe$Run$filesystem]]
+    # Check that the autosubmit configuration parameters are present
+    if (!("auto_conf" %in% names(recipe$Run))) {
+      error(recipe$Run$logger,
+	    "The 'auto_conf' is missing from the 'Run' section of the recipe.")
+      error_status <- T
+    } else if (!all(AUTO_PARAMS %in% names(recipe$Run$auto_conf))) {
+      error(recipe$Run$logger,
+	    paste0("The element 'Run:auto_conf' must contain all of the ",
+		   "following: ", paste(AUTO_PARAMS, collapse = ", "), "."))
+      error_status <- T
+    }
+    # Check that the experiment ID exists
+    if (is.null(recipe$Run$auto_conf$expid)) {
+      error(recipe$Run$logger,
+	    paste("The Autosubmit EXPID is missing. You can create one by",
+		  "running the following commands on the autosubmit machine:"))
+      error(recipe$Run$logger,
+	    paste("module load", auto_specs$module_version))
+      error(recipe$Run$logger,
+	    paste("autosubmit expid -H", auto_specs$platform,
+		  "-d <Description>"))
+    } else if (!dir.exists(paste0(auto_specs$experiment_dir,
+				  recipe$Run$auto_conf$expid))) {
+      error(recipe$Run$logger,
+	    paste0("No folder in ", auto_specs$experiment_dir,
+		   " for the EXPID", recipe$Run$auto_conf$expid,
+		   ". Please make sure it is correct."))
+    }
+    if ((recipe$Run$auto_conf$email_notifications) &&
+	(is.null(recipe$Run$auto_conf$email_address))) {
+      error(recipe$Run$logger,
+	    "Autosubmit notifications are enabled but email address is empty!")
+    }
+    if (is.null(recipe$Run$auto_conf$hpc_user)) {
+      error(recipe$Run$logger,
+	    "The 'Run:auto_conf:hpc_user' field can not be empty.")
+    } else if ((recipe$Run$filesystem == "esarchive") &&
+	       (!substr(recipe$Run$auto_conf$hpc_user, 1, 5) == "bsc32")) {
+      error(recipe$Run$logger,
+	    "Please check your hpc_user ID. It should look like: 'bsc32xxx'")
+    }
+  }
+
   # ---------------------------------------------------------------------
   # WORKFLOW CHECKS
   # ---------------------------------------------------------------------
diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R
index d2a263bc..403304c4 100644
--- a/tools/divide_recipe.R
+++ b/tools/divide_recipe.R
@@ -19,7 +19,7 @@ divide_recipe <- function(recipe) {
                                       Output_format = 
 				        recipe$Analysis$Output_format),
 		      Run = recipe$Run[c("Loglevel", "output_dir", "Terminal",
-					 "code_dir", "logfile")])
+					 "code_dir", "logfile", "filesystem")])
   
   # duplicate recipe by independent variables:
   all_recipes <- rep(list(beta_recipe), length(recipe$Analysis$Variables))
diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R
index c9bdd74c..85be5c46 100644
--- a/tools/prepare_outputs.R
+++ b/tools/prepare_outputs.R
@@ -30,6 +30,9 @@ prepare_outputs <- function(recipe_file,
   recipe <- read_yaml(recipe_file)
   recipe$recipe_path <- recipe_file
   recipe$name <- tools::file_path_sans_ext(basename(recipe_file))
+  if (is.null(recipe$Run$filesystem)) {
+    recipe$Run$filesystem <- "esarchive"
+  }
   
   output_dir = recipe$Run$output_dir
   # Create output folders
diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R
index bc99a713..097b2afd 100644
--- a/tools/write_autosubmit_conf.R
+++ b/tools/write_autosubmit_conf.R
@@ -1,19 +1,22 @@
+# Function to write autosubmit configuration from an Auto-S2S recipe
 write_autosubmit_conf <- function(recipe, nchunks) {
-  # Function to write autosubmit configuration from an Auto-S2S recipe
+  # Experiment ID
   expid <- recipe$Run$auto_conf$expid
-  template_dir <- "autosubmit/conf/"
-  conf_dest_dir <- paste0("/esarchive/autosubmit/", expid, "/conf/")
+  # Directory with the experiment templates
+  template_dir <- paste0("autosubmit/conf_", recipe$Run$filesystem, "/")
+  # Read autosubmit info for the specific filesystem (e.g. esarchive)
+  auto_specs <- read_yaml("conf/autosubmit.yml")[[recipe$Run$filesystem]]
+  # Output directory
+  dest_dir <- paste0(auto_specs$experiment_dir, expid, "/conf/")
   # Modify the configuration files according to the info in the recipe
   for (file in list.files(template_dir)) {
-    # Define destionation directory and file name accordin to expid
     conf_type <- strsplit(file, split = "[.]")[[1]][1]
     extension <- strsplit(file, split = "[.]")[[1]][2]
     dest_file <- paste0(conf_type, "_", expid, ".", extension)
-    dest_dir <- paste0("/esarchive/autosubmit/", expid, "/")
     # Read configuration file
     conf <- read.config(file = paste0(template_dir, file))
     if (conf_type == "autosubmit") {
-      ## Section 1: autosubmit.conf
+      # Section 1: autosubmit.conf
       ## expid, email notifications and address
       conf$config$EXPID <- expid
       if (recipe$Run$auto_conf$email_notifications) {
@@ -23,14 +26,14 @@ write_autosubmit_conf <- function(recipe, nchunks) {
       }
       conf$mail$TO <- recipe$Run$auto_conf$email_address
     } else if (conf_type == "expdef") {
-      ## Section 2: expdef
+      # Section 2: expdef
       ## expid, numchunks, project_type?, project_destination = auto-s2s
       conf$DEFAULT$EXPID <- expid
       conf$experiment$DATELIST <- format(Sys.Date(), "%Y%m%d")
       conf$experiment$NUMCHUNKS <- nchunks
       conf$local$PROJECT_PATH <- recipe$Run$code_dir
     } else if (conf_type == "jobs") {
-      ## Section 3: jobs
+      # Section 3: jobs
       ## wallclock, notify_on, platform?, processors
       conf$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock
       if (recipe$Run$auto_conf$notify_completed) {
@@ -43,23 +46,37 @@ write_autosubmit_conf <- function(recipe, nchunks) {
       }
       conf$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores?
     } else if (conf_type == "platforms") {
-      ## Section 4: platform configuration
+      # Section 4: platform configuration
       ## nord3v2 configuration... platform name? user, processors_per_node
-      conf$nord3v2$USER <- recipe$Run$auto_conf$hpc_user
+      conf[[auto_specs$platform]]$USER <- recipe$Run$auto_conf$hpc_user
     } else if (conf_type == "proj") {
-      ## Section 5: proj
+      # Section 5: proj
       ## modules? Info that goes on script, e.g. output directory
       conf$common$OUTDIR <- recipe$Run$output_dir
     }
     # Write config file inside autosubmit dir
-    write.config(conf, paste0(conf_dest_dir, dest_file), write.type = "ini")
-    Sys.chmod(paste0(conf_dest_dir, dest_file), mode = "755", use_umask = F)
+    write.config(conf, paste0(dest_dir, dest_file), write.type = "ini")
+    Sys.chmod(paste0(dest_dir, dest_file), mode = "755", use_umask = F)
   }
   info(recipe$Run$logger,
        paste("##### AUTOSUBMIT CONFIGURATION WRITTEN FOR", expid, "#####"))
-  print(paste("Please SSH into bscesautosubmit01 or bscesautosubmit02 and run",
-	      "the following commands:"))
-  print("module load autosubmit/3.14.0-foss-2015a-Python-2.7.9")
-  print(paste("autosubmit create", expid))
-  print(paste("nohup autosubmit run", expid, "& disown"))
+  # Print instructions/commands for user
+  if (recipe$Run$Terminal) {
+    ## TODO: Change SSH message for other environments (outside BSC)
+    info(recipe$Run$logger,
+	 paste("Please SSH into bscesautosubmit01 or bscesautosubmit02 and run",
+	       "the following commands:"))
+    info(recipe$Run$logger,
+	 paste("module load", auto_specs$module_version))
+    info(recipe$Run$logger,
+	 paste("autosubmit create", expid))
+    info(recipe$Run$logger,
+	 paste("nohup autosubmit run", expid, "& disown"))
+  } else {
+    print(paste("Please SSH into bscesautosubmit01 or bscesautosubmit02 and run",
+  	        "the following commands:"))
+    print(paste("module load", auto_specs$module_version))
+    print(paste("autosubmit create", expid))
+    print(paste("nohup autosubmit run", expid, "& disown"))
+  }
 }
-- 
GitLab


From f8bbaa4a8adc15b8b01223c35c058236994dd0e6 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 23 Feb 2023 15:43:51 +0100
Subject: [PATCH 37/57] Add message about conf dir

---
 tools/write_autosubmit_conf.R | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R
index 097b2afd..8e6c6ba1 100644
--- a/tools/write_autosubmit_conf.R
+++ b/tools/write_autosubmit_conf.R
@@ -60,6 +60,9 @@ write_autosubmit_conf <- function(recipe, nchunks) {
   }
   info(recipe$Run$logger,
        paste("##### AUTOSUBMIT CONFIGURATION WRITTEN FOR", expid, "#####"))
+  info(recipe$Run$logger,
+       paste0("You can check your experiment configuration at: ",
+	      "/esarchive/autosubmit/", expid, "/conf/"))
   # Print instructions/commands for user
   if (recipe$Run$Terminal) {
     ## TODO: Change SSH message for other environments (outside BSC)
-- 
GitLab


From 0c11cc44ce155aa1a2c0ff351a47c2ecfb485256 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Mon, 27 Feb 2023 09:21:43 +0100
Subject: [PATCH 38/57] Move scripts and update example recipe

---
 .../auto-verification.sh                             |  0
 autosubmit/conf_esarchive/jobs.conf                  |  2 +-
 recipes/tests/recipe_seasonal_example.yml            | 12 ++++++------
 modules/split.R => split.R                           |  0
 4 files changed, 7 insertions(+), 7 deletions(-)
 rename auto-verification.sh => autosubmit/auto-verification.sh (100%)
 rename modules/split.R => split.R (100%)

diff --git a/auto-verification.sh b/autosubmit/auto-verification.sh
similarity index 100%
rename from auto-verification.sh
rename to autosubmit/auto-verification.sh
diff --git a/autosubmit/conf_esarchive/jobs.conf b/autosubmit/conf_esarchive/jobs.conf
index b4369031..88f3565c 100644
--- a/autosubmit/conf_esarchive/jobs.conf
+++ b/autosubmit/conf_esarchive/jobs.conf
@@ -1,5 +1,5 @@
 [verification]
-FILE = auto-verification.sh
+FILE = autosubmit/auto-verification.sh
 RUNNING = chunk
 WALLCLOCK = 
 NOTIFY_ON = 
diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml
index 8b40d21a..8c38b8bc 100644
--- a/recipes/tests/recipe_seasonal_example.yml
+++ b/recipes/tests/recipe_seasonal_example.yml
@@ -17,11 +17,11 @@ Analysis:
     - {name: prlr, freq: monthly_mean}
   Datasets:
     System: # multiple systems for single model, split if Multimodel = F
-      - {name: system5c3s}
-      - {name: system7c3s}
+      - {name: ECMWF-SEAS5}
+      - {name: Meteo-France-System7}
     Multimodel: False # single option
     Reference:
-      - {name: era5} # multiple references for single model?
+      - {name: ERA5} # multiple references for single model?
   Time:
     sdate: # list, split
       - '0101'
@@ -30,11 +30,11 @@ Analysis:
     hcst_start: '2000' # single option
     hcst_end: '2016' # single option
     ftime_min: 1 # single option
-    ftime_max: 3 # single option
+    ftime_max: 6 # single option
   Region: # multiple lists, Add region name if there is more than 1 region
     - {latmin: -5, latmax: 5, lonmin: -10, lonmax: 10}
   Regrid:
-    method: bilinear ## TODO: allow multiple methods?
+    method: conservative ## TODO: allow multiple methods?
     type: to_system
   Workflow:
     Anomalies:
@@ -48,7 +48,7 @@ Analysis:
       percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]] # list, don't split
     Indicators:
       index: no # ? 
-  ncores: 7
+  ncores: 8
   remove_NAs: yes # bool, don't split
   Output_format: Scorecards # string, don't split
 
diff --git a/modules/split.R b/split.R
similarity index 100%
rename from modules/split.R
rename to split.R
-- 
GitLab


From 3612aba02819687dd4819b5858262e1644bd8842 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Mon, 27 Feb 2023 09:40:30 +0100
Subject: [PATCH 39/57] Change output format in example

---
 recipes/tests/recipe_seasonal_example.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml
index 8c38b8bc..a866bf5c 100644
--- a/recipes/tests/recipe_seasonal_example.yml
+++ b/recipes/tests/recipe_seasonal_example.yml
@@ -50,7 +50,7 @@ Analysis:
       index: no # ? 
   ncores: 8
   remove_NAs: yes # bool, don't split
-  Output_format: Scorecards # string, don't split
+  Output_format: S2S4E # string, don't split
 
 ################################################################################
 ## Run CONFIGURATION
-- 
GitLab


From 2e647dad4193b29dcaab865342dcc9a70af300bb Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 2 Mar 2023 08:37:19 +0100
Subject: [PATCH 40/57] Remove hardcoded recipe

---
 split.R | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/split.R b/split.R
index ca6adca1..8e33f916 100644
--- a/split.R
+++ b/split.R
@@ -1,9 +1,9 @@
 suppressMessages(source("tools/libs.R"))
 
 # Retrieve recipe file path
-# args = commandArgs(trailingOnly = TRUE)
-# recipe_file <- args[1]
-recipe_file <- "recipes/tests/recipe_seasonal_example.yml"
+args = commandArgs(trailingOnly = TRUE)
+recipe_file <- args[1]
+# recipe_file <- "recipes/tests/recipe_seasonal_example.yml"
 # Check recipe and prepare output directories
 recipe <- prepare_outputs(recipe_file)
 # Split recipe into atomic recipes
-- 
GitLab


From 3b47c23bd99905c2426da3bc56940aca998d3270 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Fri, 17 Mar 2023 13:02:32 +0100
Subject: [PATCH 41/57] Modify saving functions to retrieve archive from
 recipe, rearrange parameters

---
 modules/Saving/Saving.R | 118 +++++++++++++++++-----------------------
 tools/get_archive.R     |  12 ++++
 tools/libs.R            |   1 +
 3 files changed, 63 insertions(+), 68 deletions(-)
 create mode 100644 tools/get_archive.R

diff --git a/modules/Saving/Saving.R b/modules/Saving/Saving.R
index 608c1faf..0a5cce93 100644
--- a/modules/Saving/Saving.R
+++ b/modules/Saving/Saving.R
@@ -4,8 +4,7 @@ source("modules/Saving/paths2save.R")
 
 save_data <- function(recipe, data,
 		      skill_metrics = NULL,
-		      probabilities = NULL,
-		      archive = NULL) {
+		      probabilities = NULL) {
   # Wrapper for the saving functions.
   # recipe: The auto-s2s recipe
   # archive: The auto-s2s archive
@@ -15,6 +14,7 @@ save_data <- function(recipe, data,
   # probabilities: output of compute_probabilities()
   # mean_bias: output of compute_mean_bias()
 
+  # Sanity checks
   if (is.null(recipe)) {
     error(recipe$Run$logger, "The 'recipe' parameter is mandatory.")
     stop()
@@ -26,29 +26,18 @@ save_data <- function(recipe, data,
 		"of at least two s2dv_cubes containing the hcst and obs."))
     stop()
   }
-  if (is.null(archive)) {
-    if (tolower(recipe$Analysis$Horizon) == "seasonal") {
-      archive <- 
-	read_yaml(paste0("conf/archive.yml"))[[recipe$Run$filesystem]]
-    } else if (tolower(recipe$Analysis$Horizon) == "decadal") {
-      archive <- 
-	read_yaml(paste0("conf/archive_decadal.yml"))[[recipe$Run$filesystem]]
-    }
-  }
-  dict <- read_yaml("conf/variable-dictionary.yml")
-
   # Create output directory
   outdir <- get_dir(recipe)
   dir.create(outdir, showWarnings = FALSE, recursive = TRUE)
 
   # Export hindcast, forecast and observations onto outfile
-  save_forecast(data$hcst, recipe, dict, outdir, archive = archive,
+  save_forecast(recipe = recipe, data_cube = data$hcst, outdir = outdir,
 		type = 'hcst')
   if (!is.null(data$fcst)) {
-    save_forecast(data$fcst, recipe, dict, outdir,
-		  archive = archive, type = 'fcst')
+    save_forecast(recipe = recipe, data_cube = data$fcst, outdir =  outdir,
+		  type = 'fcst')
   }
-  save_observations(data$obs, recipe, dict, outdir, archive = archive)
+  save_observations(recipe = recipe, data_cube = data$obs, outdir = outdir)
 
   # Separate ensemble correlation from the rest of the metrics, as it has one
   # extra dimension "ensemble" and must be saved to a different file
@@ -65,23 +54,30 @@ save_data <- function(recipe, data,
 
   # Export skill metrics onto outfile
   if (!is.null(skill_metrics)) {
-    save_metrics(skill_metrics, recipe, dict, data$hcst, outdir,
-		 archive = archive)
+    save_metrics(recipe = recipe, skill = skill_metrics,
+		 data_cube = data$hcst,
+		 outdir = outdir)
   }
   if (!is.null(corr_metrics)) {
-    save_corr(corr_metrics, recipe, dict, data$hcst, outdir,
-	      archive = archive)
+    save_corr(recipe = recipe, skill = corr_metrics,
+	      data_cube = data$hcst,
+	      outdir = outdir)
   }
   
   # Export probabilities onto outfile
   if (!is.null(probabilities)) {
-    save_percentiles(probabilities$percentiles, recipe, data$hcst, outdir, 
-		     archive = archive)
-    save_probabilities(probabilities$probs, recipe, data$hcst, outdir, 
-		       archive = archive, type = "hcst")
+    save_percentiles(recipe = recipe, percentiles = probabilities$percentiles,
+		     data_cube = data$hcst,
+		     outdir = outdir)
+    save_probabilities(recipe = recipe, probs = probabilities$probs,
+		       data_cube = data$hcst, 
+		       outdir = outdir,
+		       type = "hcst")
     if (!is.null(probabilities$probs_fcst)) {
-      save_probabilities(probabilities$probs_fcst, recipe, data$fcst, outdir,
-		         archive = archive, type = "fcst")
+      save_probabilities(recipe = recipe, probs = probabilities$probs_fcst, 
+			 data_cube = data$fcst,
+			 outdir = outdir,
+		         type = "fcst")
     }
   }
 }
@@ -156,12 +152,10 @@ get_latlon <- function(latitude, longitude) {
 
 }
 
-save_forecast <- function(data_cube,
-			  recipe,
-			  dictionary,
+save_forecast <- function(recipe,
+			  data_cube,
                           outdir,
                           agg = "global",
-                          archive = NULL,
                           type = NULL) {
   # Loops over the years in the s2dv_cube containing a hindcast or forecast
   # and exports each year to a netCDF file.
@@ -171,7 +165,8 @@ save_forecast <- function(data_cube,
   # agg: aggregation, "global" or "country"
 
   lalo <- c('longitude', 'latitude')
-  
+  archive <- get_archive(recipe)
+  dictionary <- read_yaml("conf/variable-dictionary.yml")
   variable <- data_cube$Variable$varName
   var.longname <- attr(data_cube$Variable, 'variable')$long_name
   global_attributes <- get_global_attributes(recipe, archive)
@@ -179,23 +174,16 @@ save_forecast <- function(data_cube,
   store.freq <- recipe$Analysis$Variables$freq
   calendar <- archive$System[[global_attributes$system]]$calendar
 
-#  if (fcst.horizon == "seasonal") {
-#    calendar <- attr(data_cube$Variable, "variable")$dim$time$calendar
-#  } else {
-#    calendar <- attr(data_cube$Variable, "variable")$dim[[3]]$calendar
-#  }
-  
   # Generate vector containing leadtimes
   dates <- as.PCICt(ClimProjDiags::Subset(data_cube$Dates$start, 'syear', 1),
 		    cal = calendar)
   if (fcst.horizon == 'decadal') {
     ## Method 1: Use the first date as init_date. But it may be better to use 
     ## the real initialized date (ask users)
-#    init_date <- as.Date(data_cube$Dates$start[1], format = '%Y%m%d')
+    # init_date <- as.Date(data_cube$Dates$start[1], format = '%Y%m%d')
     ## Method 2: use initial month
     init_month <- archive$System[[recipe$Analysis$Datasets$System$name]]$initial_month
     if (type == 'hcst') {
-    ## PROBLEM for fcst!!!!!!!!!!!!
       init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, '-', 
                                    sprintf('%02d', init_month), '-01'),
 			    cal = calendar)
@@ -299,12 +287,10 @@ save_forecast <- function(data_cube,
 }
 
 
-save_observations <- function(data_cube,
-			      recipe,
-			      dictionary,
+save_observations <- function(recipe,
+			      data_cube,
                               outdir,
-                              agg = "global",
-                              archive = NULL) {
+                              agg = "global") {
   # Loops over the years in the s2dv_cube containing the observations and
   # exports each year to a netCDF file.
   # data_cube: s2dv_cube containing the data and metadata
@@ -313,7 +299,8 @@ save_observations <- function(data_cube,
   # agg: aggregation, "global" or "country"
 
   lalo <- c('longitude', 'latitude')
-  
+  archive <- get_archive(recipe)
+  dictionary <- read_yaml("conf/variable-dictionary.yml")
   variable <- data_cube$Variable$varName
   var.longname <- attr(data_cube$Variable, 'variable')$long_name
   global_attributes <- get_global_attributes(recipe, archive)
@@ -443,24 +430,23 @@ save_observations <- function(data_cube,
 #                lat=attr(var.obs, 'Variables')$dat1$latitude)
 # }
 
-save_metrics <- function(skill,
-			 recipe,
-			 dictionary,
+save_metrics <- function(recipe,
+			 skill,
                          data_cube,
                          outdir,
-                         agg = "global",
-                         archive = NULL) {
+                         agg = "global") {
   # This function adds metadata to the skill metrics in 'skill'
   # and exports them to a netCDF file inside 'outdir'.
   
   # Define grid dimensions and names
   lalo <- c('longitude', 'latitude')
+  archive <- get_archive(recipe)
+  dictionary <- read_yaml("conf/variable-dictionary.yml")
   # Remove singleton dimensions and rearrange lon, lat and time dims 
   if (tolower(agg) == "global") {
     skill <- lapply(skill, function(x) {
                     Reorder(x, c(lalo, 'time'))})
   }
-
   # Add global and variable attributes
   global_attributes <- get_global_attributes(recipe, archive)
   ## TODO: Sort out the logic once default behavior is decided
@@ -559,16 +545,16 @@ save_metrics <- function(skill,
   info(recipe$Run$logger, "##### SKILL METRICS SAVED TO NETCDF FILE #####")
 }
 
-save_corr <- function(skill,
-		      recipe,
-		      dictionary,
+save_corr <- function(recipe,
+		      skill,
                       data_cube,
                       outdir,
-                      agg = "global",
-                      archive = NULL) {
+                      agg = "global") {
   # This function adds metadata to the ensemble correlation in 'skill'
   # and exports it to a netCDF file inside 'outdir'.
   
+  archive <- get_archive(recipe)
+  dictionary <- read_yaml("conf/variable-dictionary.yml")
   # Define grid dimensions and names
   lalo <- c('longitude', 'latitude')
   # Remove singleton dimensions and rearrange lon, lat and time dims 
@@ -576,7 +562,6 @@ save_corr <- function(skill,
     skill <- lapply(skill, function(x) {
                     Reorder(x, c(lalo, 'ensemble', 'time'))})
   }
-
   # Add global and variable attributes
   global_attributes <- get_global_attributes(recipe, archive)
   ## TODO: Sort out the logic once default behavior is decided
@@ -675,14 +660,14 @@ save_corr <- function(skill,
        "##### ENSEMBLE CORRELATION SAVED TO NETCDF FILE #####")
 }
 
-save_percentiles <- function(percentiles,
-			     recipe,
+save_percentiles <- function(recipe,
+			     percentiles,
                              data_cube,
                              outdir,
-                             agg = "global",
-                             archive = NULL) {
+                             agg = "global") {
   # This function adds metadata to the percentiles
   # and exports them to a netCDF file inside 'outdir'.
+  archive <- get_archive(recipe)
   
   # Define grid dimensions and names
   lalo <- c('longitude', 'latitude')
@@ -758,14 +743,12 @@ save_percentiles <- function(percentiles,
       fcst.sdate <- paste0("1970", recipe$Analysis$Time$sdate)
     }
   }
-
   times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate, calendar)
   time <- times$time
 
   # Generate name of output file
   outfile <- get_filename(outdir, recipe, data_cube$Variable$varName,
 			  fcst.sdate, agg, "percentiles")
-
   # Get grid data and metadata and export to netCDF
   if (tolower(agg) == "country") { 
     country <- get_countries(grid)
@@ -782,12 +765,11 @@ save_percentiles <- function(percentiles,
   info(recipe$Run$logger, "##### PERCENTILES SAVED TO NETCDF FILE #####")
 }
 
-save_probabilities <- function(probs,
-			       recipe,
+save_probabilities <- function(recipe,
+			       probs,
 			       data_cube,
                                outdir,
                                agg = "global",
-                               archive = NULL,
 			       type = "hcst") {
   # Loops over the years in the s2dv_cube containing a hindcast or forecast
   # and exports the corresponding category probabilities to a netCDF file.
@@ -800,7 +782,7 @@ save_probabilities <- function(probs,
   # type: 'hcst' or 'fcst'
 
   lalo <- c('longitude', 'latitude')
-  
+  archive <- get_archive(recipe) 
   variable <- data_cube$Variable$varName
   var.longname <- attr(data_cube$Variable, 'variable')$long_name
   global_attributes <- get_global_attributes(recipe, archive)
diff --git a/tools/get_archive.R b/tools/get_archive.R
new file mode 100644
index 00000000..11602698
--- /dev/null
+++ b/tools/get_archive.R
@@ -0,0 +1,12 @@
+get_archive <- function(recipe) {
+  if (tolower(recipe$Analysis$Horizon) == "seasonal") {
+    archive <- 
+      read_yaml(paste0("conf/archive.yml"))[[recipe$Run$filesystem]]
+  } else if (tolower(recipe$Analysis$Horizon) == "decadal") {
+    archive <- 
+      read_yaml(paste0("conf/archive_decadal.yml"))[[recipe$Run$filesystem]]
+  }
+  ## TODO: Add dictionary filesystem dependency?
+  # dict <- read_yaml("conf/variable-dictionary.yml")
+  return(archive)
+}
diff --git a/tools/libs.R b/tools/libs.R
index 826a1f19..54a6b4c2 100644
--- a/tools/libs.R
+++ b/tools/libs.R
@@ -35,4 +35,5 @@ source("tools/divide_recipe.R")
 source("tools/data_summary.R")
 source("tools/read_atomic_recipe.R")
 source("tools/write_autosubmit_conf.R")
+source("tools/get_archive.R")
 # source("tools/add_dims.R") # Not sure if necessary yet
-- 
GitLab


From b24c8bd1403d4af36d9f7300b85e209a54ecf041 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Fri, 17 Mar 2023 15:28:47 +0100
Subject: [PATCH 42/57] Add script parameter to recipe autosubmit conf

---
 autosubmit/auto-verification.sh           | 3 ++-
 autosubmit/conf_esarchive/proj.conf       | 1 +
 recipes/tests/recipe_seasonal_example.yml | 5 +++--
 tools/check_recipe.R                      | 6 +++++-
 tools/write_autosubmit_conf.R             | 5 +++++
 5 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh
index c2328a72..8408912b 100644
--- a/autosubmit/auto-verification.sh
+++ b/autosubmit/auto-verification.sh
@@ -3,13 +3,14 @@
 ############ AUTOSUBMIT INPUTS ############
 proj_dir=%PROJDIR%
 outdir=%OUTDIR%
+script=%SCRIPT%
 CHUNK=%CHUNK%
 ###############################
 
 ## TODO: How to define the script
 cd $proj_dir
 
-script=modules/test_parallel_workflow.R
+# script=modules/test_parallel_workflow.R
 atomic_recipe_number=$(printf "%02d" $CHUNK)
 atomic_recipe=${outdir}/logs/recipes/atomic_recipe_${atomic_recipe_number}.yml
 
diff --git a/autosubmit/conf_esarchive/proj.conf b/autosubmit/conf_esarchive/proj.conf
index 7eacf6c4..c57e37eb 100644
--- a/autosubmit/conf_esarchive/proj.conf
+++ b/autosubmit/conf_esarchive/proj.conf
@@ -2,3 +2,4 @@
 
 MODULES = "MODULES"
 OUTDIR = 
+SCRIPT = 
diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml
index a866bf5c..0a22d3ab 100644
--- a/recipes/tests/recipe_seasonal_example.yml
+++ b/recipes/tests/recipe_seasonal_example.yml
@@ -32,9 +32,9 @@ Analysis:
     ftime_min: 1 # single option
     ftime_max: 6 # single option
   Region: # multiple lists, Add region name if there is more than 1 region
-    - {latmin: -5, latmax: 5, lonmin: -10, lonmax: 10}
+    - {latmin: -10, latmax: 10, lonmin: -10, lonmax: 10}
   Regrid:
-    method: conservative ## TODO: allow multiple methods?
+    method: conservative
     type: to_system
   Workflow:
     Anomalies:
@@ -64,6 +64,7 @@ Run:
   autosubmit: yes
   # fill only if using autosubmit
   auto_conf:
+    script: /esarchive/scratch/vagudets/repos/auto-s2s/modules/test_parallel_workflow.R # path to the script to run
     expid: a5no ## if left empty, create new exp?
     hpc_user: bsc32762 # your hpc username
     wallclock: 04:00 # hh:mm
diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index 3a8759c5..ce4c15e2 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -336,7 +336,11 @@ check_recipe <- function(recipe) {
   AUTO_PARAMS <- c("expid", "hpc_user", "wallclock", "processors_per_job",
 		   "platform", "email_notifications", "email_address",
 		   "notify_completed", "notify_failed")
-
+  # Autosubmit false by default
+  if (is.null(recipe$Run$autosubmit)) {
+    recipe$Run$autosubmit <- F
+  }
+  # Autosubmit configuration checks
   if (recipe$Run$autosubmit) {
     # Read autosubmit info for the specific filesystem (e.g. esarchive)
     auto_specs <- read_yaml("conf/autosubmit.yml")[[recipe$Run$filesystem]]
diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R
index 8e6c6ba1..571294ce 100644
--- a/tools/write_autosubmit_conf.R
+++ b/tools/write_autosubmit_conf.R
@@ -53,8 +53,10 @@ write_autosubmit_conf <- function(recipe, nchunks) {
       # Section 5: proj
       ## modules? Info that goes on script, e.g. output directory
       conf$common$OUTDIR <- recipe$Run$output_dir
+      conf$common$SCRIPT <- recipe$Run$auto_conf$script
     }
     # Write config file inside autosubmit dir
+    ## TODO: Change write.type depending on autosubmit version
     write.config(conf, paste0(dest_dir, dest_file), write.type = "ini")
     Sys.chmod(paste0(dest_dir, dest_file), mode = "755", use_umask = F)
   }
@@ -73,6 +75,8 @@ write_autosubmit_conf <- function(recipe, nchunks) {
 	 paste("module load", auto_specs$module_version))
     info(recipe$Run$logger,
 	 paste("autosubmit create", expid))
+    info(recipe$Run$logger,
+	 paste("autosubmit refresh", expid))
     info(recipe$Run$logger,
 	 paste("nohup autosubmit run", expid, "& disown"))
   } else {
@@ -80,6 +84,7 @@ write_autosubmit_conf <- function(recipe, nchunks) {
   	        "the following commands:"))
     print(paste("module load", auto_specs$module_version))
     print(paste("autosubmit create", expid))
+    print(paste("autosubmit refresh", expid))
     print(paste("nohup autosubmit run", expid, "& disown"))
   }
 }
-- 
GitLab


From ceaa8f8641a88d80889e5fde68affc2835a8b067 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 21 Mar 2023 12:01:38 +0100
Subject: [PATCH 43/57] Rearrange and simplify saving module fun args

---
 modules/Saving/Saving.R | 71 ++++++++++++++++++++++++++++-------------
 1 file changed, 48 insertions(+), 23 deletions(-)

diff --git a/modules/Saving/Saving.R b/modules/Saving/Saving.R
index 0a5cce93..47669fa5 100644
--- a/modules/Saving/Saving.R
+++ b/modules/Saving/Saving.R
@@ -31,13 +31,16 @@ save_data <- function(recipe, data,
   dir.create(outdir, showWarnings = FALSE, recursive = TRUE)
 
   # Export hindcast, forecast and observations onto outfile
-  save_forecast(recipe = recipe, data_cube = data$hcst, outdir = outdir,
-		type = 'hcst')
+  save_forecast(recipe = recipe, data_cube = data$hcst,
+		type = 'hcst',
+		outdir = outdir)
   if (!is.null(data$fcst)) {
-    save_forecast(recipe = recipe, data_cube = data$fcst, outdir =  outdir,
-		  type = 'fcst')
+    save_forecast(recipe = recipe, data_cube = data$fcst,
+		  type = 'fcst',
+		  outdir = outdir)
   }
-  save_observations(recipe = recipe, data_cube = data$obs, outdir = outdir)
+  save_observations(recipe = recipe, data_cube = data$obs,
+		    outdir = outdir)
 
   # Separate ensemble correlation from the rest of the metrics, as it has one
   # extra dimension "ensemble" and must be saved to a different file
@@ -70,14 +73,14 @@ save_data <- function(recipe, data,
 		     data_cube = data$hcst,
 		     outdir = outdir)
     save_probabilities(recipe = recipe, probs = probabilities$probs,
-		       data_cube = data$hcst, 
-		       outdir = outdir,
-		       type = "hcst")
+		       data_cube = data$hcst,
+		       type = "hcst",
+		       outdir = outdir)
     if (!is.null(probabilities$probs_fcst)) {
       save_probabilities(recipe = recipe, probs = probabilities$probs_fcst, 
 			 data_cube = data$fcst,
-			 outdir = outdir,
-		         type = "fcst")
+			 type = "fcst",
+			 outdir = outdir)
     }
   }
 }
@@ -154,9 +157,9 @@ get_latlon <- function(latitude, longitude) {
 
 save_forecast <- function(recipe,
 			  data_cube,
-                          outdir,
-                          agg = "global",
-                          type = NULL) {
+			  type = "hcst",
+			  agg = "global",
+                          outdir = NULL) {
   # Loops over the years in the s2dv_cube containing a hindcast or forecast
   # and exports each year to a netCDF file.
   # data_cube: s2dv_cube containing the data and metadata
@@ -174,6 +177,10 @@ save_forecast <- function(recipe,
   store.freq <- recipe$Analysis$Variables$freq
   calendar <- archive$System[[global_attributes$system]]$calendar
 
+  if (is.null(outdir)) {
+    outdir <- get_dir(recipe)
+  }
+
   # Generate vector containing leadtimes
   dates <- as.PCICt(ClimProjDiags::Subset(data_cube$Dates$start, 'syear', 1),
 		    cal = calendar)
@@ -289,8 +296,8 @@ save_forecast <- function(recipe,
 
 save_observations <- function(recipe,
 			      data_cube,
-                              outdir,
-                              agg = "global") {
+                              agg = "global",
+			      outdir = NULL) {
   # Loops over the years in the s2dv_cube containing the observations and
   # exports each year to a netCDF file.
   # data_cube: s2dv_cube containing the data and metadata
@@ -307,6 +314,10 @@ save_observations <- function(recipe,
   fcst.horizon <- tolower(recipe$Analysis$Horizon)
   store.freq <- recipe$Analysis$Variables$freq
   calendar <- archive$Reference[[global_attributes$reference]]$calendar
+
+  if (is.null(outdir)) {
+    outdir <- get_dir(recipe)
+  }
   
   # Generate vector containing leadtimes
   ## TODO: Move to a separate function?
@@ -433,8 +444,8 @@ save_observations <- function(recipe,
 save_metrics <- function(recipe,
 			 skill,
                          data_cube,
-                         outdir,
-                         agg = "global") {
+                         agg = "global",
+			 outdir = NULL) {
   # This function adds metadata to the skill metrics in 'skill'
   # and exports them to a netCDF file inside 'outdir'.
   
@@ -442,6 +453,8 @@ save_metrics <- function(recipe,
   lalo <- c('longitude', 'latitude')
   archive <- get_archive(recipe)
   dictionary <- read_yaml("conf/variable-dictionary.yml")
+
+
   # Remove singleton dimensions and rearrange lon, lat and time dims 
   if (tolower(agg) == "global") {
     skill <- lapply(skill, function(x) {
@@ -526,6 +539,9 @@ save_metrics <- function(recipe,
   time <- times$time
 
   # Generate name of output file
+  if (is.null(outdir)) {
+    outdir <- get_dir(recipe)
+  }
   outfile <- get_filename(outdir, recipe, data_cube$Variable$varName,
 			  fcst.sdate, agg, "skill")
 
@@ -548,8 +564,8 @@ save_metrics <- function(recipe,
 save_corr <- function(recipe,
 		      skill,
                       data_cube,
-                      outdir,
-                      agg = "global") {
+                      agg = "global",
+		      outdir = NULL) {
   # This function adds metadata to the ensemble correlation in 'skill'
   # and exports it to a netCDF file inside 'outdir'.
   
@@ -640,6 +656,9 @@ save_corr <- function(recipe,
   time <- times$time
 
   # Generate name of output file
+  if (is.null(outdir)) {
+    outdir <- get_dir(recipe)
+  }
   outfile <- get_filename(outdir, recipe, data_cube$Variable$varName,
 			  fcst.sdate, agg,  "corr")
 
@@ -663,8 +682,8 @@ save_corr <- function(recipe,
 save_percentiles <- function(recipe,
 			     percentiles,
                              data_cube,
-                             outdir,
-                             agg = "global") {
+                             agg = "global",
+			     outdir = NULL) {
   # This function adds metadata to the percentiles
   # and exports them to a netCDF file inside 'outdir'.
   archive <- get_archive(recipe)
@@ -747,6 +766,9 @@ save_percentiles <- function(recipe,
   time <- times$time
 
   # Generate name of output file
+  if (is.null(outdir)) {
+    outdir <- get_dir(recipe)
+  }
   outfile <- get_filename(outdir, recipe, data_cube$Variable$varName,
 			  fcst.sdate, agg, "percentiles")
   # Get grid data and metadata and export to netCDF
@@ -768,9 +790,9 @@ save_percentiles <- function(recipe,
 save_probabilities <- function(recipe,
 			       probs,
 			       data_cube,
-                               outdir,
                                agg = "global",
-			       type = "hcst") {
+			       type = "hcst",
+			       outdir = NULL) {
   # Loops over the years in the s2dv_cube containing a hindcast or forecast
   # and exports the corresponding category probabilities to a netCDF file.
   # probs: array containing the probability data
@@ -786,6 +808,9 @@ save_probabilities <- function(recipe,
   variable <- data_cube$Variable$varName
   var.longname <- attr(data_cube$Variable, 'variable')$long_name
   global_attributes <- get_global_attributes(recipe, archive)
+  if (is.null(outdir)) {
+    outdir <- get_dir(recipe)
+  }
   # Add anomaly computation to global attributes
   ## TODO: Sort out the logic once default behavior is decided
   if ((!is.null(recipe$Analysis$Workflow$Anomalies$compute)) && 
-- 
GitLab


From 67b2f0f17a763abcdcadabc678762bab3a76caa3 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 21 Mar 2023 13:36:07 +0100
Subject: [PATCH 44/57] Adapt region checks to atomic and general case, fix
 pipeline

---
 tests/testthat/test-decadal_monthly_1.R |  2 +-
 tests/testthat/test-seasonal_daily.R    |  2 +-
 tests/testthat/test-seasonal_monthly.R  |  2 +-
 tools/check_recipe.R                    | 39 +++++++++++++++----------
 tools/prepare_outputs.R                 | 11 +++++--
 5 files changed, 35 insertions(+), 21 deletions(-)

diff --git a/tests/testthat/test-decadal_monthly_1.R b/tests/testthat/test-decadal_monthly_1.R
index b76a216c..406ceaec 100644
--- a/tests/testthat/test-decadal_monthly_1.R
+++ b/tests/testthat/test-decadal_monthly_1.R
@@ -33,7 +33,7 @@ probs <- compute_probabilities(recipe, calibrated_data)
 # Saving
 suppressWarnings({invisible(capture.output(
 save_data(recipe = recipe, data = calibrated_data, 
-          skill_metrics = skill_metrics, probabilities = probs, archive = archive)
+          skill_metrics = skill_metrics, probabilities = probs)
 ))})
 
 # Plotting
diff --git a/tests/testthat/test-seasonal_daily.R b/tests/testthat/test-seasonal_daily.R
index 53a235e7..0aba1a0d 100644
--- a/tests/testthat/test-seasonal_daily.R
+++ b/tests/testthat/test-seasonal_daily.R
@@ -6,7 +6,7 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 
 recipe_file <- "tests/recipes/recipe-seasonal_daily_1.yml"
-recipe <- prepare_outputs(recipe_file, disable_checks = T)
+recipe <- prepare_outputs(recipe_file, disable_checks = F)
 # Load datasets
 suppressWarnings({invisible(capture.output(
 data <- load_datasets(recipe)
diff --git a/tests/testthat/test-seasonal_monthly.R b/tests/testthat/test-seasonal_monthly.R
index 4ea7f9ab..d4084c81 100644
--- a/tests/testthat/test-seasonal_monthly.R
+++ b/tests/testthat/test-seasonal_monthly.R
@@ -7,7 +7,7 @@ source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
 recipe_file <- "tests/recipes/recipe-seasonal_monthly_1.yml"
-recipe <- prepare_outputs(recipe_file, disable_checks = T)
+recipe <- prepare_outputs(recipe_file, disable_checks = F)
 archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 
 # Load datasets
diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index ce4c15e2..d191f420 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -44,7 +44,7 @@ check_recipe <- function(recipe) {
   # Check time settings
   if (tolower(recipe$Analysis$Horizon) == "seasonal") {
     ## TODO: Specify filesystem
-    archive <- read_yaml(ARCHIVE_SEASONAL)$archive
+    archive <- read_yaml(ARCHIVE_SEASONAL)[[recipe$Run$filesystem]]
     if (!all(TIME_SETTINGS_SEASONAL %in% names(recipe$Analysis$Time))) {
       error(recipe$Run$logger,
 	    paste0("The element 'Time' in the recipe must contain all of the ",
@@ -53,7 +53,7 @@ check_recipe <- function(recipe) {
       error_status <- T
     }
   } else if (tolower(recipe$Analysis$Horizon) == "decadal") {
-    archive <- read_yaml(ARCHIVE_DECADAL)$archive
+    archive <- read_yaml(ARCHIVE_DECADAL)[[recipe$Run$filesystem]]
     if (!all(TIME_SETTINGS_DECADAL %in% names(recipe$Analysis$Time))) {
       error(recipe$Run$logger,
 	    paste0("The element 'Time' in the recipe must contain all of the ",
@@ -187,22 +187,31 @@ check_recipe <- function(recipe) {
 
   # Region checks:
   LIMITS <- c('latmin', 'latmax', 'lonmin', 'lonmax')
-  for (region in recipe$Analysis$Region) {
-    if (!all(LIMITS %in% names(region))) {
-      error(recipe$Run$logger,
-            paste0("There must be 4 elements in 'Region': ",
-		   paste(LIMITS, collapse = ", "), "."))
-      error_status <- T
-    }
-  }
-  if (length(recipe$Analysis$Region) > 1) {
+  # Ordinary recipe
+  if (is.null(names(recipe$Analysis$Region))) {
     for (region in recipe$Analysis$Region) {
-      if (is.null(region$name)) {
-	error(recipe$Run$logger,
-	      paste("If more than one region has been defined, every region",
-		    "must have a unique name."))
+      if (!all(LIMITS %in% names(region))) {
+        error(recipe$Run$logger,
+              paste0("There must be 4 elements in 'Region': ",
+          	   paste(LIMITS, collapse = ", "), "."))
+        error_status <- T
       }
     }
+    if (length(recipe$Analysis$Region) > 1) {
+      for (region in recipe$Analysis$Region) {
+        if (!("name" %in% names(region)) || (is.null(region$name))) {
+          error(recipe$Run$logger,
+                paste("If more than one region has been defined, every region",
+          	      "must have a unique name."))
+        }
+      }
+    }
+  # Atomic recipe
+  } else if (!all(LIMITS %in% names(recipe$Analysis$Region))) {
+    error(recipe$Run$logger,
+	  paste0("There must be 4 elements in 'Region': ",
+		 paste(LIMITS, collapse = ", "), "."))
+    error_status <- T
   }
     ## TODO: Implement multiple regions
     # nregions <- length(recipe$Analysis$Region)
diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R
index 85be5c46..6d191d6e 100644
--- a/tools/prepare_outputs.R
+++ b/tools/prepare_outputs.R
@@ -30,9 +30,7 @@ prepare_outputs <- function(recipe_file,
   recipe <- read_yaml(recipe_file)
   recipe$recipe_path <- recipe_file
   recipe$name <- tools::file_path_sans_ext(basename(recipe_file))
-  if (is.null(recipe$Run$filesystem)) {
-    recipe$Run$filesystem <- "esarchive"
-  }
+
   
   output_dir = recipe$Run$output_dir
   # Create output folders
@@ -81,6 +79,13 @@ prepare_outputs <- function(recipe_file,
   recipe$Run$output_dir <- file.path(output_dir, folder_name)
   recipe$Run$logger <- logger
   recipe$Run$logfile <- logfile
+
+  # Set up default filesystem
+  if (is.null(recipe$Run$filesystem)) {
+    recipe$Run$filesystem <- "esarchive"
+    warn(recipe$Run$logger,
+	 "Filesystem not specified in the recipe. Setting it to 'esarchive'.")
+  }
   # Run recipe checker
   if (disable_checks) {
     warn(recipe$Run$logger,
-- 
GitLab


From 36b65cf77e8f75fdaab60540492126e21c8ff8ba Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 21 Mar 2023 14:53:00 +0100
Subject: [PATCH 45/57] Change ncores in test recipe, rename 'startup' to 'log'

---
 modules/Loading/testing_recipes/recipe_system7c3s-tas.yml | 2 +-
 tools/prepare_outputs.R                                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/Loading/testing_recipes/recipe_system7c3s-tas.yml b/modules/Loading/testing_recipes/recipe_system7c3s-tas.yml
index c8d3b5e8..47cfc31b 100644
--- a/modules/Loading/testing_recipes/recipe_system7c3s-tas.yml
+++ b/modules/Loading/testing_recipes/recipe_system7c3s-tas.yml
@@ -39,7 +39,7 @@ Analysis:
       percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]]
     Indicators:
       index: no
-  ncores: 1
+  ncores: 10
   remove_NAs: yes
   Output_format: S2S4E
 Run:
diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R
index 6d191d6e..61825738 100644
--- a/tools/prepare_outputs.R
+++ b/tools/prepare_outputs.R
@@ -51,7 +51,7 @@ prepare_outputs <- function(recipe_file,
   file.copy(recipe$recipe_path, file.path(output_dir, folder_name, 'logs',
 				          'recipes'))
   # Create log output file
-  logfile <- file.path(output_dir, folder_name, 'logs', 'startup.log')
+  logfile <- file.path(output_dir, folder_name, 'logs', 'main.log')
   file.create(logfile)
 
   # Set default behaviour of logger
-- 
GitLab


From 02966e548092d63c0821fdd6569a22e836d1d265 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 21 Mar 2023 14:59:16 +0100
Subject: [PATCH 46/57] Move testing atomic recipes to another directory

---
 modules/test_decadal.R                                          | 2 +-
 .../atomic_recipes}/recipe_decadal.yml                          | 0
 .../atomic_recipes}/recipe_decadal_daily.yml                    | 0
 .../atomic_recipes}/recipe_decadal_monthly_2.yml                | 0
 .../atomic_recipes}/recipe_seasonal-tests.yml                   | 0
 .../atomic_recipes}/recipe_system5c3s-rsds.yml                  | 0
 .../atomic_recipes}/recipe_system5c3s-tas.yml                   | 0
 .../atomic_recipes}/recipe_system7c3s-prlr.yml                  | 0
 .../atomic_recipes}/recipe_system7c3s-tas.yml                   | 0
 .../atomic_recipes}/recipe_tas-daily-regrid-to-reference.yml    | 0
 .../atomic_recipes}/recipe_tas-daily-regrid-to-system.yml       | 0
 .../atomic_recipes}/recipe_test-logging.yml                     | 0
 .../atomic_recipes}/recipe_test-new-metrics.yml                 | 0
 .../atomic_recipes}/recipe_test_anomalies.yml                   | 0
 .../atomic_recipes}/recipe_testing_nadia.yml                    | 0
 .../atomic_recipes}/wrong_recipe_example.yml                    | 0
 16 files changed, 1 insertion(+), 1 deletion(-)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_decadal.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_decadal_daily.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_decadal_monthly_2.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_seasonal-tests.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_system5c3s-rsds.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_system5c3s-tas.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_system7c3s-prlr.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_system7c3s-tas.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_tas-daily-regrid-to-reference.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_tas-daily-regrid-to-system.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_test-logging.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_test-new-metrics.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_test_anomalies.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/recipe_testing_nadia.yml (100%)
 rename {modules/Loading/testing_recipes => recipes/atomic_recipes}/wrong_recipe_example.yml (100%)

diff --git a/modules/test_decadal.R b/modules/test_decadal.R
index 8998cfbe..f9f9521d 100644
--- a/modules/test_decadal.R
+++ b/modules/test_decadal.R
@@ -5,7 +5,7 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
-recipe_file <- "modules/Loading/testing_recipes/recipe_decadal.yml"
+recipe_file <- "recipes/atomic_recipes/recipe_decadal.yml"
 recipe <- prepare_outputs(recipe_file)
 # archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
 
diff --git a/modules/Loading/testing_recipes/recipe_decadal.yml b/recipes/atomic_recipes/recipe_decadal.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_decadal.yml
rename to recipes/atomic_recipes/recipe_decadal.yml
diff --git a/modules/Loading/testing_recipes/recipe_decadal_daily.yml b/recipes/atomic_recipes/recipe_decadal_daily.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_decadal_daily.yml
rename to recipes/atomic_recipes/recipe_decadal_daily.yml
diff --git a/modules/Loading/testing_recipes/recipe_decadal_monthly_2.yml b/recipes/atomic_recipes/recipe_decadal_monthly_2.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_decadal_monthly_2.yml
rename to recipes/atomic_recipes/recipe_decadal_monthly_2.yml
diff --git a/modules/Loading/testing_recipes/recipe_seasonal-tests.yml b/recipes/atomic_recipes/recipe_seasonal-tests.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_seasonal-tests.yml
rename to recipes/atomic_recipes/recipe_seasonal-tests.yml
diff --git a/modules/Loading/testing_recipes/recipe_system5c3s-rsds.yml b/recipes/atomic_recipes/recipe_system5c3s-rsds.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_system5c3s-rsds.yml
rename to recipes/atomic_recipes/recipe_system5c3s-rsds.yml
diff --git a/modules/Loading/testing_recipes/recipe_system5c3s-tas.yml b/recipes/atomic_recipes/recipe_system5c3s-tas.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_system5c3s-tas.yml
rename to recipes/atomic_recipes/recipe_system5c3s-tas.yml
diff --git a/modules/Loading/testing_recipes/recipe_system7c3s-prlr.yml b/recipes/atomic_recipes/recipe_system7c3s-prlr.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_system7c3s-prlr.yml
rename to recipes/atomic_recipes/recipe_system7c3s-prlr.yml
diff --git a/modules/Loading/testing_recipes/recipe_system7c3s-tas.yml b/recipes/atomic_recipes/recipe_system7c3s-tas.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_system7c3s-tas.yml
rename to recipes/atomic_recipes/recipe_system7c3s-tas.yml
diff --git a/modules/Loading/testing_recipes/recipe_tas-daily-regrid-to-reference.yml b/recipes/atomic_recipes/recipe_tas-daily-regrid-to-reference.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_tas-daily-regrid-to-reference.yml
rename to recipes/atomic_recipes/recipe_tas-daily-regrid-to-reference.yml
diff --git a/modules/Loading/testing_recipes/recipe_tas-daily-regrid-to-system.yml b/recipes/atomic_recipes/recipe_tas-daily-regrid-to-system.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_tas-daily-regrid-to-system.yml
rename to recipes/atomic_recipes/recipe_tas-daily-regrid-to-system.yml
diff --git a/modules/Loading/testing_recipes/recipe_test-logging.yml b/recipes/atomic_recipes/recipe_test-logging.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_test-logging.yml
rename to recipes/atomic_recipes/recipe_test-logging.yml
diff --git a/modules/Loading/testing_recipes/recipe_test-new-metrics.yml b/recipes/atomic_recipes/recipe_test-new-metrics.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_test-new-metrics.yml
rename to recipes/atomic_recipes/recipe_test-new-metrics.yml
diff --git a/modules/Loading/testing_recipes/recipe_test_anomalies.yml b/recipes/atomic_recipes/recipe_test_anomalies.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_test_anomalies.yml
rename to recipes/atomic_recipes/recipe_test_anomalies.yml
diff --git a/modules/Loading/testing_recipes/recipe_testing_nadia.yml b/recipes/atomic_recipes/recipe_testing_nadia.yml
similarity index 100%
rename from modules/Loading/testing_recipes/recipe_testing_nadia.yml
rename to recipes/atomic_recipes/recipe_testing_nadia.yml
diff --git a/modules/Loading/testing_recipes/wrong_recipe_example.yml b/recipes/atomic_recipes/wrong_recipe_example.yml
similarity index 100%
rename from modules/Loading/testing_recipes/wrong_recipe_example.yml
rename to recipes/atomic_recipes/wrong_recipe_example.yml
-- 
GitLab


From 4190ef62a29cba395ac6aa1173883804eb20a833 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 21 Mar 2023 17:05:25 +0100
Subject: [PATCH 47/57] Add MARS autosubmit configuration files (WIP)

---
 autosubmit/conf_mars/autosubmit.yml | 22 +++++++++++++++
 autosubmit/conf_mars/expdef.yml     | 43 +++++++++++++++++++++++++++++
 autosubmit/conf_mars/jobs.yml       |  9 ++++++
 autosubmit/conf_mars/platforms.yml  | 13 +++++++++
 autosubmit/conf_mars/proj.yml       |  4 +++
 5 files changed, 91 insertions(+)
 create mode 100644 autosubmit/conf_mars/autosubmit.yml
 create mode 100644 autosubmit/conf_mars/expdef.yml
 create mode 100644 autosubmit/conf_mars/jobs.yml
 create mode 100644 autosubmit/conf_mars/platforms.yml
 create mode 100644 autosubmit/conf_mars/proj.yml

diff --git a/autosubmit/conf_mars/autosubmit.yml b/autosubmit/conf_mars/autosubmit.yml
new file mode 100644
index 00000000..97534284
--- /dev/null
+++ b/autosubmit/conf_mars/autosubmit.yml
@@ -0,0 +1,22 @@
+config:
+  EXPID:
+  AUTOSUBMIT_VERSION: 3.14.0
+  MAXWAITINGJOBS: 16
+  # Default maximum number of jobs to be running at the same time at any platform
+  # Default: 6
+  TOTALJOBS: 16
+  SAFETYSLEEPTIME: 10
+  RETRIALS: 0
+mail:
+  NOTIFICATIONS:  
+  TO: 
+communications:
+  # Communications library used to connect with platforms: paramiko or saga.
+  # Default: paramiko
+  API: paramiko
+storage:
+  # Defines the way of storing the progress of the experiment. The available options are:
+  # A PICKLE file (pkl) or an SQLite database (db). Default: pkl
+  TYPE: pkl
+  # Defines if the remote logs will be copied to the local platform. Default: True.
+  COPY_REMOTE_LOGS: True
diff --git a/autosubmit/conf_mars/expdef.yml b/autosubmit/conf_mars/expdef.yml
new file mode 100644
index 00000000..eee755d7
--- /dev/null
+++ b/autosubmit/conf_mars/expdef.yml
@@ -0,0 +1,43 @@
+DeFault:
+  EXPID:
+  HPCARCH: nord3v2
+experiment:
+  DATELIST:
+  MEMBERS: fc0
+  CHUNKSIZEUNIT: month
+  NUMCHUNKS:
+  CHUNKINI:
+  CALENDAR: standard
+project:
+  PROJECT_TYPE: local
+  # Destination folder name for project. type: STRING, default: leave empty,
+  PROJECT_DESTINATION: auto-s2s
+# If PROJECT_TYPE is not git, no need to change
+git:
+  # Repository URL  STRING: 'https://github.com/torvalds/linux.git'
+  PROJECT_ORIGIN: https://earth.bsc.es/gitlab/es/auto-s2s.git 
+  # Select branch or tag, STRING, default: 'master', help: {'master' (default), 'develop', 'v3.1b', ...}
+  PROJECT_BRANCH: master
+  # type: STRING, default: leave empty, help: if model branch is a TAG leave empty
+  PROJECT_COMMIT: ''
+svn:
+  PROJECT_URL: ''
+  PROJECT_REVISION: ''
+# If PROJECT_TYPE is not local, no need to change
+local:
+  # type: STRING, help: /foo/bar/ecearth 
+  PROJECT_PATH: /esarchive/scratch/vagudets/repos/auto-s2s/
+# If PROJECT_TYPE is none, no need to change
+project_files:
+  # Where is PROJECT CONFIGURATION file location relative to project root path
+  FILE_PROJECT_CONF: '' 
+  # Where is JOBS CONFIGURATION file location relative to project root path
+  FILE_JOBS_CONF: ''
+  # Default job scripts type in the project. type: STRING, default: bash, supported: 'bash', 'python' or 'r'
+  JOB_SCRIPTS_TYPE: ''
+rerun:
+  # Is a rerun or not? [Default: Do set FALSE]. BOOLEAN: TRUE, FALSE
+  RERUN: FALSE
+  # If RERUN: TRUE then supply the list of chunks to rerun
+  # LIST: [ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]
+  CHUNKLIST: ''
diff --git a/autosubmit/conf_mars/jobs.yml b/autosubmit/conf_mars/jobs.yml
new file mode 100644
index 00000000..daab7ed8
--- /dev/null
+++ b/autosubmit/conf_mars/jobs.yml
@@ -0,0 +1,9 @@
+JOBS:
+  verification:
+    FILE: autosubmit/auto-verification.sh
+    RUNNING: chunk
+    WALLCLOCK: 
+    NOTIFY_ON: 
+    PLATFORM: nord3v2
+    PROCESSORS: 
+
diff --git a/autosubmit/conf_mars/platforms.yml b/autosubmit/conf_mars/platforms.yml
new file mode 100644
index 00000000..79aea18e
--- /dev/null
+++ b/autosubmit/conf_mars/platforms.yml
@@ -0,0 +1,13 @@
+## TODO: Change platform
+Platforms:
+  nord3v2:
+    TYPE: slurm
+    HOST: nord4.bsc.es
+    PROJECT: bsc32
+    ADD_PROJECT_TO_HOST: False
+    USER: 
+    SCRATCH_DIR: /gpfs/scratch
+    PROCESSORS_PER_NODE: 16
+    SERIAL_QUEUE: debug
+    QUEUE: bsc_es
+    CUSTOM_DIRECTIVES: ["#SBATCH --exclusive"]
diff --git a/autosubmit/conf_mars/proj.yml b/autosubmit/conf_mars/proj.yml
new file mode 100644
index 00000000..679cf63b
--- /dev/null
+++ b/autosubmit/conf_mars/proj.yml
@@ -0,0 +1,4 @@
+common:
+  MODULES: "MODULES"
+  OUTDIR: 
+  SCRIPT: 
-- 
GitLab


From f67cc696f74e595bbedfea06c6a2f606286c499c Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Wed, 22 Mar 2023 12:03:17 +0100
Subject: [PATCH 48/57] Add autosubmit v4 configuration for CERISE and dummy
 'mars' filesystem

---
 autosubmit/auto-verification-v4.sh           |  19 ++
 autosubmit/conf_mars/autosubmit.yml          |   2 +-
 autosubmit/conf_mars/expdef.yml              |   7 +-
 autosubmit/conf_mars/jobs.yml                |   5 +-
 autosubmit/conf_mars/platforms.yml           |   7 +-
 conf/archive.yml                             | 181 +++++++++++++++++++
 conf/autosubmit.yml                          |   9 +
 recipes/tests/recipe_autosubmit_marstest.yml |  76 ++++++++
 tools/write_autosubmit_conf.R                |  30 ++-
 9 files changed, 319 insertions(+), 17 deletions(-)
 create mode 100644 autosubmit/auto-verification-v4.sh
 create mode 100644 recipes/tests/recipe_autosubmit_marstest.yml

diff --git a/autosubmit/auto-verification-v4.sh b/autosubmit/auto-verification-v4.sh
new file mode 100644
index 00000000..e21bef2d
--- /dev/null
+++ b/autosubmit/auto-verification-v4.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+############ AUTOSUBMIT INPUTS ############
+proj_dir=%PROJDIR%
+outdir=%common.OUTDIR%
+script=%common.SCRIPT%
+CHUNK=%CHUNK%
+###############################
+
+## TODO: How to define the script
+cd $proj_dir
+
+# script=modules/test_parallel_workflow.R
+atomic_recipe_number=$(printf "%02d" $CHUNK)
+atomic_recipe=${outdir}/logs/recipes/atomic_recipe_${atomic_recipe_number}.yml
+
+source MODULES
+
+Rscript ${script} ${atomic_recipe}
diff --git a/autosubmit/conf_mars/autosubmit.yml b/autosubmit/conf_mars/autosubmit.yml
index 97534284..0fd5d5c6 100644
--- a/autosubmit/conf_mars/autosubmit.yml
+++ b/autosubmit/conf_mars/autosubmit.yml
@@ -1,6 +1,6 @@
 config:
   EXPID:
-  AUTOSUBMIT_VERSION: 3.14.0
+  AUTOSUBMIT_VERSION: 4.0.0b0
   MAXWAITINGJOBS: 16
   # Default maximum number of jobs to be running at the same time at any platform
   # Default: 6
diff --git a/autosubmit/conf_mars/expdef.yml b/autosubmit/conf_mars/expdef.yml
index eee755d7..b4327f65 100644
--- a/autosubmit/conf_mars/expdef.yml
+++ b/autosubmit/conf_mars/expdef.yml
@@ -1,12 +1,13 @@
-DeFault:
+DEFAULT:
   EXPID:
-  HPCARCH: nord3v2
+  HPCARCH: NORD3
 experiment:
   DATELIST:
   MEMBERS: fc0
   CHUNKSIZEUNIT: month
+  CHUNKSIZE: 1
   NUMCHUNKS:
-  CHUNKINI:
+  CHUNKINI: 1
   CALENDAR: standard
 project:
   PROJECT_TYPE: local
diff --git a/autosubmit/conf_mars/jobs.yml b/autosubmit/conf_mars/jobs.yml
index daab7ed8..2d8e32af 100644
--- a/autosubmit/conf_mars/jobs.yml
+++ b/autosubmit/conf_mars/jobs.yml
@@ -1,9 +1,8 @@
 JOBS:
   verification:
-    FILE: autosubmit/auto-verification.sh
+    FILE: autosubmit/auto-verification-CERISE.sh
     RUNNING: chunk
     WALLCLOCK: 
     NOTIFY_ON: 
-    PLATFORM: nord3v2
+    PLATFORM: NORD3
     PROCESSORS: 
-
diff --git a/autosubmit/conf_mars/platforms.yml b/autosubmit/conf_mars/platforms.yml
index 79aea18e..5f76557f 100644
--- a/autosubmit/conf_mars/platforms.yml
+++ b/autosubmit/conf_mars/platforms.yml
@@ -1,12 +1,11 @@
 ## TODO: Change platform
 Platforms:
-  nord3v2:
+  NORD3:
     TYPE: slurm
     HOST: nord4.bsc.es
-    PROJECT: bsc32
-    ADD_PROJECT_TO_HOST: False
     USER: 
-    SCRATCH_DIR: /gpfs/scratch
+    PROJECT: bsc32 ## TO BE CHANGED
+    SCRATCH_DIR: /gpfs/scratch/ ## TO BE CHANGED
     PROCESSORS_PER_NODE: 16
     SERIAL_QUEUE: debug
     QUEUE: bsc_es
diff --git a/conf/archive.yml b/conf/archive.yml
index 4450706a..200dd154 100644
--- a/conf/archive.yml
+++ b/conf/archive.yml
@@ -181,3 +181,184 @@ esarchive:
             reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc"
 
 
+mars:
+    src: "/esarchive/"
+    System:
+        ECMWF-SEAS5:
+            name: "ECMWF SEAS5"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "exp/ecmwf/system5c3s/"
+            daily_mean: {"tas":"_f6h/", "rsds":"_s0-24h/",
+                         "prlr":"_s0-24h/", "sfcWind":"_f6h/",
+                         "tasmin":"_f24h/", "tasmax":"_f24h/",
+                         "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/",
+                         "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/",
+                         "tdps":"_f6h/", "hurs":"_f6h/"}
+            monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/",
+                           "prlr":"_s0-24h/", "sfcWind":"_f6h/",
+                           "tasmin":"_f24h/", "tasmax":"_f24h/",
+                           "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/",
+                           "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/",
+                           "tdps":"_f6h/"}
+            nmember:
+              fcst: 51
+              hcst: 25
+            calendar: "proleptic_gregorian"
+            time_stamp_lag: "0"
+            reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc"
+        ECMWF-SEAS5.1:
+            name: "ECMWF SEAS5 (v5.1)"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "exp/ecmwf/system51c3s/"
+            daily_mean: {"tas":"_f6h/", "prlr":"_s0-24h/", "sfcWind":"_f6h/",
+                         "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/",
+                         "tdps":"_f6h/"}
+            monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", "prlr":"_s0-24h/",
+                           "sfcWind":"_f6h/", "tasmin":"_f24h/", "tasmax":"_f24h/",
+                           "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/",
+                           "tdps":"_f6h/"}
+            nmember:
+              fcst: 51
+              hcst: 25
+            calendar: "proleptic_gregorian"
+            time_stamp_lag: "0"
+            reference_grid: "conf/grid_description/griddes_system51c3s.txt"
+        Meteo-France-System7:
+            name: "Meteo-France System 7"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "exp/meteofrance/system7c3s/"
+            monthly_mean: {"tas":"_f6h/", "g500":"_f12h/",
+                           "prlr":"_f24h/", "sfcWind": "_f6h/",
+                           "tasmax":"_f6h/", "tasmin": "_f6h/"}
+            nmember:
+              fcst: 51
+              hcst: 25
+            time_stamp_lag: "+1"
+            calendar: "proleptic_gregorian"
+            reference_grid: "conf/grid_description/griddes_system7c3s.txt"
+        DWD-GCFS2.1:
+            name: "DWD GCFS 2.1"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "exp/dwd/system21_m1/"
+            monthly_mean: {"tas":"_f6h/", "prlr":"_f24h/",
+                           "g500":"_f12h/", "sfcWind":"_f6h/",
+                           "tasmin":"_f24h/", "tasmax":"_f24h/"}
+            nmember:
+              fcst: 50
+              hcst: 30
+            calendar: "proleptic_gregorian"
+            time_stamp_lag: "+1"
+            reference_grid: "conf/grid_description/griddes_system21_m1.txt"
+        CMCC-SPS3.5:
+            name: "CMCC-SPS3.5"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "exp/cmcc/system35c3s/"
+            monthly_mean: {"tas":"_f6h/", "g500":"_f12h/",
+                           "prlr":"_f24h/", "sfcWind": "_f6h/",
+                           "tasmax":"_f24h/", "tasmin":"_f24h"}
+            nmember:
+              fcst: 50
+              hcst: 40
+            calendar: "proleptic_gregorian"
+            time_stamp_lag: "+1"
+            reference_grid: "conf/grid_description/griddes_system35c3s.txt"
+        JMA-CPS2:
+            name: "JMA System 2"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "exp/jma/system2c3s/"
+            monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/",
+                           "tasmax":"_f6h/", "tasmin":"_f6h/"}
+            nmember:
+              fcst: 10
+              hcst: 10
+            calendar: "proleptic_gregorian"
+            time_stamp_lag: "+1"
+            reference_grid: "conf/grid_description/griddes_system2c3s.txt"
+        ECCC-CanCM4i:
+            name: "ECCC CanCM4i"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "exp/eccc/eccc1/"
+            monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/",
+                           "tasmax":"_f6h/", "tasmin":"_f6h/"}
+            nmember:
+              fcst: 10
+              hcst: 10
+            calendar: "proleptic_gregorian"
+            time_stamp_lag: "+1"
+            reference_grid: "conf/grid_description/griddes_eccc1.txt"
+        UK-MetOffice-Glosea600:
+            name: "UK MetOffice GloSea 6 (v6.0)"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "exp/ukmo/glosea6_system600-c3s/"
+            monthly_mean: {"tas":"_f6h/", "tasmin":"_f24h/",
+                           "tasmax":"_f24h/", "prlr":"_f24h/"}
+            nmember:
+              fcst: 62
+              hcst: 28
+            calendar: "proleptic_gregorian"
+            time_stamp_lag: "+1"
+            reference_grid: "conf/grid_description/griddes_ukmo600.txt"
+        NCEP-CFSv2:
+            name: "NCEP CFSv2"
+            institution: "NOAA NCEP" #?
+            src: "exp/ncep/cfs-v2/"
+            monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/",
+                           "tasmax":"_f6h/", "tasmin":"_f6h/"}
+            nmember:
+              fcst: 20
+              hcst: 20
+            calendar: "gregorian"
+            time_stamp_lag: "0"
+            reference_grid: "conf/grid_description/griddes_ncep-cfsv2.txt"
+    Reference:
+        ERA5:
+            name: "ERA5"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "recon/ecmwf/era5/"
+            daily_mean: {"tas":"_f1h-r1440x721cds/",
+                         "rsds":"_f1h-r1440x721cds/",
+                         "prlr":"_f1h-r1440x721cds/",
+                          "g300":"_f1h-r1440x721cds/",
+                         "g500":"_f1h-r1440x721cds/",
+                         "g850":"_f1h-r1440x721cds/",
+                         "sfcWind":"_f1h-r1440x721cds/",
+                         "tasmax":"_f1h-r1440x721cds/",
+                         "tasmin":"_f1h-r1440x721cds/",
+                         "ta300":"_f1h-r1440x721cds/",
+                         "ta500":"_f1h-r1440x721cds/", 
+                         "ta850":"_f1h-r1440x721cds/",
+                         "hurs":"_f1h-r1440x721cds/"}
+            monthly_mean: {"tas":"_f1h-r1440x721cds/",
+                           "prlr":"_f1h-r1440x721cds/",
+                           "rsds":"_f1h-r1440x721cds/",
+                           "g300":"_f1h-r1440x721cds/",
+                           "g500":"_f1h-r1440x721cds/",
+                           "g850":"_f1h-r1440x721cds/",
+                           "sfcWind":"_f1h-r1440x721cds/",
+                           "tasmax":"_f1h-r1440x721cds/",
+                           "tasmin":"_f1h-r1440x721cds/",
+                           "ta300":"_f1h-r1440x721cds/",
+                           "ta500":"_f1h-r1440x721cds/", 
+                           "ta850":"_f1h-r1440x721cds/"}
+            calendar: "standard"
+            reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc"
+        ERA5-Land:
+            name: "ERA5-Land"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "recon/ecmwf/era5land/"
+            daily_mean: {"tas":"_f1h/", "rsds":"_f1h/",
+                         "prlr":"_f1h/", "sfcWind":"_f1h/"}
+            monthly_mean: {"tas":"_f1h/","tasmin":"_f24h/",
+                           "tasmax":"_f24h/", "prlr":"_f1h/",
+                           "sfcWind":"_f1h/", "rsds":"_f1h/",
+                           "tdps":"_f1h/"}
+            calendar: "proleptic_gregorian"
+            reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc"
+        UERRA:
+            name: "ECMWF UERRA"
+            institution: "European Centre for Medium-Range Weather Forecasts"
+            src: "recon/ecmwf/uerra_mescan/"
+            daily_mean: {"tas":"_f6h/"}
+            monthly_mean: {"tas":"_f6h/"}
+            calendar: "proleptic_gregorian"
+            reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc"
diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml
index 3f852776..8b653a65 100644
--- a/conf/autosubmit.yml
+++ b/conf/autosubmit.yml
@@ -1,5 +1,14 @@
 esarchive:
   platform: nord3v2
   module_version: autosubmit/3.14.0-foss-2015a-Python-2.7.9
+  auto_version: 3.14.0
+  conf_format: ini
   experiment_dir: /esarchive/autosubmit/
   userID: bsc32
+mars:
+  platform: NORD3 ## TO BE CHANGED
+  module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 ## TO BE CHANGED
+  auto_version: 4.0.0
+  conf_format: yaml
+  experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED
+  userID: bsc32 ## TO BE CHANGED
diff --git a/recipes/tests/recipe_autosubmit_marstest.yml b/recipes/tests/recipe_autosubmit_marstest.yml
new file mode 100644
index 00000000..dfd2159f
--- /dev/null
+++ b/recipes/tests/recipe_autosubmit_marstest.yml
@@ -0,0 +1,76 @@
+################################################################################ 
+## RECIPE DESCRIPTION  
+################################################################################ 
+
+Description:
+  Author: V. Agudetse
+  Info: Test Independent verification of two variables, two sdates, two systems
+
+################################################################################
+## ANALYSIS CONFIGURATION
+################################################################################
+
+Analysis:
+  Horizon: Seasonal
+  Variables: # ECVs and Indicators?
+    - {name: tas, freq: monthly_mean}
+    - {name: prlr, freq: monthly_mean}
+  Datasets:
+    System: # multiple systems for single model, split if Multimodel = F
+      - {name: ECMWF-SEAS5}
+      - {name: Meteo-France-System7}
+    Multimodel: False # single option
+    Reference:
+      - {name: ERA5} # multiple references for single model?
+  Time:
+    sdate: # list, split
+      - '0101'
+      - '0201'
+    fcst_year: # list, don't split, handled internally
+    hcst_start: '2000' # single option
+    hcst_end: '2016' # single option
+    ftime_min: 1 # single option
+    ftime_max: 6 # single option
+  Region: # multiple lists, Add region name if there is more than 1 region
+    - {latmin: -10, latmax: 10, lonmin: -10, lonmax: 10}
+  Regrid:
+    method: conservative
+    type: to_system
+  Workflow:
+    Anomalies:
+      compute: yes
+      cross_validation: yes
+    Calibration:
+      method: raw ## TODO: list, split?
+    Skill:
+      metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS # list, don't split
+    Probabilities:
+      percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]] # list, don't split
+    Indicators:
+      index: no # ? 
+  ncores: 8
+  remove_NAs: yes # bool, don't split
+  Output_format: S2S4E # string, don't split
+
+################################################################################
+## Run CONFIGURATION
+################################################################################
+Run:
+  Loglevel: INFO
+  Terminal: yes
+  filesystem: mars
+  output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/
+  code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/
+  autosubmit: yes
+  # fill only if using autosubmit
+  auto_conf:
+    script: /esarchive/scratch/vagudets/repos/auto-s2s/modules/test_parallel_workflow.R # path to the script to run
+    expid: a5ta ## if left empty, create new exp?
+    hpc_user: bsc32762 # your hpc username
+    wallclock: 04:00 # hh:mm
+    processors_per_job: 8 # use ncores parameter?
+    platform: nord3v2 # make this constant?
+    email_notifications: yes # enable/disable email notifications
+    email_address: victoria.agudetse@bsc.es # email address for notifications
+    notify_completed: no # notify me by email when a job finishes
+    notify_failed: yes # notify me by email when a job fails
diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R
index 571294ce..a0208a9e 100644
--- a/tools/write_autosubmit_conf.R
+++ b/tools/write_autosubmit_conf.R
@@ -35,20 +35,37 @@ write_autosubmit_conf <- function(recipe, nchunks) {
     } else if (conf_type == "jobs") {
       # Section 3: jobs
       ## wallclock, notify_on, platform?, processors
-      conf$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock
+      # Different file structure depending on autosubmit version
+      if (auto_specs$auto_version == "4.0.0") {
+	jobs <- conf$JOBS
+      } else {
+	jobs <- conf
+      }
+      jobs$verification$WALLCLOCK <- recipe$Run$auto_conf$wallclock
       if (recipe$Run$auto_conf$notify_completed) {
-        conf$verification$NOTIFY_ON <- paste(conf$verification$NOTIFY_ON,
+        jobs$verification$NOTIFY_ON <- paste(jobs$verification$NOTIFY_ON,
 					     "COMPLETED")
       }
       if (recipe$Run$auto_conf$notify_failed) {
-	conf$verification$NOTIFY_ON <- paste(conf$verification$NOTIFY_ON,
+	jobs$verification$NOTIFY_ON <- paste(jobs$verification$NOTIFY_ON,
 					     "FAILED")
       }
-      conf$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores?
+      jobs$verification$PROCESSORS <- recipe$Run$auto_conf$processors_per_job # ncores?
+      # Return to original list 
+      if (auto_specs$auto_version == "4.0.0") {
+	conf$JOBS <- jobs
+      } else {
+	conf <- jobs
+      }
     } else if (conf_type == "platforms") {
       # Section 4: platform configuration
       ## nord3v2 configuration... platform name? user, processors_per_node
-      conf[[auto_specs$platform]]$USER <- recipe$Run$auto_conf$hpc_user
+      if (auto_specs$auto_version == "4.0.0") {
+	conf$Platforms[[auto_specs$platform]]$USER <- 
+	  recipe$Run$auto_conf$hpc_user
+      } else {
+	conf[[auto_specs$platform]]$USER <- recipe$Run$auto_conf$hpc_user
+      }
     } else if (conf_type == "proj") {
       # Section 5: proj
       ## modules? Info that goes on script, e.g. output directory
@@ -57,7 +74,8 @@ write_autosubmit_conf <- function(recipe, nchunks) {
     }
     # Write config file inside autosubmit dir
     ## TODO: Change write.type depending on autosubmit version
-    write.config(conf, paste0(dest_dir, dest_file), write.type = "ini")
+    write.config(conf, paste0(dest_dir, dest_file),
+		 write.type = auto_specs$conf_format)
     Sys.chmod(paste0(dest_dir, dest_file), mode = "755", use_umask = F)
   }
   info(recipe$Run$logger,
-- 
GitLab


From 84234d8a1c3b2237ae3d4fe683fbd6a0a5d36f3c Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Wed, 22 Mar 2023 12:31:08 +0100
Subject: [PATCH 49/57] add condition

---
 split.R | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/split.R b/split.R
index 8e33f916..9a3cbf72 100644
--- a/split.R
+++ b/split.R
@@ -11,5 +11,6 @@ recipe <- prepare_outputs(recipe_file)
 run_parameters <- divide_recipe(recipe)
 if (recipe$Run$autosubmit) {
   write_autosubmit_conf(recipe, run_parameters$n_atomic_recipes)
+} else {
+  run_parameters$outdir
 }
-run_parameters$outdir
-- 
GitLab


From 52edf7dd347a8d5a53b632aba16a9411c471f33f Mon Sep 17 00:00:00 2001
From: Carlos Delgado <carlos.delgado@bsc.es>
Date: Fri, 24 Mar 2023 12:46:15 +0100
Subject: [PATCH 50/57] added decadal splitting

---
 tools/divide_recipe.R | 62 ++++++++++++++++++++++++++++++-------------
 1 file changed, 44 insertions(+), 18 deletions(-)

diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R
index 403304c4..8bbe48fa 100644
--- a/tools/divide_recipe.R
+++ b/tools/divide_recipe.R
@@ -35,35 +35,52 @@ divide_recipe <- function(recipe) {
   
   # duplicate recipe by Datasets:
   # check Systems
-  if (recipe$Analysis$Datasets$Multimodel) {
+  if (recipe$Analysis$Datasets$Multimodel %in% c(TRUE,'both')) {
     for (reci in 1:length(all_recipes)) {
       all_recipes[[reci]]$Analysis$Datasets <- 
-	list(System = recipe$Analysis$Datasets$System,
-	     Multimodel = recipe$Analysis$Datasets$Multimodel,
-	     Reference = NULL)
+        list(System = recipe$Analysis$Datasets$System,
+             Multimodel = recipe$Analysis$Datasets$Multimodel,
+             Reference = NULL)
     }
   } else {
-    for (sys in 1:length(recipe$Analysis$Datasets$System)) {
-      for (reci in 1:length(all_recipes)) {
-        all_recipes[[reci]]$Analysis$Datasets <- 
-	  list(System = recipe$Analysis$Datasets$System[[sys]],
-	       Multimodel = recipe$Analysis$Datasets$Multimodel,
-	       Reference = NULL)
+    if (tolower(recipe$Analysis$Horizon) == 'seasonal') {
+      for (sys in 1:length(recipe$Analysis$Datasets$System)) {
+        for (reci in 1:length(all_recipes)) {
+          all_recipes[[reci]]$Analysis$Datasets <- 
+            list(System = recipe$Analysis$Datasets$System[[sys]],
+                 Multimodel = recipe$Analysis$Datasets$Multimodel,
+                 Reference = NULL)
+        }
+        if (sys == 1) {
+          recipes <- all_recipes
+        } else {
+          recipes <- append(recipes, all_recipes)
+        }
       }
-      if (sys == 1) {
-        recipes <- all_recipes
-      } else {
-        recipes <- append(recipes, all_recipes)
+    } else if (tolower(recipe$Analysis$Horizon) == 'decadal') {
+      for (sys in 1:length(recipe$Analysis$Datasets$System$name)) {
+        for (reci in 1:length(all_recipes)) {
+          all_recipes[[reci]]$Analysis$Datasets <- 
+            list(System = list(name = recipe$Analysis$Datasets$System$name[[sys]],
+                               member = recipe$Analysis$Datasets$System$member[[sys]]),
+                 Multimodel = recipe$Analysis$Datasets$Multimodel,
+                 Reference = NULL)
+        }
+        if (sys == 1) {
+          recipes <- all_recipes
+        } else {
+          recipes <- append(recipes, all_recipes)
+        }
       }
-    }
+    } # Rest of horizons
     all_recipes <- recipes
     rm(list = 'recipes')
   }
   # check References
-  for (ref in 1:length(recipe$Analysis$Datasets$Reference)) {
+  for (ref in 1:length(recipe$Analysis$Datasets$Reference$name)) {
     for (reci in 1:length(all_recipes)) {
-      all_recipes[[reci]]$Analysis$Datasets$Reference <- 
-                           recipe$Analysis$Datasets$Reference[[ref]]
+      all_recipes[[reci]]$Analysis$Datasets$Reference$name <- 
+                           recipe$Analysis$Datasets$Reference$name[[ref]]
     }
     if (ref == 1) {
         recipes <- all_recipes
@@ -102,6 +119,15 @@ divide_recipe <- function(recipe) {
     }
     all_recipes <- recipes
     rm(list = 'recipes')
+  } else if (tolower(recipe$Analysis$Horizon) == 'decadal') {
+    for (reci in 1:length(all_recipes)) {
+      all_recipes[[reci]]$Analysis$Time <-
+        list(fcst_year = recipe$Analysis$Time$fcst_year,
+             hcst_start = recipe$Analysis$Time$hcst_start,
+             hcst_end = recipe$Analysis$Time$hcst_end,
+             ftime_min = recipe$Analysis$Time$ftime_min,
+             ftime_max = recipe$Analysis$Time$ftime_max)
+    }
   } # Rest of horizons
   # Save all recipes in separate YAML files
   for (reci in 1:length(all_recipes)) {
-- 
GitLab


From eb7a8b93c56e55218c847d64aaec1b7c46f967c0 Mon Sep 17 00:00:00 2001
From: Carlos Delgado <carlos.delgado@bsc.es>
Date: Fri, 24 Mar 2023 12:48:28 +0100
Subject: [PATCH 51/57] recipe used for splitting

---
 recipes/recipe_decadal_split.yml | 60 ++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 recipes/recipe_decadal_split.yml

diff --git a/recipes/recipe_decadal_split.yml b/recipes/recipe_decadal_split.yml
new file mode 100644
index 00000000..9c77ad8c
--- /dev/null
+++ b/recipes/recipe_decadal_split.yml
@@ -0,0 +1,60 @@
+Description:
+  Author: Carlos Delgado Torres
+  Info: Test for spliting a decadal recipe (two variables)
+Analysis:
+  Horizon: Decadal
+  Variables:
+    - {name: tas, freq: monthly_mean}
+    - {name: pr, freq: monthly_mean}
+  Datasets:
+    System: 
+      name: EC-Earth3-i4
+      member: r1i4p1f1,r2i4p1f1
+    Multimodel: no
+    Reference:
+      name: JRA-55
+  Time:
+    fcst_year: 
+    hcst_start: 2015 # 2015-2016 in dcppA, 2017-2018 in dcppB
+    hcst_end: 2018
+#    season: 'Annual'
+    ftime_min: 6 # Apr
+    ftime_max: 8
+  Region:
+    - {latmin: -5, latmax: 5, lonmin: -5, lonmax: 5}
+  Regrid:
+    method: bilinear
+    type: to_system
+  Workflow:
+    Anomalies: 
+      compute: no
+      cross_validation: 
+    Calibration:
+      method: 'bias'
+    Skill:
+      metric: EnsCorr RPSS
+    Probabilities:
+      percentiles: [[1/3, 2/3]]
+    Indicators:
+      index: FALSE
+  ncores: 8 # Optional, int: number of cores, defaults to 1
+  remove_NAs: FALSE # Optional, bool: Whether NAs are removed, defaults to FALSE
+  Output_format: S2S4E
+Run:
+  Loglevel: INFO
+  Terminal: yes
+  filesystem: esarchive
+  output_dir: /esarchive/scratch/cdelgado/auto-s2s_logs/
+  code_dir: /esarchive/scratch/cdelgado/gitlab/auto-s2s/
+  autosubmit: yes
+  # fill only if using autosubmit
+  auto_conf:
+    expid: a5to ## if left empty, create new exp?
+    hpc_user: bsc32924 # your hpc username
+    wallclock: 01:00 # hh:mm
+    processors_per_job: 8 # use ncores parameter?
+    platform: nord3v2 # make this constant?
+    email_notifications: yes # enable/disable email notifications
+    email_address: carlos.delgado@bsc.es # email address for notifications
+    notify_completed: yes # notify me by email when a job finishes
+    notify_failed: yes # notify me by email when a job fails
-- 
GitLab


From 3a831727092c69e3d9971b386c55c1bb05b66e30 Mon Sep 17 00:00:00 2001
From: Carlos Delgado <carlos.delgado@bsc.es>
Date: Fri, 24 Mar 2023 18:46:33 +0100
Subject: [PATCH 52/57] added script to auto_conf

---
 recipes/recipe_decadal_split.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/recipes/recipe_decadal_split.yml b/recipes/recipe_decadal_split.yml
index 9c77ad8c..708037f7 100644
--- a/recipes/recipe_decadal_split.yml
+++ b/recipes/recipe_decadal_split.yml
@@ -49,7 +49,8 @@ Run:
   autosubmit: yes
   # fill only if using autosubmit
   auto_conf:
-    expid: a5to ## if left empty, create new exp?
+    script: /esarchive/scratch/cdelgado/gitlab/cdelgado_copernicus/ESS_evaluation_tool/main_decadal.R
+    expid: a5tx ## if left empty, create new exp?
     hpc_user: bsc32924 # your hpc username
     wallclock: 01:00 # hh:mm
     processors_per_job: 8 # use ncores parameter?
-- 
GitLab


From a691460265a70b8eb17d5aad271d6cc84bd4c74c Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 4 Apr 2023 11:20:21 +0200
Subject: [PATCH 53/57] Repository clean-up and rearrangement

---
 autosubmit/auto-verification.sh        |   2 -
 conf/vars-dict.yml-OLD                 | 114 -------------------------
 launch.sh                              |  30 -------
 modules/Loading/Loading.R              |   5 +-
 modules/Loading/Loading_decadal.R      |   5 +-
 modules/Loading/{ => R}/check_latlon.R |   0
 modules/Loading/{ => R}/dates2load.R   |  53 ------------
 modules/Loading/R/get_timeidx.R        |  57 +++++++++++++
 modules/Skill/{ => R}/compute_probs.R  |   0
 modules/Skill/{ => R}/compute_quants.R |   0
 modules/Skill/{ => R}/s2s.metrics.R    |   0
 modules/Skill/Skill.R                  |   6 +-
 run_parallel_workflow.sh               |  14 ---
 split.R                                |   3 +-
 14 files changed, 68 insertions(+), 221 deletions(-)
 delete mode 100644 conf/vars-dict.yml-OLD
 delete mode 100644 launch.sh
 rename modules/Loading/{ => R}/check_latlon.R (100%)
 rename modules/Loading/{ => R}/dates2load.R (51%)
 create mode 100644 modules/Loading/R/get_timeidx.R
 rename modules/Skill/{ => R}/compute_probs.R (100%)
 rename modules/Skill/{ => R}/compute_quants.R (100%)
 rename modules/Skill/{ => R}/s2s.metrics.R (100%)
 delete mode 100644 run_parallel_workflow.sh

diff --git a/autosubmit/auto-verification.sh b/autosubmit/auto-verification.sh
index 8408912b..bbd06556 100644
--- a/autosubmit/auto-verification.sh
+++ b/autosubmit/auto-verification.sh
@@ -7,10 +7,8 @@ script=%SCRIPT%
 CHUNK=%CHUNK%
 ###############################
 
-## TODO: How to define the script
 cd $proj_dir
 
-# script=modules/test_parallel_workflow.R
 atomic_recipe_number=$(printf "%02d" $CHUNK)
 atomic_recipe=${outdir}/logs/recipes/atomic_recipe_${atomic_recipe_number}.yml
 
diff --git a/conf/vars-dict.yml-OLD b/conf/vars-dict.yml-OLD
deleted file mode 100644
index 04549d36..00000000
--- a/conf/vars-dict.yml-OLD
+++ /dev/null
@@ -1,114 +0,0 @@
-
-vars:
-# ECVs 
-  tas:
-    units: "°C"
-    longname: "Daily mean temperature at surface"
-    outname: ~ 
-  tasmin:
-    units: "°C"
-    longname: "Minimum daily temperature at surface"
-    outname: ~ 
-  tasmax:
-    units: "°C"
-    longname: "Maximum daily temperature at surface"
-    outname: ~ 
-  sfcwind:
-    units: "m/s"
-    longname: "Surface wind speed module"
-    outname: ~ 
-  rsds:
-    units: "W/m2"
-    longname: "Surface solar radiation downwards"
-    outname: ~ 
-  psl:
-    units: "hPa"
-    longname: "Mean sea level pressure"
-    outname: ~ 
-  prlr:
-    units: "mm"
-    longname: "Total precipitation"
-    outname: ~ 
-# CFs
-  cfwnd1:
-    units: "%"
-    longname: "Wind Capacity factor IEC1"
-    outname: ~ 
-  cfwnd2:
-    units: "%"
-    longname: "Wind Capacity factor IEC2"
-    outname: ~ 
-  cfwnd3:
-    units: "%"
-    longname: "Wind Capacity factor IEC3"
-    outname: ~ 
-  cfslr:
-    units: "%"
-    longname: "Solar Capacity factor"
-    outname: ~ 
-# Energy
-  edmnd:
-    units: "GW"
-    longname: "Electricity Demmand"
-    outname: ~ 
-  wndpwo:
-    units: "GW"
-    longname: "Wind Power"
-    outname: ~ 
-  dmndnetwnd:
-    units: "GW"
-    longname: "Demmand-net-Wind"
-    outname: ~
-# Indices
-  Spr32:
-    units: "days"
-    longname: >
-              Total count of days when daily maximum temp exceeded 32°C 
-              from April 21st to June 21st
-    outname: ~ 
-  SU35:
-    units: "days"
-    longname: >
-              Total count of days when daily maximum temp exceeded 35°C 
-              from June 21st to September 21st
-    outname: ~ 
-  SU36:
-    units: "days"
-    longname: >
-              Total count of days when daily maximum temp exceeded 36°C 
-              from June 21st to September 21st
-    outname: ~ 
-  SU40:
-    units: "days"
-    longname: >
-              Total count of days when daily maximum temp exceeded 40°C 
-              from June 21st to September 21st
-    outname: ~ 
-  GDD:
-    units: "days"
-    longname: >
-              The sum of the daily differences between daily mean 
-              temperature and 10°C from April 1st to October 31st
-    outname: ~ 
-  GST:
-    units: "°C"
-    longname: "The average temperature from April 1st to October 31st"
-    outname: ~ 
-  SprTX:
-    units: "°C"
-    longname: "The average daily maximum temperature from April 1st to October 31st"
-    outname: ~ 
-  WSDI:
-    units: ""
-    longname: >
-              The total count of days with at least 6 consecutives days 
-              when the daily temperature maximum exceeds its 90th percentile
-    outname: ~ 
-  SprR:
-    units: "mm"
-    longname: 'Total precipitation from April 21st to June 21st'
-    outname: ~ 
-  HarR:
-    units: "mm"
-    longname: 'Total precipitation from August 21st to September 21st'
-    outname: ~ 
diff --git a/launch.sh b/launch.sh
deleted file mode 100644
index 89bc75ce..00000000
--- a/launch.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-source MODULES
-
-# set -vx
-
-# Define recipe and script
-recipe=recipes/tests/recipe_seasonal_example.yml
-script=modules/test_parallel_workflow.R
-
-# Define tmp file to store necessary information
-tmpfile=$(mktemp /tmp/test_split.XXXXXX)
-
-# Create outdir and split recipes
-Rscript modules/split.R ${recipe} | tee $tmpfile
-outdir=$( tail -n 1 $tmpfile | sed 's/"//g' )
-outdir=($outdir)
-
-rm $tmpfile
-
-# Create directory for slurm output
-mkdir -p /esarchive/scratch/$(whoami)/tmp/auto-s2s-logs
-
-## These will have to be separate scripts to work with autosubmit
-## Create a tmpfile inside the projdir with a consistent name??
-## '??' would be a chunk
-# Launch one job per atomic recipe
-for atomic_recipe in ${outdir[1]}/logs/recipes/atomic_recipe_??.yml; do
-  sbatch run_parallel_workflow.sh ${script} ${atomic_recipe}
-done
diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R
index a8c4c613..c4191f24 100644
--- a/modules/Loading/Loading.R
+++ b/modules/Loading/Loading.R
@@ -1,8 +1,9 @@
 ## TODO: remove paths to personal scratchs 
 source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R")
 # Load required libraries/funs
-source("modules/Loading/dates2load.R")
-source("modules/Loading/check_latlon.R")
+source("modules/Loading/R/dates2load.R")
+source("modules/Loading/R/get_timeidx.R")
+source("modules/Loading/R/check_latlon.R")
 ## TODO: Move to prepare_outputs.R
 source("tools/libs.R")
 
diff --git a/modules/Loading/Loading_decadal.R b/modules/Loading/Loading_decadal.R
index 537baf50..b9a145e3 100644
--- a/modules/Loading/Loading_decadal.R
+++ b/modules/Loading/Loading_decadal.R
@@ -8,8 +8,9 @@
 source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R")
 # Load required libraries/funs
 source("modules/Loading/helper_loading_decadal.R")
-source("modules/Loading/dates2load.R")
-source("modules/Loading/check_latlon.R")
+source("modules/Loading/R/dates2load.R")
+source("modules/Loading/R/check_latlon.R")
+source("modules/Loading/R/get_timeidx.R")
 source("tools/libs.R")
 
 #====================================================================
diff --git a/modules/Loading/check_latlon.R b/modules/Loading/R/check_latlon.R
similarity index 100%
rename from modules/Loading/check_latlon.R
rename to modules/Loading/R/check_latlon.R
diff --git a/modules/Loading/dates2load.R b/modules/Loading/R/dates2load.R
similarity index 51%
rename from modules/Loading/dates2load.R
rename to modules/Loading/R/dates2load.R
index 0e3613f3..f084ce62 100644
--- a/modules/Loading/dates2load.R
+++ b/modules/Loading/R/dates2load.R
@@ -49,56 +49,3 @@ dates2load <- function(recipe, logger) {
   dim(data) <- default_dims
   return(data)
 }
-
-# Gets the corresponding dates or indices according
-# to the sdate/leadtimes requested in the recipe
-# 
-# The leadtimes are defined by months
-# Ex. 20201101 with leadtimes 1-4 corresponds to 
-# the forecasting times covering December to March
-
-get_timeidx <- function(sdates, ltmin, ltmax, 
-                        time_freq="monthly_mean") {
-
-  if (time_freq == "daily_mean") {
-
-    sdates <- ymd(sdates)
-    idx_min <- sdates + months(ltmin - 1)
-    idx_max <- sdates + months(ltmax) - days(1)
-
-    day_seq <- seq(idx_min[1], idx_max[1], by = 'days')
-    if (any("0229" %in% (format(day_seq, "%m%d")))) {
-      time_length <- as.integer(idx_max[1]-idx_min[1])
-    } else {
-      time_length <- as.integer(idx_max[1]-idx_min[1]+1)
-    }
-    indxs <- array(numeric(), c(file_date = length(sdates),
-                                time = time_length))
-                                #syear = length(sdates),
-                                #sday = 1, sweek = 1,
-
-    for (sdate in 1:length(sdates)) {
-      day_seq <- seq(idx_min[sdate], idx_max[sdate], by='days')
-      indxs[sdate,] <- day_seq[!(format(day_seq, "%m%d") == "0229")]
-    }
-    indxs <- as.POSIXct(indxs*86400, 
-                        tz = 'UTC', origin = '1970-01-01')
-    lubridate::hour(indxs) <- 12 
-    lubridate::minute(indxs) <- 00
-    dim(indxs) <- c(file_date = length(sdates),
-                    time = time_length)
-
-  } else if (time_freq == "monthly_mean") {
-  
-    idx_min <- ltmin
-    idx_max <- ltmax
-    indxs <- indices(idx_min:idx_max)
-
-  }
-
-  # TODO: 6 hourly case
-  #idx1 <- (sdates + months(ltmin-1) - sdates)*4
-  #idx2 <- idx1 + ndays*4 - 1
-
-  return(indxs)
-}
diff --git a/modules/Loading/R/get_timeidx.R b/modules/Loading/R/get_timeidx.R
new file mode 100644
index 00000000..e4f61a1d
--- /dev/null
+++ b/modules/Loading/R/get_timeidx.R
@@ -0,0 +1,57 @@
+#'Gets the corresponding dates or indices according
+#'to the start dates, min. and max. leadtimes and 
+#'time frequency.
+# 
+#'The leadtimes are defined by months
+#'Ex. 20201101 with leadtimes 1-4 corresponds to 
+#'the forecasting times covering November to february
+#'
+#'@param sdates vector containind the start dates
+#'@param ltmin first leadtime
+#'@param ltmax last leadtime
+#'@param time_freq time frequency ("monthly_mean" or "daily_mean")
+
+get_timeidx <- function(sdates, ltmin, ltmax, 
+                        time_freq="monthly_mean") {
+
+  if (time_freq == "daily_mean") {
+
+    sdates <- ymd(sdates)
+    idx_min <- sdates + months(ltmin - 1)
+    idx_max <- sdates + months(ltmax) - days(1)
+
+    day_seq <- seq(idx_min[1], idx_max[1], by = 'days')
+    if (any("0229" %in% (format(day_seq, "%m%d")))) {
+      time_length <- as.integer(idx_max[1]-idx_min[1])
+    } else {
+      time_length <- as.integer(idx_max[1]-idx_min[1]+1)
+    }
+    indxs <- array(numeric(), c(file_date = length(sdates),
+                                time = time_length))
+                                #syear = length(sdates),
+                                #sday = 1, sweek = 1,
+
+    for (sdate in 1:length(sdates)) {
+      day_seq <- seq(idx_min[sdate], idx_max[sdate], by='days')
+      indxs[sdate,] <- day_seq[!(format(day_seq, "%m%d") == "0229")]
+    }
+    indxs <- as.POSIXct(indxs*86400, 
+                        tz = 'UTC', origin = '1970-01-01')
+    lubridate::hour(indxs) <- 12 
+    lubridate::minute(indxs) <- 00
+    dim(indxs) <- c(file_date = length(sdates),
+                    time = time_length)
+
+  } else if (time_freq == "monthly_mean") {
+  
+    idx_min <- ltmin
+    idx_max <- ltmax
+    indxs <- indices(idx_min:idx_max)
+  }
+
+  # TODO: 6 hourly case
+  #idx1 <- (sdates + months(ltmin-1) - sdates)*4
+  #idx2 <- idx1 + ndays*4 - 1
+
+  return(indxs)
+}
diff --git a/modules/Skill/compute_probs.R b/modules/Skill/R/compute_probs.R
similarity index 100%
rename from modules/Skill/compute_probs.R
rename to modules/Skill/R/compute_probs.R
diff --git a/modules/Skill/compute_quants.R b/modules/Skill/R/compute_quants.R
similarity index 100%
rename from modules/Skill/compute_quants.R
rename to modules/Skill/R/compute_quants.R
diff --git a/modules/Skill/s2s.metrics.R b/modules/Skill/R/s2s.metrics.R
similarity index 100%
rename from modules/Skill/s2s.metrics.R
rename to modules/Skill/R/s2s.metrics.R
diff --git a/modules/Skill/Skill.R b/modules/Skill/Skill.R
index afe697ac..d83d92c1 100644
--- a/modules/Skill/Skill.R
+++ b/modules/Skill/Skill.R
@@ -7,9 +7,9 @@
 # - reliability diagram
 # - ask Carlos which decadal metrics he is currently using
 
-source("modules/Skill/compute_quants.R")
-source("modules/Skill/compute_probs.R")
-source("modules/Skill/s2s.metrics.R")
+source("modules/Skill/R/compute_quants.R")
+source("modules/Skill/R/compute_probs.R")
+source("modules/Skill/R/s2s.metrics.R")
 
 ## TODO: Implement this in the future
 ## Which parameter are required?
diff --git a/run_parallel_workflow.sh b/run_parallel_workflow.sh
deleted file mode 100644
index dd325b8d..00000000
--- a/run_parallel_workflow.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-#SBATCH -n 1
-#SBATCH -t 23:59:59
-#SBATCH -J med
-#SBATCH -e /esarchive/scratch/%u/tmp/auto-s2s-logs/run-%j.err
-#SBATCH -o /esarchive/scratch/%u/tmp/auto-s2s-logs/run-%j.out
-
-script=$1
-atomic_recipe=$2
-
-source MODULES
-
-Rscript ${script} ${atomic_recipe}
diff --git a/split.R b/split.R
index 9a3cbf72..869b47e3 100644
--- a/split.R
+++ b/split.R
@@ -3,10 +3,11 @@ suppressMessages(source("tools/libs.R"))
 # Retrieve recipe file path
 args = commandArgs(trailingOnly = TRUE)
 recipe_file <- args[1]
-# recipe_file <- "recipes/tests/recipe_seasonal_example.yml"
+
 # Check recipe and prepare output directories
 recipe <- prepare_outputs(recipe_file)
 # Split recipe into atomic recipes
+
 ## TODO: Add autosubmit yes/no to the parameters?
 run_parameters <- divide_recipe(recipe)
 if (recipe$Run$autosubmit) {
-- 
GitLab


From 7cb1f70778abd9f807ec6ad1010c6ea6e9854680 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 4 Apr 2023 11:30:34 +0200
Subject: [PATCH 54/57] Remove updated s2dv plotting functions

---
 modules/Visualization/Visualization.R         |   5 -
 modules/Visualization/tmp/PlotCombinedMap.R   | 608 ------------------
 .../tmp/PlotMostLikelyQuantileMap.R           | 196 ------
 modules/Visualization/tmp/clim.palette.R      |  69 --
 .../atomic_recipes/recipe_system7c3s-tas.yml  |   2 +-
 5 files changed, 1 insertion(+), 879 deletions(-)
 delete mode 100644 modules/Visualization/tmp/PlotCombinedMap.R
 delete mode 100644 modules/Visualization/tmp/PlotMostLikelyQuantileMap.R
 delete mode 100644 modules/Visualization/tmp/clim.palette.R

diff --git a/modules/Visualization/Visualization.R b/modules/Visualization/Visualization.R
index 84e0a139..3d065e6c 100644
--- a/modules/Visualization/Visualization.R
+++ b/modules/Visualization/Visualization.R
@@ -1,8 +1,3 @@
-#G# TODO: Remove once released in s2dv/CSTools
-source("modules/Visualization/tmp/PlotMostLikelyQuantileMap.R")
-source("modules/Visualization/tmp/PlotCombinedMap.R")
-source("modules/Visualization/tmp/clim.palette.R")
-
 ## TODO: Add the possibility to read the data directly from netCDF
 ## TODO: Adapt to multi-model case
 ## TODO: Add param 'raw'?
diff --git a/modules/Visualization/tmp/PlotCombinedMap.R b/modules/Visualization/tmp/PlotCombinedMap.R
deleted file mode 100644
index a7b5fc97..00000000
--- a/modules/Visualization/tmp/PlotCombinedMap.R
+++ /dev/null
@@ -1,608 +0,0 @@
-#'Plot Multiple Lon-Lat Variables In a Single Map According to a Decision Function
-#'@description Plot a number a two dimensional matrices with (longitude, latitude) dimensions on a single map with the cylindrical equidistant latitude and longitude projection.
-#'@author Nicolau Manubens, \email{nicolau.manubens@bsc.es}
-#'@author Veronica Torralba, \email{veronica.torralba@bsc.es}
-#'
-#'@param maps List of matrices to plot, each with (longitude, latitude) dimensions, or 3-dimensional array with the dimensions (longitude, latitude, map). Dimension names are required.
-#'@param lon Vector of longitudes. Must match the length of the corresponding dimension in 'maps'.
-#'@param lat Vector of latitudes. Must match the length of the corresponding dimension in 'maps'.
-#'@param map_select_fun Function that selects, for each grid point, which value to take among all the provided maps. This function receives as input a vector of values for a same grid point for all the provided maps, and must return a single selected value (not its index!) or NA. For example, the \code{min} and \code{max} functions are accepted.
-#'@param display_range Range of values to be displayed for all the maps. This must be a numeric vector c(range min, range max). The values in the parameter 'maps' can go beyond the limits specified in this range. If the selected value for a given grid point (according to 'map_select_fun') falls outside the range, it will be coloured with 'col_unknown_map'.
-#'@param map_dim Optional name for the dimension of 'maps' along which the multiple maps are arranged. Only applies when 'maps' is provided as a 3-dimensional array. Takes the value 'map' by default.
-#'@param brks Colour levels to be sent to PlotEquiMap. This parameter is optional and adjusted automatically by the function.
-#'@param cols List of vectors of colours to be sent to PlotEquiMap for the colour bar of each map. This parameter is optional and adjusted automatically by the function (up to 5 maps). The colours provided for each colour bar will be automatically interpolated to match the number of breaks. Each item in this list can be named, and the name will be used as title for the corresponding colour bar (equivalent to the parameter 'bar_titles').
-#'@param col_unknown_map Colour to use to paint the grid cells for which a map is not possible to be chosen according to 'map_select_fun' or for those values that go beyond 'display_range'. Takes the value 'white' by default.
-#'@param mask Optional numeric array with dimensions (latitude, longitude), with values in the range [0, 1], indicating the opacity of the mask over each grid point. Cells with a 0 will result in no mask, whereas cells with a 1 will result in a totally opaque superimposed pixel coloured in 'col_mask'.
-#'@param col_mask Colour to be used for the superimposed mask (if specified in 'mask'). Takes the value 'grey' by default.
-#'@param dots Array of same dimensions as 'var' or with dimensions 
-#'  c(n, dim(var)), where n is the number of dot/symbol layers to add to the 
-#'  plot. A value of TRUE at a grid cell will draw a dot/symbol on the 
-#'  corresponding square of the plot. By default all layers provided in 'dots' 
-#'  are plotted with dots, but a symbol can be specified for each of the 
-#'  layers via the parameter 'dot_symbol'.
-#'@param bar_titles Optional vector of character strings providing the titles to be shown on top of each of the colour bars.
-#'@param legend_scale Scale factor for the size of the colour bar labels. Takes 1 by default.
-#'@param cex_bar_titles Scale factor for the sizes of the bar titles. Takes 1.5 by default.
-#'@param fileout File where to save the plot. If not specified (default) a graphics device will pop up. Extensions allowed: eps/ps, jpeg, png, pdf, bmp and tiff
-#'@param width File width, in the units specified in the parameter size_units (inches by default). Takes 8 by default.
-#'@param height File height, in the units specified in the parameter size_units (inches by default). Takes 5 by default.
-#'@param size_units Units of the size of the device (file or window) to plot in. Inches ('in') by default. See ?Devices and the creator function of the corresponding device.
-#'@param res Resolution of the device (file or window) to plot in. See ?Devices and the creator function of the corresponding device.
-#'@param drawleg Where to draw the common colour bar. Can take values TRUE, 
-#'  FALSE or:\cr
-#'  'up', 'u', 'U', 'top', 't', 'T', 'north', 'n', 'N'\cr
-#'  'down', 'd', 'D', 'bottom', 'b', 'B', 'south', 's', 'S' (default)\cr
-#'  'right', 'r', 'R', 'east', 'e', 'E'\cr
-#'  'left', 'l', 'L', 'west', 'w', 'W' 
-#'@param ... Additional parameters to be passed on to \code{PlotEquiMap}.
-
-#'@seealso \code{PlotCombinedMap} and \code{PlotEquiMap}
-#' 
-#'@importFrom s2dv PlotEquiMap ColorBar
-#'@importFrom maps map 
-#'@importFrom graphics box image layout mtext par plot.new
-#'@importFrom grDevices adjustcolor bmp colorRampPalette dev.cur dev.new dev.off hcl jpeg pdf png postscript svg tiff
-#'@examples
-#'# Simple example
-#'x <- array(1:(20 * 10), dim = c(lat = 10, lon = 20)) / 200
-#'a <- x * 0.6
-#'b <- (1 - x) * 0.6
-#'c <- 1 - (a + b)
-#'lons <- seq(0, 359.5, length = 20)
-#'lats <- seq(-89.5, 89.5, length = 10)
-#'PlotCombinedMap(list(a, b, c), lons, lats, 
-#'                toptitle = 'Maximum map',
-#'                map_select_fun = max,
-#'                display_range = c(0, 1),
-#'                bar_titles = paste('% of belonging to', c('a', 'b', 'c')), 
-#'                brks = 20, width = 10, height = 8)
-#'
-#'Lon <- c(0:40, 350:359)
-#'Lat <- 51:26
-#'data <- rnorm(51 * 26 * 3)
-#'dim(data) <- c(map = 3, lon = 51, lat = 26)
-#'mask <-  sample(c(0,1), replace = TRUE, size = 51 * 26)
-#'dim(mask) <- c(lat = 26, lon = 51)
-#'PlotCombinedMap(data, lon = Lon, lat = Lat, map_select_fun = max,
-#'                display_range = range(data), mask = mask,
-#'                width = 12, height = 8) 
-#'
-#'@export
-PlotCombinedMap <- function(maps, lon, lat, 
-                            map_select_fun, display_range, 
-                            map_dim = 'map',
-                            brks = NULL, cols = NULL,  
-                            col_unknown_map = 'white',
-                            mask = NULL, col_mask = 'grey',
-                            dots = NULL,
-                            bar_titles = NULL, legend_scale = 1,
-                            cex_bar_titles = 1.5,
-                            plot_margin = NULL, bar_margin = rep(0, 4),
-                            fileout = NULL, width = 8, height = 5, 
-                            size_units = 'in', res = 100, drawleg = T,
-                            ...) {
-  args <- list(...)
-
-  # If there is any filenames to store the graphics, process them
-  # to select the right device 
-  if (!is.null(fileout)) {
-    deviceInfo <- .SelectDevice(fileout = fileout, width = width, height = height, 
-                                units = size_units, res = res)
-    saveToFile <- deviceInfo$fun
-    fileout <- deviceInfo$files
-  }
-  
-  # Check probs
-  error <- FALSE
-  if (is.list(maps)) {
-    if (length(maps) < 1) {
-      stop("Parameter 'maps' must be of length >= 1 if provided as a list.")
-    }
-    check_fun <- function(x) {
-      is.numeric(x) && (length(dim(x)) == 2)
-    }
-    if (!all(sapply(maps, check_fun))) {
-      error <- TRUE
-    }
-    ref_dims <- dim(maps[[1]])
-    equal_dims <- all(sapply(maps, function(x) identical(dim(x), ref_dims)))
-    if (!equal_dims) {
-      stop("All arrays in parameter 'maps' must have the same dimension ",
-           "sizes and names when 'maps' is provided as a list of arrays.")
-    }
-    num_maps <- length(maps)
-    maps <- unlist(maps)
-    dim(maps) <- c(ref_dims, map = num_maps)
-    map_dim <- 'map'
-  }
-  if (!is.numeric(maps)) {
-    error <- TRUE
-  }
-  if (is.null(dim(maps))) {
-    error <- TRUE
-  }
-  if (length(dim(maps)) != 3) {
-    error <- TRUE
-  }
-  if (error) {
-    stop("Parameter 'maps' must be either a numeric array with 3 dimensions ",
-         " or a list of numeric arrays of the same size with the 'lon' and ",
-         "'lat' dimensions.")
-  }
-  dimnames <- names(dim(maps))
-  
-  # Check map_dim
-  if (is.character(map_dim)) {
-    if (is.null(dimnames)) {
-      stop("Specified a dimension name in 'map_dim' but no dimension names provided ",
-           "in 'maps'.")
-    }
-    map_dim <- which(dimnames == map_dim)
-    if (length(map_dim) < 1) {
-      stop("Dimension 'map_dim' not found in 'maps'.")
-    } else {
-      map_dim <- map_dim[1]
-    }
-  } else if (!is.numeric(map_dim)) {
-    stop("Parameter 'map_dim' must be either a numeric value or a ",
-         "dimension name.")
-  }
-  if (length(map_dim) != 1) {
-    stop("Parameter 'map_dim' must be of length 1.")
-  }
-  map_dim <- round(map_dim)
-  
-  # Work out lon_dim and lat_dim
-  lon_dim <- NULL
-  if (!is.null(dimnames)) {
-    lon_dim <- which(dimnames %in% c('lon', 'longitude'))[1]
-  }
-  if (length(lon_dim) < 1) {
-    lon_dim <- (1:3)[-map_dim][1]
-  }
-  lon_dim <- round(lon_dim)
-  
-  lat_dim <- NULL
-  if (!is.null(dimnames)) {
-    lat_dim <- which(dimnames %in% c('lat', 'latitude'))[1]
-  }
-  if (length(lat_dim) < 1) {
-    lat_dim <- (1:3)[-map_dim][2]
-  }
-  lat_dim <- round(lat_dim)
-  
-  # Check lon
-  if (!is.numeric(lon)) {
-    stop("Parameter 'lon' must be a numeric vector.")
-  }
-  if (length(lon) != dim(maps)[lon_dim]) {
-    stop("Parameter 'lon' does not match the longitude dimension in 'maps'.")
-  }
-  
-  # Check lat
-  if (!is.numeric(lat)) {
-    stop("Parameter 'lat' must be a numeric vector.")
-  }
-  if (length(lat) != dim(maps)[lat_dim]) {
-    stop("Parameter 'lat' does not match the longitude dimension in 'maps'.")
-  }
-  
-  # Check map_select_fun
-  if (is.numeric(map_select_fun)) {
-    if (length(dim(map_select_fun)) != 2) {
-      stop("Parameter 'map_select_fun' must be an array with dimensions ",
-           "'lon' and 'lat' if provided as an array.")
-    }
-    if (!identical(dim(map_select_fun), dim(maps)[-map_dim])) {
-      stop("The dimensions 'lon' and 'lat' in the 'map_select_fun' array must ",
-           "have the same size, name and order as in the 'maps' parameter.")
-    }
-  }
-  if (!is.function(map_select_fun)) {
-    stop("The parameter 'map_select_fun' must be a function or a numeric array.")
-  }
-  
-  # Check display_range
-  if (!is.numeric(display_range) || length(display_range) != 2) {
-    stop("Parameter 'display_range' must be a numeric vector of length 2.")
-  }
-  
-  # Check brks
-  if (is.null(brks) || (is.numeric(brks) && length(brks) == 1)) {
-    num_brks <- 5
-    if (is.numeric(brks)) {
-      num_brks <- brks
-    }
-    brks <- seq(from = display_range[1], to = display_range[2], length.out = num_brks)
-  }
-  if (!is.numeric(brks)) {
-    stop("Parameter 'brks' must be a numeric vector.")
-  }
-  
-  # Check cols
-  col_sets <- list(c("#A1D99B", "#74C476", "#41AB5D", "#238B45"),
-                   c("#6BAED6FF", "#4292C6FF", "#2171B5FF", "#08519CFF"),
-                   c("#FFEDA0FF", "#FED976FF", "#FEB24CFF", "#FD8D3CFF"),
-                   c("#FC4E2AFF", "#E31A1CFF", "#BD0026FF", "#800026FF"),
-                   c("#FCC5C0", "#FA9FB5", "#F768A1", "#DD3497"))
-  if (is.null(cols)) {
-    if (length(col_sets) >= dim(maps)[map_dim]) {
-      chosen_sets <- 1:(dim(maps)[map_dim])
-      chosen_sets <- chosen_sets + floor((length(col_sets) - length(chosen_sets)) / 2)
-    } else {
-      chosen_sets <- array(1:length(col_sets), dim(maps)[map_dim])
-    }
-    cols <- col_sets[chosen_sets]
-  } else {
-    if (!is.list(cols)) {
-      stop("Parameter 'cols' must be a list of character vectors.")
-    }
-    if (!all(sapply(cols, is.character))) {
-      stop("Parameter 'cols' must be a list of character vectors.")
-    }
-    if (length(cols) != dim(maps)[map_dim]) {
-      stop("Parameter 'cols' must be a list of the same length as the number of ",
-           "maps in 'maps'.")
-    }
-  }
-  for (i in 1:length(cols)) {
-    if (length(cols[[i]]) != (length(brks) - 1)) {
-      cols[[i]] <- colorRampPalette(cols[[i]])(length(brks) - 1)
-    }
-  }
-  
-  # Check bar_titles
-  if (is.null(bar_titles)) {
-    if (!is.null(names(cols))) {
-      bar_titles <- names(cols)
-    } else {
-      bar_titles <- paste0("Map ", 1:length(cols))
-    }
-  } else {
-    if (!is.character(bar_titles)) {
-      stop("Parameter 'bar_titles' must be a character vector.")
-    }
-    if (length(bar_titles) != length(cols)) {
-      stop("Parameter 'bar_titles' must be of the same length as the number of ",
-           "maps in 'maps'.")
-    }
-  }
-  
-  # Check legend_scale
-  if (!is.numeric(legend_scale)) {
-    stop("Parameter 'legend_scale' must be numeric.")
-  }
-  
-  # Check col_unknown_map
-  if (!is.character(col_unknown_map)) {
-    stop("Parameter 'col_unknown_map' must be a character string.")
-  }
-  
-  # Check col_mask
-  if (!is.character(col_mask)) {
-    stop("Parameter 'col_mask' must be a character string.")
-  }
-  
-  # Check mask
-  if (!is.null(mask)) {
-    if (!is.numeric(mask)) {
-      stop("Parameter 'mask' must be numeric.")
-    }
-    if (length(dim(mask)) != 2) {
-      stop("Parameter 'mask' must have two dimensions.")
-    }
-    if ((dim(mask)[1] != dim(maps)[lat_dim]) || 
-        (dim(mask)[2] != dim(maps)[lon_dim])) {
-      stop("Parameter 'mask' must have dimensions c(lat, lon).")
-    }
-  }
-  # Check dots
-  if (!is.null(dots)) {
-    if (length(dim(dots)) != 2) {
-      stop("Parameter 'mask' must have two dimensions.")
-    }
-    if ((dim(dots)[1] != dim(maps)[lat_dim]) ||
-        (dim(dots)[2] != dim(maps)[lon_dim])) {
-      stop("Parameter 'mask' must have dimensions c(lat, lon).")
-    }
-  }
- 
-  #----------------------
-  # Identify the most likely map
-  #----------------------
-  brks_norm <- seq(0, 1, length.out = length(brks))
-  if (is.function(map_select_fun)) {
-    range_width <- display_range[2] - display_range[1]
-    ml_map <- apply(maps, c(lat_dim, lon_dim), function(x) {
-      if (any(is.na(x))) {
-        res <- NA
-      } else {
-        res <- which(x == map_select_fun(x))
-        if (length(res) > 0) {
-          res <- res[1]
-          if (map_select_fun(x) < display_range[1] || 
-              map_select_fun(x) > display_range[2]) {
-            res <- -0.5
-          } else {
-            res <- res + (map_select_fun(x) - display_range[1]) / range_width
-            if (map_select_fun(x) == display_range[1]) {
-	      res <- res + brks_norm[2] / (num_brks * 2)
-            }
-          }
-        } else {
-          res <- -0.5
-        }
-      }
-      res
-    })
-  } else {
-    stop("Providing 'map_select_fun' as array not implemented yet.")
-    ml_map <- map_select_fun
-  }
-  nmap <- dim(maps)[map_dim]
-  nlat <- length(lat)
-  nlon <- length(lon)
-  
-  #----------------------
-  # Set latitudes from minimum to maximum
-  #----------------------
-  if (lat[1] > lat[nlat]){
-    lat <- lat[nlat:1]
-    indices <- list(nlat:1, TRUE)
-    ml_map <- do.call("[", c(list(x = ml_map), indices))
-    if (!is.null(mask)){
-      mask <- mask[nlat:1, ]
-    }
-    if (!is.null(dots)){
-      dots <- dots[nlat:1,]
-    }
-  }
-  
-  #----------------------
-  # Set layout and parameters
-  #----------------------
-  # Open connection to graphical device
-  if (!is.null(fileout)) {
-    saveToFile(fileout)
-  } else if (names(dev.cur()) == 'null device') {
-    dev.new(units = size_units, res = res, width = width, height = height)
-  }
-  #NOTE: I think plot.new() is not necessary in any case.
-#  plot.new()
-  par(font.main = 1)
-  # If colorbars need to be plotted, re-define layout.
-  if (drawleg) {
-    layout(matrix(c(rep(1, nmap),2:(nmap + 1)), 2, nmap, byrow = TRUE), heights = c(6, 1.5))
-  }
-
-  #----------------------
-  # Set colors and breaks and then PlotEquiMap
-  #----------------------
-  tcols <- c(col_unknown_map, cols[[1]])
-  for (k in 2:nmap) {
-    tcols <- append(tcols, c(col_unknown_map, cols[[k]]))
-  }
-
-  tbrks <- c(-1, brks_norm + rep(1:nmap, each = length(brks)))
-
-  if (is.null(plot_margin)) {
-    plot_margin <- c(5, 4, 4, 2) + 0.1  # default of par()$mar
-  }
-
-  PlotEquiMap(var = ml_map, lon = lon, lat = lat, 
-              brks = tbrks, cols = tcols, drawleg = FALSE, 
-              filled.continents = FALSE, dots = dots, mar = plot_margin, ...)
-  
-  #----------------------
-  # Add overplot on top
-  #----------------------
-  if (!is.null(mask)) {
-    dims_mask <- dim(mask)
-    latb <- sort(lat, index.return = TRUE)
-    dlon <- lon[2:dims_mask[2]] - lon[1:(dims_mask[2] - 1)]
-    wher <- which(dlon > (mean(dlon) + 1))
-    if (length(wher) > 0) {
-      lon[(wher + 1):dims_mask[2]] <- lon[(wher + 1):dims_mask[2]] - 360
-    }
-    lonb <- sort(lon, index.return = TRUE)
-
-    cols_mask <- sapply(seq(from = 0, to = 1, length.out = 10), 
-                        function(x) adjustcolor(col_mask, alpha.f = x))
-    image(lonb$x, latb$x, t(mask)[lonb$ix, latb$ix], 
-          axes = FALSE, col = cols_mask, 
-          breaks = seq(from = 0, to = 1, by = 0.1), 
-          xlab='', ylab='', add = TRUE, xpd = TRUE)
-    if (!exists('coast_color')) {
-      coast_color <- 'black'
-    }
-    if (min(lon) < 0) {
-      map('world', interior = FALSE, add = TRUE, lwd = 1, col = coast_color) # Low resolution world map (lon -180 to 180).
-    } else {
-      map('world2', interior = FALSE, add = TRUE, lwd = 1, col = coast_color) # Low resolution world map (lon 0 to 360).
-    }
-    box()
-  }
-  
-  #----------------------
-  # Add colorbars 
-  #----------------------
-  if ('toptitle' %in% names(args)) {
-    size_title <- 1
-    if ('title_scale' %in% names(args)) {
-      size_title <- args[['title_scale']]
-    }
-    old_mar <- par('mar')
-    old_mar[3] <- old_mar[3] - (2 * size_title + 1)
-    par(mar = old_mar)
-  }
-
-  if (drawleg) {
-    for (k in 1:nmap) {
-      ColorBar(brks = brks, cols = cols[[k]], vertical = FALSE,
-               draw_separators = TRUE, extra_margin = c(2, 0, 2, 0),
-               label_scale = legend_scale * 1.5)
-      if (!is.null(bar_titles)) {
-        mtext(bar_titles[[k]], 3, line = -3, cex = cex_bar_titles)
-      }
-      #TODO: Change to below code. Plot title together. extra_margin needs to be adjusted.
-#      ColorBar(brks = brks, cols = cols[[k]], vertical = FALSE, 
-#               draw_separators = TRUE, extra_margin = c(1, 0, 1, 0), 
-#               label_scale = legend_scale * 1.5, title = bar_titles[[k]], title_scale = cex_bar_titles)
-    }
-  }
-  
-  # If the graphic was saved to file, close the connection with the device
-  if (!is.null(fileout)) dev.off()
-}
-
-# Color bar for PlotMostLikelyQuantileMap
-multi_ColorBar <- function(nmap, brks = NULL, cols = NULL, vertical = TRUE, subsampleg = NULL,
-                           bar_limits = NULL, var_limits = NULL,
-                           triangle_ends = NULL, plot = TRUE,
-                           draw_separators = FALSE,
-                           bar_titles = NULL, title_scale = 1, label_scale = 1, extra_margin = rep(0, 4),
-                           ...) {
-
-  minimum_value <- ceiling(1 / nmap * 10 * 1.1) * 10
-  display_range = c(minimum_value, 100)
-
-  # Check brks
-  if (is.null(brks) || (is.numeric(brks) && length(brks) == 1)) {
-    num_brks <- 5
-    if (is.numeric(brks)) {
-      num_brks <- brks
-    }
-    brks <- seq(from = display_range[1], to = display_range[2], length.out = num_brks)
-  }
-  if (!is.numeric(brks)) {
-    stop("Parameter 'brks' must be a numeric vector.")
-  }
-  # Check cols
-  col_sets <- list(c("#A1D99B", "#74C476", "#41AB5D", "#238B45"),
-                   c("#6BAED6FF", "#4292C6FF", "#2171B5FF", "#08519CFF"),
-                   c("#FFEDA0FF", "#FED976FF", "#FEB24CFF", "#FD8D3CFF"),
-                   c("#FC4E2AFF", "#E31A1CFF", "#BD0026FF", "#800026FF"),
-                   c("#FCC5C0", "#FA9FB5", "#F768A1", "#DD3497"))
-  if (is.null(cols)) {
-    if (length(col_sets) >= nmap) {
-      chosen_sets <- 1:nmap
-      chosen_sets <- chosen_sets + floor((length(col_sets) - length(chosen_sets)) / 2)
-    } else {
-      chosen_sets <- array(1:length(col_sets), nmap)
-    }
-    cols <- col_sets[chosen_sets]
-  } else {
-    if (!is.list(cols)) {
-      stop("Parameter 'cols' must be a list of character vectors.")
-    }
-    if (!all(sapply(cols, is.character))) {
-      stop("Parameter 'cols' must be a list of character vectors.")
-    }
-    if (length(cols) != dim(maps)[map_dim]) {
-      stop("Parameter 'cols' must be a list of the same length as the number of ",
-           "maps in 'maps'.")
-    }
-  }
-  for (i in 1:length(cols)) {
-    if (length(cols[[i]]) != (length(brks) - 1)) {
-      cols[[i]] <- grDevices::colorRampPalette(cols[[i]])(length(brks) - 1)
-    }
-  }
-
-  # Check bar_titles
-  if (is.null(bar_titles)) {
-    if (nmap == 3) {
-      bar_titles <- c("Below normal (%)", "Normal (%)", "Above normal (%)")
-    } else if (nmap == 5) {
-      bar_titles <- c("Low (%)", "Below normal (%)",
-                         "Normal (%)", "Above normal (%)", "High (%)")
-    } else {
-      bar_titles <- paste0("Cat. ", 1:nmap, " (%)")
-    }
-  }
-
-  if (plot) {
-    for (k in 1:nmap) {
-      s2dv::ColorBar(brks = brks, cols = cols[[k]], vertical = FALSE, subsampleg = subsampleg,
-                     bar_limits = bar_limits, var_limits = var_limits,
-                     triangle_ends = triangle_ends, plot = TRUE,
-                     draw_separators = draw_separators,
-                     title = bar_titles[[k]], title_scale = title_scale,
-                     label_scale = label_scale, extra_margin = extra_margin)
-    }
-  } else {
-    #TODO: col_inf and col_sup
-    return(list(brks = brks, cols = cols))
-  }
-
-}
-
-#TODO: use s2dv:::.SelectDevice and remove this function here?
-.SelectDevice <- function(fileout, width, height, units, res) {
-  # This function is used in the plot functions to check the extension of the 
-  # files where the graphics will be stored and select the right R device to 
-  # save them.
-  # If the vector of filenames ('fileout') has files with different 
-  # extensions, then it will only accept the first one, changing all the rest 
-  # of the filenames to use that extension.
-
-  # We extract the extension of the filenames: '.png', '.pdf', ...
-  ext <- regmatches(fileout, regexpr("\\.[a-zA-Z0-9]*$", fileout))
-
-  if (length(ext) != 0) {
-    # If there is an extension specified, select the correct device
-    ## units of width and height set to accept inches
-    if (ext[1] == ".png") {
-      saveToFile <- function(fileout) {
-        png(filename = fileout, width = width, height = height, res = res, units = units)
-      }
-    } else if (ext[1] == ".jpeg") {
-      saveToFile <- function(fileout) {
-        jpeg(filename = fileout, width = width, height = height, res = res, units = units)
-      }
-    } else if (ext[1] %in% c(".eps", ".ps")) {
-      saveToFile <- function(fileout) {
-        postscript(file = fileout, width = width, height = height)
-      }
-    } else if (ext[1] == ".pdf") {
-      saveToFile <- function(fileout) {
-        pdf(file = fileout, width = width, height = height)
-      }
-    } else if (ext[1] == ".svg") {
-      saveToFile <- function(fileout) {
-        svg(filename = fileout, width = width, height = height)
-      }
-    } else if (ext[1] == ".bmp") {
-      saveToFile <- function(fileout) {
-        bmp(filename = fileout, width = width, height = height, res = res, units = units)
-      }
-    } else if (ext[1] == ".tiff") {
-      saveToFile <- function(fileout) {
-        tiff(filename = fileout, width = width, height = height, res = res, units = units)
-      }
-    } else {
-      warning("file extension not supported, it will be used '.eps' by default.")
-      ## In case there is only one filename
-      fileout[1] <- sub("\\.[a-zA-Z0-9]*$", ".eps", fileout[1])
-      ext[1] <- ".eps"
-      saveToFile <- function(fileout) {
-        postscript(file = fileout, width = width, height = height)
-      }
-    }
-    # Change filenames when necessary
-    if (any(ext != ext[1])) {
-      warning(paste0("some extensions of the filenames provided in 'fileout' are not ", ext[1],". The extensions are being converted to ", ext[1], "."))
-      fileout <- sub("\\.[a-zA-Z0-9]*$", ext[1], fileout)
-    }
-  } else {
-    # Default filenames when there is no specification
-    warning("there are no extensions specified in the filenames, default to '.eps'")
-    fileout <- paste0(fileout, ".eps")
-    saveToFile <- postscript
-  }
-
-  # return the correct function with the graphical device, and the correct 
-  # filenames
-  list(fun = saveToFile, files = fileout)
-}
-
diff --git a/modules/Visualization/tmp/PlotMostLikelyQuantileMap.R b/modules/Visualization/tmp/PlotMostLikelyQuantileMap.R
deleted file mode 100644
index 9f9f1914..00000000
--- a/modules/Visualization/tmp/PlotMostLikelyQuantileMap.R
+++ /dev/null
@@ -1,196 +0,0 @@
-#'Plot Maps of Most Likely Quantiles
-#'
-#'@author Veronica Torralba, \email{veronica.torralba@bsc.es}, Nicolau Manubens, \email{nicolau.manubens@bsc.es}
-#'@description This function receives as main input (via the parameter \code{probs}) a collection of longitude-latitude maps, each containing the probabilities (from 0 to 1) of the different grid cells of belonging to a category. As many categories as maps provided as inputs are understood to exist. The maps of probabilities must be provided on a common rectangular regular grid, and a vector with the longitudes and a vector with the latitudes of the grid must be provided. The input maps can be provided in two forms, either as a list of multiple two-dimensional arrays (one for each category) or as a three-dimensional array, where one of the dimensions corresponds to the different categories.
-#'
-#'@param probs a list of bi-dimensional arrays with the named dimensions 'latitude' (or 'lat') and 'longitude' (or 'lon'), with equal size and in the same order, or a single tri-dimensional array with an additional dimension (e.g. 'bin') for the different categories. The arrays must contain probability values between 0 and 1, and the probabilities for all categories of a grid cell should not exceed 1 when added.
-#'@param lon a numeric vector with the longitudes of the map grid, in the same order as the values along the corresponding dimension in \code{probs}.
-#'@param lat a numeric vector with the latitudes of the map grid, in the same order as the values along the corresponding dimension in \code{probs}.
-#'@param cat_dim the name of the dimension along which the different categories are stored in \code{probs}. This only applies if \code{probs} is provided in the form of 3-dimensional array. The default expected name is 'bin'.
-#'@param bar_titles vector of character strings with the names to be drawn on top of the color bar for each of the categories. As many titles as categories provided in \code{probs} must be provided.
-#'@param col_unknown_cat character string with a colour representation of the colour to be used to paint the cells for which no category can be clearly assigned. Takes the value 'white' by default.
-#'@param drawleg Where to draw the common colour bar. Can take values TRUE, 
-#'  FALSE or:\cr
-#'  'up', 'u', 'U', 'top', 't', 'T', 'north', 'n', 'N'\cr
-#'  'down', 'd', 'D', 'bottom', 'b', 'B', 'south', 's', 'S' (default)\cr
-#'  'right', 'r', 'R', 'east', 'e', 'E'\cr
-#'  'left', 'l', 'L', 'west', 'w', 'W'
-#'@param ... additional parameters to be sent to \code{PlotCombinedMap} and \code{PlotEquiMap}.
-#'@seealso \code{PlotCombinedMap} and \code{PlotEquiMap}
-#' 
-#'@importFrom maps map 
-#'@importFrom graphics box image layout mtext par plot.new
-#'@importFrom grDevices adjustcolor bmp colorRampPalette dev.cur dev.new dev.off hcl jpeg pdf png postscript svg tiff
-#'@examples
-#'# Simple example
-#'x <- array(1:(20 * 10), dim = c(lat = 10, lon = 20)) / 200
-#'a <- x * 0.6
-#'b <- (1 - x) * 0.6
-#'c <- 1 - (a + b)
-#'lons <- seq(0, 359.5, length = 20)
-#'lats <- seq(-89.5, 89.5, length = 10)
-#'PlotMostLikelyQuantileMap(list(a, b, c), lons, lats, 
-#'                          toptitle = 'Most likely tercile map',
-#'                          bar_titles = paste('% of belonging to', c('a', 'b', 'c')), 
-#'                          brks = 20, width = 10, height = 8)
-#'
-#'# More complex example
-#'n_lons <- 40
-#'n_lats <- 20
-#'n_timesteps <- 100
-#'n_bins <- 4
-#'
-#'# 1. Generation of sample data
-#'lons <- seq(0, 359.5, length = n_lons)
-#'lats <- seq(-89.5, 89.5, length = n_lats)
-#'
-#'# This function builds a 3-D gaussian at a specified point in the map.
-#'make_gaussian <- function(lon, sd_lon, lat, sd_lat) {
-#'  w <- outer(lons, lats, function(x, y) dnorm(x, lon, sd_lon) * dnorm(y, lat, sd_lat))
-#'  min_w <- min(w)
-#'  w <- w - min_w
-#'  w <- w / max(w)
-#'  w <- t(w)
-#'  names(dim(w)) <- c('lat', 'lon')
-#'  w
-#'}
-#'
-#'# This function generates random time series (with values ranging 1 to 5) 
-#'# according to 2 input weights.
-#'gen_data <- function(w1, w2, n) {
-#'  r <- sample(1:5, n, 
-#'              prob = c(.05, .9 * w1, .05, .05, .9 * w2), 
-#'              replace = TRUE)
-#'  r <- r + runif(n, -0.5, 0.5)
-#'  dim(r) <- c(time = n)
-#'  r
-#'}
-#'
-#'# We build two 3-D gaussians.
-#'w1 <- make_gaussian(120, 80, 20, 30)
-#'w2 <- make_gaussian(260, 60, -10, 40)
-#'
-#'# We generate sample data (with dimensions time, lat, lon) according 
-#'# to the generated gaussians
-#'sample_data <- multiApply::Apply(list(w1, w2), NULL, 
-#'                                 gen_data, n = n_timesteps)$output1
-#'
-#'# 2. Binning sample data
-#'prob_thresholds <- 1:n_bins / n_bins
-#'prob_thresholds <- prob_thresholds[1:(n_bins - 1)]
-#'thresholds <- quantile(sample_data, prob_thresholds)
-#'
-#'binning <- function(x, thresholds) {
-#'  n_samples <- length(x)
-#'  n_bins <- length(thresholds) + 1
-#'
-#'  thresholds <- c(thresholds, max(x))
-#'  result <- 1:n_bins
-#'  lower_threshold <- min(x) - 1
-#'  for (i in 1:n_bins) {
-#'    result[i] <- sum(x > lower_threshold & x <= thresholds[i]) / n_samples
-#'    lower_threshold <- thresholds[i]
-#'  }
-#'
-#'  dim(result) <- c(bin = n_bins)
-#'  result
-#'}
-#'
-#'bins <- multiApply::Apply(sample_data, 'time', binning, thresholds)$output1
-#'
-#'# 3. Plotting most likely quantile/bin
-#'PlotMostLikelyQuantileMap(bins, lons, lats, 
-#'                          toptitle = 'Most likely quantile map',
-#'                          bar_titles = paste('% of belonging to', letters[1:n_bins]),
-#'                          mask = 1 - (w1 + w2 / max(c(w1, w2))), 
-#'                          brks = 20, width = 10, height = 8)
-#'
-#'@export
-PlotMostLikelyQuantileMap <- function(probs, lon, lat, cat_dim = 'bin',
-                                      bar_titles = NULL, 
-                                      col_unknown_cat = 'white', drawleg = T,
-                                      ...) {
-  # Check probs
-  error <- FALSE
-  if (is.list(probs)) {
-    if (length(probs) < 1) {
-      stop("Parameter 'probs' must be of length >= 1 if provided as a list.")
-    }
-    check_fun <- function(x) {
-      is.numeric(x) && (length(dim(x)) == 2)
-    }
-    if (!all(sapply(probs, check_fun))) {
-      error <- TRUE
-    }
-    ref_dims <- dim(probs[[1]])
-    equal_dims <- all(sapply(probs, function(x) identical(dim(x), ref_dims)))
-    if (!equal_dims) {
-      stop("All arrays in parameter 'probs' must have the same dimension ",
-           "sizes and names when 'probs' is provided as a list of arrays.")
-    }
-    num_probs <- length(probs)
-    probs <- unlist(probs)
-    dim(probs) <- c(ref_dims, map = num_probs)
-    cat_dim <- 'map'
-  }
-  if (!is.numeric(probs)) {
-    error <- TRUE
-  }
-  if (is.null(dim(probs))) {
-    error <- TRUE
-  }
-  if (length(dim(probs)) != 3) {
-    error <- TRUE
-  }
-  if (error) {
-    stop("Parameter 'probs' must be either a numeric array with 3 dimensions ",
-         " or a list of numeric arrays of the same size with the 'lon' and ",
-         "'lat' dimensions.")
-  }
-  dimnames <- names(dim(probs))
-  
-  # Check cat_dim
-  if (is.character(cat_dim)) {
-    if (is.null(dimnames)) {
-      stop("Specified a dimension name in 'cat_dim' but no dimension names provided ",
-           "in 'probs'.")
-    }
-    cat_dim <- which(dimnames == cat_dim)
-    if (length(cat_dim) < 1) {
-      stop("Dimension 'cat_dim' not found in 'probs'.")
-    }
-    cat_dim <- cat_dim[1]
-  } else if (!is.numeric(cat_dim)) {
-    stop("Parameter 'cat_dim' must be either a numeric value or a ",
-         "dimension name.")
-  }
-  if (length(cat_dim) != 1) {
-    stop("Parameter 'cat_dim' must be of length 1.")
-  }
-  cat_dim <- round(cat_dim)
-  nprobs <- dim(probs)[cat_dim]
-  
-  # Check bar_titles
-  if (is.null(bar_titles)) {
-    if (nprobs == 3) {
-      bar_titles <- c("Below normal (%)", "Normal (%)", "Above normal (%)")
-    } else if (nprobs == 5) {
-      bar_titles <- c("Low (%)", "Below normal (%)", 
-                         "Normal (%)", "Above normal (%)", "High (%)")
-    } else {
-      bar_titles <- paste0("Cat. ", 1:nprobs, " (%)")
-    }
-  }
-  
-  minimum_value <- ceiling(1 / nprobs * 10 * 1.1) * 10
-
-  # By now, the PlotCombinedMap function is included below in this file. 
-  # In the future, PlotCombinedMap will be part of s2dverification and will 
-  # be properly imported.
-  PlotCombinedMap(probs * 100, lon, lat, map_select_fun = max, 
-                  display_range = c(minimum_value, 100),
-                  map_dim = cat_dim, 
-                  bar_titles = bar_titles,
-                  col_unknown_map = col_unknown_cat, 
-                  drawleg = drawleg, ...)
-}
diff --git a/modules/Visualization/tmp/clim.palette.R b/modules/Visualization/tmp/clim.palette.R
deleted file mode 100644
index b23ff842..00000000
--- a/modules/Visualization/tmp/clim.palette.R
+++ /dev/null
@@ -1,69 +0,0 @@
-#'Generate Climate Color Palettes
-#'
-#'Generates a colorblind friendly color palette with color ranges useful in 
-#'climate temperature variable plotting.
-#'
-#'@param palette Which type of palette to generate: from blue through white 
-#'  to red ('bluered'), from red through white to blue ('redblue'), from 
-#'  yellow through orange to red ('yellowred'),  from red through orange to
-#'  red ('redyellow'), from purple through white to orange ('purpleorange'),
-#'  and from orange through white to purple ('orangepurple').
-#'@param n Number of colors to generate.
-#'
-#'@examples
-#'lims <- seq(-1, 1, length.out = 21)
-#'
-#'ColorBar(lims, color_fun = clim.palette('redyellow'))
-#'
-#'cols <- clim.colors(20)
-#'ColorBar(lims, cols)
-#'
-#'@rdname clim.palette
-#'@importFrom grDevices colorRampPalette
-#'@export
-clim.palette <- function(palette = "bluered") {
-  if (palette == "bluered") {
-    colorbar <- colorRampPalette(rev(c("#67001f", "#b2182b", "#d6604d",
-                                       "#f4a582", "#fddbc7", "#f7f7f7", 
-                                       "#d1e5f0", "#92c5de", "#4393c3", 
-                                       "#2166ac", "#053061")))
-    attr(colorbar, 'na_color') <- 'pink'
-  } else if (palette == "redblue") {
-    colorbar <- colorRampPalette(c("#67001f", "#b2182b", "#d6604d",
-                                   "#f4a582", "#fddbc7", "#f7f7f7", 
-                                   "#d1e5f0", "#92c5de", "#4393c3", 
-                                   "#2166ac", "#053061"))
-    attr(colorbar, 'na_color') <- 'pink'
-  } else if (palette == "yellowred") {
-    colorbar <- colorRampPalette(c("#ffffcc", "#ffeda0", "#fed976",
-                                   "#feb24c", "#fd8d3c", "#fc4e2a",
-                                   "#e31a1c", "#bd0026", "#800026"))
-    attr(colorbar, 'na_color') <- 'pink'
-  } else if (palette == "redyellow") {
-    colorbar <- colorRampPalette(rev(c("#ffffcc", "#ffeda0", "#fed976",
-                                       "#feb24c", "#fd8d3c", "#fc4e2a",
-                                       "#e31a1c", "#bd0026", "#800026")))
-    attr(colorbar, 'na_color') <- 'pink'
-  } else if (palette == "purpleorange") {
-    colorbar <- colorRampPalette(c("#2d004b", "#542789", "#8073ac",
-				   "#b2abd2", "#d8daeb", "#f7f7f7",
-				   "#fee0b6", "#fdb863", "#e08214", 
-				   "#b35806", "#7f3b08"))
-    attr(colorbar, 'na_color') <- 'pink'
-  } else if (palette == "orangepurple") {
-    colorbar <- colorRampPalette(rev(c("#2d004b", "#542789", "#8073ac",
-				       "#b2abd2", "#d8daeb", "#f7f7f7",
-				       "#fee0b6", "#fdb863", "#e08214",
-				       "#b35806", "#7f3b08")))
-    attr(colorbar, 'na_color') <- 'pink'
-  } else {
-    stop("Parameter 'palette' must be one of 'bluered', 'redblue', 'yellowred'",
-	 "'redyellow', 'purpleorange' or 'orangepurple'.")
-  }
-  colorbar
-}
-#'@rdname clim.palette
-#'@export
-clim.colors <- function(n, palette = "bluered") {
-  clim.palette(palette)(n)
-}
diff --git a/recipes/atomic_recipes/recipe_system7c3s-tas.yml b/recipes/atomic_recipes/recipe_system7c3s-tas.yml
index 47cfc31b..a2a87c23 100644
--- a/recipes/atomic_recipes/recipe_system7c3s-tas.yml
+++ b/recipes/atomic_recipes/recipe_system7c3s-tas.yml
@@ -45,5 +45,5 @@ Analysis:
 Run:
   Loglevel: INFO
   Terminal: yes
-  output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/
+  output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/
   code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/
-- 
GitLab


From 547e251d61ca52fa7bceafc0f4c74f7b2118f2da Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 4 Apr 2023 11:39:53 +0200
Subject: [PATCH 55/57] Change recipe

---
 modules/test_seasonal.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/test_seasonal.R b/modules/test_seasonal.R
index cddc1b37..4dd34b61 100644
--- a/modules/test_seasonal.R
+++ b/modules/test_seasonal.R
@@ -5,7 +5,7 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
-recipe_file <- "recipes/tests/recipe_seasonal_two-variables.yml"
+recipe_file <- "recipes/atomic_recipes/recipe_system7c3s-tas.yml"
 recipe <- prepare_outputs(recipe_file)
 
 # Load datasets
-- 
GitLab


From 130ffbcc91538a0b84e82860121dcfc60ab8a3e2 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 4 Apr 2023 13:06:49 +0200
Subject: [PATCH 56/57] Add region to path if multiple regions are chosen

---
 modules/Saving/paths2save.R               |  5 +-
 recipes/tests/recipe_multiregion.yml      | 76 +++++++++++++++++++++++
 recipes/tests/recipe_seasonal_example.yml | 40 ++++++------
 3 files changed, 100 insertions(+), 21 deletions(-)
 create mode 100644 recipes/tests/recipe_multiregion.yml

diff --git a/modules/Saving/paths2save.R b/modules/Saving/paths2save.R
index 93196b86..d22d9856 100644
--- a/modules/Saving/paths2save.R
+++ b/modules/Saving/paths2save.R
@@ -48,7 +48,7 @@ get_filename <- function(dir, recipe, var, date, agg, file.type) {
   	   "exp"   = {file <- paste0(var, gg, "_", date)},
   	   "obs"   = {file <- paste0(var, gg, "-obs_", date)},
   	   "percentiles" = {file <- paste0(var, gg, "-percentiles_", dd, 
-  	          			 shortdate)},
+  	          			   shortdate)},
   	   "probs" = {file <- paste0(var, gg, "-probs_", date)},
   	   "bias"  = {file <- paste0(var, gg, "-bias_", date)})
   }
@@ -96,6 +96,9 @@ get_dir <- function(recipe, agg = "global") {
     calib.method <- tolower(recipe$Analysis$Workflow$Calibration$method)
     store.freq <- recipe$Analysis$Variables$freq
     ## TODO: Change "_country"
+    if (!is.null(recipe$Analysis$Region$name)) {
+      outdir <- paste0(outdir, "/", recipe$Analysis$Region$name)
+    }
     switch(tolower(agg),
   	 "country" = {dir <- paste0(outdir, "/", system, "/", calib.method,
 				    "-", store.freq, "/", variable, 
diff --git a/recipes/tests/recipe_multiregion.yml b/recipes/tests/recipe_multiregion.yml
new file mode 100644
index 00000000..bcb4d126
--- /dev/null
+++ b/recipes/tests/recipe_multiregion.yml
@@ -0,0 +1,76 @@
+################################################################################ 
+## RECIPE DESCRIPTION  
+################################################################################ 
+
+Description:
+  Author: V. Agudetse
+  Info: Test Independent verification of two variables, two sdates, two systems
+
+################################################################################
+## ANALYSIS CONFIGURATION
+################################################################################
+
+Analysis:
+  Horizon: Seasonal
+  Variables:
+    - {name: tas, freq: monthly_mean}
+  Datasets:
+    System:
+      - {name: ECMWF-SEAS5}
+      - {name: Meteo-France-System7}
+    Multimodel: no
+    Reference:
+      - {name: ERA5}
+  Time:
+    sdate:
+      - '0101'
+      - '0601'
+    fcst_year:
+    hcst_start: '2000'
+    hcst_end: '2016'
+    ftime_min: 1
+    ftime_max: 6
+  Region:
+    - {name: "tropics", latmin: -5, latmax: 5, lonmin: -10, lonmax: 10}
+    - {name: "nino34", latmin: -5, latmax: 5, lonmin: -10, lonmax: 60}
+  Regrid:
+    method: conservative
+    type: to_system
+  Workflow:
+    Anomalies:
+      compute: yes
+      cross_validation: yes
+    Calibration:
+      method: raw
+    Skill:
+      metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS
+    Probabilities:
+      percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]]
+    Indicators:
+      index: no
+  ncores: 8
+  remove_NAs: yes
+  Output_format: S2S4E
+
+################################################################################
+## Run CONFIGURATION
+################################################################################
+Run:
+  Loglevel: INFO
+  Terminal: yes
+  filesystem: esarchive
+  output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/
+  code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/
+  autosubmit: yes
+  # fill only if using autosubmit
+  auto_conf:
+    script: /esarchive/scratch/vagudets/repos/auto-s2s/modules/test_parallel_workflow.R
+    expid: a5no # autosubmit experiment ID
+    hpc_user: bsc32762 # your hpc username
+    wallclock: 04:00 # hh:mm
+    processors_per_job: 8 # use ncores parameter?
+    platform: nord3v2 # make this constant?
+    email_notifications: yes # enable/disable email notifications
+    email_address: victoria.agudetse@bsc.es # email address for notifications
+    notify_completed: no # notify me by email when a job finishes
+    notify_failed: yes # notify me by email when a job fails
diff --git a/recipes/tests/recipe_seasonal_example.yml b/recipes/tests/recipe_seasonal_example.yml
index 0a22d3ab..cb941f84 100644
--- a/recipes/tests/recipe_seasonal_example.yml
+++ b/recipes/tests/recipe_seasonal_example.yml
@@ -12,26 +12,26 @@ Description:
 
 Analysis:
   Horizon: Seasonal
-  Variables: # ECVs and Indicators?
+  Variables:
     - {name: tas, freq: monthly_mean}
     - {name: prlr, freq: monthly_mean}
   Datasets:
-    System: # multiple systems for single model, split if Multimodel = F
+    System:
       - {name: ECMWF-SEAS5}
       - {name: Meteo-France-System7}
-    Multimodel: False # single option
+    Multimodel: no
     Reference:
-      - {name: ERA5} # multiple references for single model?
+      - {name: ERA5}
   Time:
-    sdate: # list, split
+    sdate:
       - '0101'
-      - '0201'
-    fcst_year: # list, don't split, handled internally
-    hcst_start: '2000' # single option
-    hcst_end: '2016' # single option
-    ftime_min: 1 # single option
-    ftime_max: 6 # single option
-  Region: # multiple lists, Add region name if there is more than 1 region
+      - '0601'
+    fcst_year:
+    hcst_start: '2000'
+    hcst_end: '2016'
+    ftime_min: 1
+    ftime_max: 6
+  Region:
     - {latmin: -10, latmax: 10, lonmin: -10, lonmax: 10}
   Regrid:
     method: conservative
@@ -41,16 +41,16 @@ Analysis:
       compute: yes
       cross_validation: yes
     Calibration:
-      method: raw ## TODO: list, split?
+      method: raw
     Skill:
-      metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS # list, don't split
+      metric: RPS, RPSS, CRPS, CRPSS, FRPSS, BSS10, BSS90, mean_bias, mean_bias_SS
     Probabilities:
-      percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]] # list, don't split
+      percentiles: [[1/3, 2/3], [1/10, 9/10], [1/4, 2/4, 3/4]]
     Indicators:
-      index: no # ? 
+      index: no
   ncores: 8
-  remove_NAs: yes # bool, don't split
-  Output_format: S2S4E # string, don't split
+  remove_NAs: yes
+  Output_format: S2S4E
 
 ################################################################################
 ## Run CONFIGURATION
@@ -64,8 +64,8 @@ Run:
   autosubmit: yes
   # fill only if using autosubmit
   auto_conf:
-    script: /esarchive/scratch/vagudets/repos/auto-s2s/modules/test_parallel_workflow.R # path to the script to run
-    expid: a5no ## if left empty, create new exp?
+    script: /esarchive/scratch/vagudets/repos/auto-s2s/modules/test_parallel_workflow.R
+    expid: a5no # autosubmit experiment ID
     hpc_user: bsc32762 # your hpc username
     wallclock: 04:00 # hh:mm
     processors_per_job: 8 # use ncores parameter?
-- 
GitLab


From d2bf2ae9b5281513d37a82a66bef8649c75cea11 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 6 Apr 2023 11:52:01 +0200
Subject: [PATCH 57/57] Add checks for script in auto_conf

---
 tools/check_recipe.R | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/tools/check_recipe.R b/tools/check_recipe.R
index fe8260dc..eaec8f9a 100644
--- a/tools/check_recipe.R
+++ b/tools/check_recipe.R
@@ -346,9 +346,9 @@ check_recipe <- function(recipe) {
   # AUTOSUBMIT CHECKS
   # ---------------------------------------------------------------------
 
-  AUTO_PARAMS <- c("expid", "hpc_user", "wallclock", "processors_per_job",
-		   "platform", "email_notifications", "email_address",
-		   "notify_completed", "notify_failed")
+  AUTO_PARAMS <- c("script", "expid", "hpc_user", "wallclock",
+		   "processors_per_job", "platform", "email_notifications",
+		   "email_address", "notify_completed", "notify_failed")
   # Autosubmit false by default
   if (is.null(recipe$Run$autosubmit)) {
     recipe$Run$autosubmit <- F
@@ -368,6 +368,16 @@ check_recipe <- function(recipe) {
 		   "following: ", paste(AUTO_PARAMS, collapse = ", "), "."))
       error_status <- T
     }
+    # Check that the script is not NULL and exists
+    if (is.null(recipe$Run$auto_conf$script)) {
+      error(recipe$Run$logger,
+	    "A script must be provided to run the recipe with autosubmit.")
+      error_status <- T
+    } else if (!file.exists(recipe$Run$auto_conf$script)) {
+      error(recipe$Run$logger,
+	    "Could not find the file for the script in 'auto_conf'.")
+      error_status <- T
+    }
     # Check that the experiment ID exists
     if (is.null(recipe$Run$auto_conf$expid)) {
       error(recipe$Run$logger,
-- 
GitLab