From b0bf948dfb9452724eb77322e4dd8d727777de9d Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 13 Oct 2022 08:45:08 +0200
Subject: [PATCH 01/13] Rearrange function arguments for consistency, make
 archive optional

---
 modules/Calibration/Calibration.R       |  2 +-
 modules/Saving/Saving.R                 | 19 +++++++++++++------
 modules/Skill/Skill.R                   |  4 ++--
 modules/Visualization/Visualization.R   | 12 +++++++++++-
 tests/testthat/test-decadal_monthly_1.R |  6 +++---
 tests/testthat/test-decadal_monthly_2.R |  6 +++---
 tests/testthat/test-decadal_monthly_3.R |  6 +++---
 tests/testthat/test-seasonal_daily.R    |  4 ++--
 tests/testthat/test-seasonal_monthly.R  | 14 +++++++-------
 9 files changed, 45 insertions(+), 28 deletions(-)

diff --git a/modules/Calibration/Calibration.R b/modules/Calibration/Calibration.R
index 85b1b007..d49dd9d9 100644
--- a/modules/Calibration/Calibration.R
+++ b/modules/Calibration/Calibration.R
@@ -3,7 +3,7 @@
 source("tools/tmp/CST_Calibration.R")
 
 ## Entry params data and recipe?
-calibrate_datasets <- function(data, recipe) {
+calibrate_datasets <- function(recipe, data) {
   # Function that calibrates the hindcast using the method stated in the
   # recipe. If the forecast is not null, it calibrates it as well.
   #
diff --git a/modules/Saving/Saving.R b/modules/Saving/Saving.R
index 713741fb..cf2ec1d0 100644
--- a/modules/Saving/Saving.R
+++ b/modules/Saving/Saving.R
@@ -2,10 +2,11 @@
 
 source("modules/Saving/paths2save.R")
 
-save_data <- function(recipe, archive, data,
+save_data <- function(recipe, data,
 		      calibrated_data = NULL,
 		      skill_metrics = NULL,
-		      probabilities = NULL) {
+		      probabilities = NULL,
+		      archive = NULL) {
 
   # Wrapper for the saving functions.
   # recipe: The auto-s2s recipe
@@ -19,14 +20,20 @@ save_data <- function(recipe, archive, data,
   if (is.null(recipe)) {
     stop("The 'recipe' parameter is mandatory.")
   }
-  if (is.null(archive)) {
-    stop("The 'archive' parameter is mandatory.")
-  }
+
   if (is.null(data)) {
     stop("The 'data' parameter is mandatory. It should be the output of",
 	 "load_datasets().")
   }
-
+  if (is.null(archive)) {
+    if (tolower(recipe$Analysis$Horizon) == "seasonal") {
+      archive <- read_yaml(paste0(recipe$Run$code_dir,
+	                          "conf/archive.yml"))$archive
+    } else if (tolower(recipe$Analysis$Horizon) == "decadal") {
+      archive <- read_yaml(paste0(recipe$Run$code_dir,
+				  "conf/archive_decadal.yml"))$archive
+    }
+  }
   dict <- read_yaml("conf/variable-dictionary.yml")
 
   # Create output directory
diff --git a/modules/Skill/Skill.R b/modules/Skill/Skill.R
index fb5498e6..726cd605 100644
--- a/modules/Skill/Skill.R
+++ b/modules/Skill/Skill.R
@@ -53,7 +53,7 @@ source("modules/Skill/tmp/AbsBiasSS.R")
 #                    " running Skill module ", "\n",
 #                    " it can call ", metric_fun ))
 
-compute_skill_metrics <- function(exp, obs, recipe) {
+compute_skill_metrics <- function(recipe, exp, obs) {
   # exp: s2dv_cube containing the hindcast
   # obs: s2dv_cube containing the observations
   # recipe: auto-s2s recipe as provided by read_yaml
@@ -207,7 +207,7 @@ compute_skill_metrics <- function(exp, obs, recipe) {
   return(skill_metrics)
 }
 
-compute_probabilities <- function(data, recipe) {
+compute_probabilities <- function(recipe, data) {
 
   if (is.null(recipe$Analysis$ncores)) {
     ncores <- 1
diff --git a/modules/Visualization/Visualization.R b/modules/Visualization/Visualization.R
index c1bf9121..4b63de8c 100644
--- a/modules/Visualization/Visualization.R
+++ b/modules/Visualization/Visualization.R
@@ -8,11 +8,11 @@ source("modules/Visualization/tmp/PlotLayout.R")
 ## TODO: Add param 'raw'?
 
 plot_data <- function(recipe,
-		      archive,
 		      data,
 		      calibrated_data = NULL,
 		      skill_metrics = NULL,
 		      probabilities = NULL,
+		      archive = NULL,
 		      significance = F) {
 
   # Try to produce and save several basic plots.
@@ -33,6 +33,16 @@ plot_data <- function(recipe,
 	 "that can be plotted.")
   }
 
+  if (is.null(archive)) {
+    if (tolower(recipe$Analysis$Horizon) == "seasonal") {
+      archive <- read_yaml(paste0(recipe$Run$code_dir,
+				  "conf/archive.yml"))$archive
+    } else if (tolower(recipe$Analysis$Horizon) == "decadal") {
+      archive <- read_yaml(paste0(recipe$Run$code_dir,
+				   "conf/archive_decadal.yml"))$archive
+    }
+  }
+
   # Plot skill metrics
   if (!is.null(skill_metrics)) {
     plot_skill_metrics(recipe, archive, data$hcst, skill_metrics, outdir,
diff --git a/tests/testthat/test-decadal_monthly_1.R b/tests/testthat/test-decadal_monthly_1.R
index 7bb5031e..39c8d900 100644
--- a/tests/testthat/test-decadal_monthly_1.R
+++ b/tests/testthat/test-decadal_monthly_1.R
@@ -19,15 +19,15 @@ data <- load_datasets(recipe_file)
 
 # Calibrate datasets
 suppressWarnings({invisible(capture.output(
-  calibrated_data <- calibrate_datasets(data, recipe)
+  calibrated_data <- calibrate_datasets(recipe, data)
 ))})
 
 # Compute skill metrics
 suppressWarnings({invisible(capture.output(
-skill_metrics <- compute_skill_metrics(calibrated_data$hcst, data$obs, recipe)
+skill_metrics <- compute_skill_metrics(recipe, calibrated_data$hcst, data$obs)
 ))})
 suppressWarnings({invisible(capture.output(
-probs <- compute_probabilities(calibrated_data$hcst, recipe)
+probs <- compute_probabilities(recipe, calibrated_data$hcst)
 ))})
 
 # Saving
diff --git a/tests/testthat/test-decadal_monthly_2.R b/tests/testthat/test-decadal_monthly_2.R
index ac4f2fff..98fa66cb 100644
--- a/tests/testthat/test-decadal_monthly_2.R
+++ b/tests/testthat/test-decadal_monthly_2.R
@@ -18,15 +18,15 @@ data <- load_datasets(recipe_file)
 
 # Calibrate datasets
 suppressWarnings({invisible(capture.output(
-  calibrated_data <- calibrate_datasets(data, recipe)
+  calibrated_data <- calibrate_datasets(recipe, data)
 ))})
 
 # Compute skill metrics
 suppressMessages({invisible(capture.output(
-skill_metrics <- compute_skill_metrics(calibrated_data$hcst, data$obs, recipe)
+skill_metrics <- compute_skill_metrics(recipe, calibrated_data$hcst, data$obs)
 ))})
 suppressWarnings({invisible(capture.output(
-probs <- compute_probabilities(calibrated_data$hcst, recipe)
+probs <- compute_probabilities(recipe, calibrated_data$hcst)
 ))})
 
 #======================================
diff --git a/tests/testthat/test-decadal_monthly_3.R b/tests/testthat/test-decadal_monthly_3.R
index 21665f6e..22f47d4c 100644
--- a/tests/testthat/test-decadal_monthly_3.R
+++ b/tests/testthat/test-decadal_monthly_3.R
@@ -18,15 +18,15 @@ data <- load_datasets(recipe_file)
 
 # Calibrate datasets
 suppressWarnings({invisible(capture.output(
-  calibrated_data <- calibrate_datasets(data, recipe)
+  calibrated_data <- calibrate_datasets(recipe, data)
 ))})
 
 # Compute skill metrics
 suppressWarnings({invisible(capture.output(
-skill_metrics <- compute_skill_metrics(calibrated_data$hcst, data$obs, recipe)
+skill_metrics <- compute_skill_metrics(recipe, calibrated_data$hcst, data$obs)
 ))})
 suppressWarnings({invisible(capture.output(
-probs <- compute_probabilities(calibrated_data$hcst, recipe)
+probs <- compute_probabilities(recipe, calibrated_data$hcst)
 ))})
 
 #======================================
diff --git a/tests/testthat/test-seasonal_daily.R b/tests/testthat/test-seasonal_daily.R
index c37b5514..11c01f19 100644
--- a/tests/testthat/test-seasonal_daily.R
+++ b/tests/testthat/test-seasonal_daily.R
@@ -15,12 +15,12 @@ data <- load_datasets(recipe_file)
 recipe <- read_yaml(recipe_file)
 
 suppressWarnings({invisible(capture.output(
-calibrated_data <- calibrate_datasets(data, recipe)
+calibrated_data <- calibrate_datasets(recipe, data)
 ))})
 
 # Compute skill metrics
 suppressWarnings({invisible(capture.output(
-skill_metrics <- compute_skill_metrics(calibrated_data$hcst, data$obs, recipe)
+skill_metrics <- compute_skill_metrics(recipe, calibrated_data$hcst, data$obs)
 ))})
 
 test_that("1. Loading", {
diff --git a/tests/testthat/test-seasonal_monthly.R b/tests/testthat/test-seasonal_monthly.R
index 90938d62..9423cde9 100644
--- a/tests/testthat/test-seasonal_monthly.R
+++ b/tests/testthat/test-seasonal_monthly.R
@@ -16,29 +16,29 @@ data <- load_datasets(recipe_file)
 ))})
 
 suppressWarnings({invisible(capture.output(
-calibrated_data <- calibrate_datasets(data, recipe)
+calibrated_data <- calibrate_datasets(recipe, data)
 ))})
 
 # Compute skill metrics
 suppressWarnings({invisible(capture.output(
-skill_metrics <- compute_skill_metrics(calibrated_data$hcst, data$obs, recipe)
+skill_metrics <- compute_skill_metrics(recipe, calibrated_data$hcst, data$obs)
 ))})
 
 suppressWarnings({invisible(capture.output(
-probs <- compute_probabilities(calibrated_data$hcst, recipe)
+probs <- compute_probabilities(recipe, calibrated_data$hcst)
 ))})
 
 # Saving
 suppressWarnings({invisible(capture.output(
 save_data(recipe = recipe, data = data, calibrated_data = calibrated_data,
-	  skill_metrics = skill_metrics, probabilities = probs, archive = archive)
+	  skill_metrics = skill_metrics, probabilities = probs)
 ))})
 
 # Plotting
 suppressWarnings({invisible(capture.output(
-plot_data(recipe = recipe, archive = archive, data = data,
-	  calibrated_data = calibrated_data, skill_metrics = skill_metrics,
-	  probabilities = probs, significance = T)
+plot_data(recipe = recipe,  data = data, calibrated_data = calibrated_data,
+	  skill_metrics = skill_metrics, probabilities = probs,
+	  significance = T)
 ))})
 outdir <- get_dir(recipe)
 
-- 
GitLab


From 8de3ad947480896a0c08cbd36f4dce946c321aa8 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 18 Oct 2022 11:14:35 +0200
Subject: [PATCH 02/13] Refine longitude and latitude mismatch error messages

---
 modules/Loading/Loading.R | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R
index a93be8ca..39208d01 100644
--- a/modules/Loading/Loading.R
+++ b/modules/Loading/Loading.R
@@ -298,10 +298,31 @@ load_datasets <- function(recipe_file) {
   # Check for consistency between hcst and obs grid
   if (!(recipe$Analysis$Regrid$type == 'none')) {
     if (!identical(as.vector(hcst$lat), as.vector(obs$lat))) {
-      stop("hcst and obs don't share the same latitude.")
+      lat_error_msg <- paste("Latitude mismatch between hcst and obs.",
+			     "Please check the original grids and the",
+			     "regrid parameters in your recipe.")
+      error(logger, lat_error_msg)
+      hcst_lat_msg <- paste0("First hcst lat: ", hcst$lat[1],
+			     "; Last hcst lat: ", hcst$lat[length(hcst$lat)])
+      info(logger, hcst_lat_msg)
+      obs_lat_msg <- paste0("First obs lat: ", obs$lat[1],
+			     "; Last obs lat: ", obs$lat[length(obs$lat)])
+      info(logger, obs_lat_msg)
+      stop("hcst and obs don't share the same latitudes.")
     }
     if (!identical(as.vector(hcst$lon), as.vector(obs$lon))) {
-      stop("hcst and obs don't share the same longitude.")
+      lon_error_msg <- paste("Longitude mismatch between hcst and obs.",
+			     "Please check the original grids and the",
+			     "regrid parameters in your recipe.")
+      error(logger, lon_error_msg)
+      hcst_lon_msg <- paste0("First hcst lon: ", hcst$lon[1],
+			     "; Last hcst lon: ", hcst$lon[length(hcst$lon)])
+      info(logger, hcst_lon_msg)
+      obs_lon_msg <- paste0("First obs lon: ", obs$lon[1],
+			     "; Last obs lon: ", obs$lon[length(obs$lon)])
+      info(logger, obs_lon_msg)
+      stop("hcst and obs don't share the same longitudes.")
+
     }
   }
 
-- 
GitLab


From 8ca31a17a907a31aa91ae563f6550e9a329726e5 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 18 Oct 2022 15:23:16 +0200
Subject: [PATCH 03/13] Rearrange function arguments in seasonal sample script

---
 modules/test_seasonal.R | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/modules/test_seasonal.R b/modules/test_seasonal.R
index 5f59794f..eb7fbf41 100644
--- a/modules/test_seasonal.R
+++ b/modules/test_seasonal.R
@@ -12,13 +12,13 @@ archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 # Load datasets
 data <- load_datasets(recipe_file)
 # Calibrate datasets
-calibrated_data <- calibrate_datasets(data, recipe)
+calibrated_data <- calibrate_datasets(recipe, data)
 # Compute skill metrics
-skill_metrics <- compute_skill_metrics(calibrated_data$hcst, data$obs, recipe)
+skill_metrics <- compute_skill_metrics(recipe, calibrated_data$hcst, data$obs)
 # Compute percentiles and probability bins
-probabilities <- compute_probabilities(calibrated_data$hcst, recipe)
+probabilities <- compute_probabilities(recipe, calibrated_data$hcst)
 # Export all data to netCDF
-save_data(recipe, archive, data, calibrated_data, skill_metrics, probabilities)
+save_data(recipe, data, calibrated_data, skill_metrics, probabilities)
 # Plot data
-plot_data(recipe, archive, data, calibrated_data, skill_metrics,
-	  probabilities, significance = T)
+plot_data(recipe, data, calibrated_data, skill_metrics, probabilities,
+	  significance = T)
-- 
GitLab


From 6d4f80f2a158b94dd94179e46113aa9bef0506d4 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 18 Oct 2022 15:25:12 +0200
Subject: [PATCH 04/13] Rearrange function args in decadal sample script,
 comment archive line

---
 modules/test_decadal.R  | 14 +++++++-------
 modules/test_seasonal.R |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/modules/test_decadal.R b/modules/test_decadal.R
index 01cf2d92..686d91bb 100644
--- a/modules/test_decadal.R
+++ b/modules/test_decadal.R
@@ -7,24 +7,24 @@ source("modules/Visualization/Visualization.R")
 
 recipe_file <- "modules/Loading/testing_recipes/recipe_decadal.yml"
 recipe <- read_yaml(recipe_file)
-archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
+# archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
 
 # Load datasets
 data <- load_datasets(recipe_file)
 
 # Calibrate datasets
-calibrated_data <- calibrate_datasets(data, recipe)
+calibrated_data <- calibrate_datasets(recipe, data)
 
 # Compute skill metrics
-skill_metrics <- compute_skill_metrics(calibrated_data$hcst, data$obs, recipe)
+skill_metrics <- compute_skill_metrics(recipe, calibrated_data$hcst, data$obs)
 
 # Compute percentiles and probability bins
-probabilities <- compute_probabilities(calibrated_data$hcst, recipe)
+probabilities <- compute_probabilities(recipe, calibrated_data$hcst)
 
 # Export all data to netCDF
-save_data(recipe, archive, data, calibrated_data, skill_metrics, probabilities)
+save_data(recipe, data, calibrated_data, skill_metrics, probabilities)
 
 # Plot data
-plot_data(recipe, archive, data, calibrated_data, skill_metrics,
-	  probabilities, significance = T)
+plot_data(recipe, data, calibrated_data, skill_metrics, probabilities,
+	  significance = T)
 
diff --git a/modules/test_seasonal.R b/modules/test_seasonal.R
index eb7fbf41..436f8c9e 100644
--- a/modules/test_seasonal.R
+++ b/modules/test_seasonal.R
@@ -7,7 +7,7 @@ source("modules/Visualization/Visualization.R")
 
 recipe_file <- "modules/Loading/testing_recipes/recipe_system7c3s-tas.yml"
 recipe <- read_yaml(recipe_file)
-archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
+# archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 
 # Load datasets
 data <- load_datasets(recipe_file)
-- 
GitLab


From 27598a8a7785094d9633f54ad92244f8454604aa Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Fri, 21 Oct 2022 14:38:51 +0200
Subject: [PATCH 05/13] Incorporate logger

---
 modules/Calibration/Calibration.R             | 20 ++++----
 modules/Loading/Loading.R                     | 41 ++++++----------
 .../testing_recipes/recipe_test-logging.yml   | 47 +++++++++++++++++++
 modules/Saving/Saving.R                       | 29 +++++++-----
 modules/Skill/Skill.R                         | 15 ++++--
 modules/Visualization/Visualization.R         | 19 ++++----
 modules/test_seasonal.R                       |  7 ++-
 tools/data_summary.R                          | 24 +++++-----
 tools/prepare_outputs.R                       | 30 +++++++-----
 9 files changed, 145 insertions(+), 87 deletions(-)
 create mode 100644 modules/Loading/testing_recipes/recipe_test-logging.yml

diff --git a/modules/Calibration/Calibration.R b/modules/Calibration/Calibration.R
index d49dd9d9..3320a7f5 100644
--- a/modules/Calibration/Calibration.R
+++ b/modules/Calibration/Calibration.R
@@ -13,9 +13,9 @@ calibrate_datasets <- function(recipe, data) {
   method <- tolower(recipe$Analysis$Workflow$Calibration$method)
 
   if (method == "raw") {
-    warning("The Calibration module has been called, but the calibration ",
-	    "method in the recipe is 'raw'. The hcst and fcst will not be ",
-	    "calibrated.")
+    warn(recipe$Run$logger, "The Calibration module has been called, 
+	 but the calibration method in the recipe is 'raw'. 
+	 The hcst and fcst will not be calibrated.")
     fcst_calibrated <- data$fcst
     hcst_calibrated <- data$hcst
     CALIB_MSG <- "##### NO CALIBRATION PERFORMED #####"
@@ -53,8 +53,9 @@ calibrate_datasets <- function(recipe, data) {
       ## TODO: implement other calibration methods
       ## TODO: Restructure the code?
       if (!(method %in% CST_CALIB_METHODS)) {
-        stop("Calibration method in the recipe is not available for monthly",
-  	   " data.")
+	error(recipe$Run$logger, "Calibration method in the recipe is not
+	      available for monthly data.")
+        stop()
       } else {
         ## Alba's version of CST_Calibration (pending merge) is being used
         # Calibrate the hindcast
@@ -89,8 +90,10 @@ calibrate_datasets <- function(recipe, data) {
     } else if (recipe$Analysis$Variables$freq == "daily_mean") {
       # Daily data calibration using Quantile Mapping
       if (!(method %in% c("qmap"))) {
-        stop("Calibration method in the recipe is not available at daily ",
-  	   "frequency. Only quantile mapping 'qmap' is implemented.")
+	error(recipe$Run$logger, "Calibration method in the recipe is not
+	      available for daily data. Only quantile mapping 'qmap is
+	      implemented.")
+        stop()
       }
       # Calibrate the hindcast
       hcst_calibrated <- CST_QuantileMapping(data$hcst, data$obs,
@@ -121,7 +124,6 @@ calibrate_datasets <- function(recipe, data) {
       }
     }
   }
-print(CALIB_MSG)
-  ## TODO: Return observations too?
+  info(recipe$Run$logger, CALIB_MSG)
   return(list(hcst = hcst_calibrated, fcst = fcst_calibrated))
 }
diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R
index a93be8ca..2493af35 100644
--- a/modules/Loading/Loading.R
+++ b/modules/Loading/Loading.R
@@ -5,24 +5,8 @@ source("modules/Loading/dates2load.R")
 source("modules/Loading/check_latlon.R")
 source("tools/libs.R")
 
-# RECIPE FOR TESTING
-# --------------------------------------------------------------------------------
-# recipe_file <- "modules/Loading/testing_recipes/recipe_3.yml"
-# recipe_file <- "modules/Loading/testing_recipes/recipe_2.yml"
-# recipe_file <- "modules/Loading/testing_recipes/recipe_1.yml"
 
-load_datasets <- function(recipe_file) {
-
-  recipe <- read_yaml(recipe_file)
-  recipe$filepath <- recipe_file
-  recipe$name <- tools::file_path_sans_ext(basename(recipe_file))
-
-  ## TODO: this should come from the main script 
-  # Create output folder and log:
-  logger <- prepare_outputs(recipe = recipe) 
-  folder <- logger$foldername
-  log_file <- logger$logname
-  logger <- logger$logger
+load_datasets <- function(recipe) {
 
   # -------------------------------------------
   # Set params -----------------------------------------
@@ -40,7 +24,8 @@ load_datasets <- function(recipe_file) {
   store.freq <- recipe$Analysis$Variables$freq
 
   # get sdates array
-  sdates <- dates2load(recipe, logger)
+  ## LOGGER: Change dates2load to extract logger from recipe?
+  sdates <- dates2load(recipe, recipe$Run$logger)
 
   idxs <- NULL
   idxs$hcst <- get_timeidx(sdates$hcst,
@@ -308,7 +293,8 @@ load_datasets <- function(recipe_file) {
   # Remove negative values in accumulative variables
   dictionary <- read_yaml("conf/variable-dictionary.yml")
   if (dictionary$vars[[variable]]$accum) {
-    info(logger, "Accumulated variable: setting negative values to zero.")
+    info(recipe$Run$logger,
+	 "Accumulated variable: setting negative values to zero.")
     obs$data[obs$data < 0] <- 0
     hcst$data[hcst$data < 0] <- 0
     if (!is.null(fcst)) {
@@ -324,7 +310,7 @@ load_datasets <- function(recipe_file) {
 	 attr(hcst$Variable, "variable")$units) && 
 	(attr(obs$Variable, "variable")$units == "m s-1")) {
       
-      info(logger, "Converting precipitation from m/s to mm/day.")
+      info(recipe$Run$logger, "Converting precipitation from m/s to mm/day.")
       obs$data <- obs$data*84000*1000
       attr(obs$Variable, "variable")$units <- "mm/day"
       hcst$data <- hcst$data*84000*1000
@@ -337,13 +323,14 @@ load_datasets <- function(recipe_file) {
   }
 
   # Print a summary of the loaded data for the user, for each object
-  data_summary(hcst, store.freq)
-  data_summary(obs, store.freq)
+  data_summary(hcst, recipe)
+  data_summary(obs, recipe)
   if (!is.null(fcst)) {
-    data_summary(fcst, store.freq)
+    data_summary(fcst, recipe)
   }
 
-  print("##### DATA LOADING COMPLETED SUCCESSFULLY #####")
+  info(recipe$Run$logger,
+       "##### DATA LOADING COMPLETED SUCCESSFULLY #####")
   
   ############################################################################
   #
@@ -360,7 +347,7 @@ load_datasets <- function(recipe_file) {
   #                       freq.obs,"obs.grid","/",variable,"_",obs.NA_dates,".nc")
   #
   #if (any(is.na(hcst))){
-  #  fatal(logger,
+  #  fatal(recipe$Run$logger,
   #    paste(" ERROR: MISSING HCST VALUES FOUND DURING LOADING # ",
   #          " ################################################# ",
   #          " ######           MISSING FILES               #### ",
@@ -374,7 +361,7 @@ load_datasets <- function(recipe_file) {
   #}
   #
   #if (any(is.na(obs)) && !identical(obs.NA_dates,character(0))){
-  #  fatal(logger,
+  #  fatal(recipe$logger,
   #    paste(" ERROR: MISSING OBS VALUES FOUND DURING  LOADING # ",
   #          " ################################################# ",
   #          " ######           MISSING FILES               #### ",
@@ -387,7 +374,7 @@ load_datasets <- function(recipe_file) {
   #  quit(status=1)
   #}
   #
-  #info(logger,
+  #info(recipe$logger,
   #     "######### DATA LOADING COMPLETED SUCCESFULLY ##############")
   
   ############################################################################
diff --git a/modules/Loading/testing_recipes/recipe_test-logging.yml b/modules/Loading/testing_recipes/recipe_test-logging.yml
new file mode 100644
index 00000000..372f6d83
--- /dev/null
+++ b/modules/Loading/testing_recipes/recipe_test-logging.yml
@@ -0,0 +1,47 @@
+Description:
+  Author: V. Agudetse
+  Info: Light recipe to raise some errors/warnings and test the logging system
+
+Analysis:
+  Horizon: Seasonal
+  Variables:
+    name: tas
+    freq: monthly_mean
+  Datasets:
+    System:
+      name: system7c3s
+    Multimodel: False
+    Reference:
+      name: era5
+  Time:
+    sdate: '1101'
+    fcst_year: '2020'
+    hcst_start: '1993'
+    hcst_end: '1996'
+    ftime_min: 1
+    ftime_max: 1
+  Region:
+    latmin: -10
+    latmax: 10
+    lonmin: 0
+    lonmax: 20
+  Regrid:
+    method: bilinear
+    type: to_system
+  Workflow:
+    Calibration:
+      method: qmap
+    Skill:
+      metric: mean_bias bias_SS
+    Probabilities:
+      percentiles:
+    Indicators:
+      index: no
+  ncores: 1
+  remove_NAs: yes
+  Output_format: S2S4E
+Run:
+  Loglevel: INFO
+  Terminal: yes
+  output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/
+  code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/
diff --git a/modules/Saving/Saving.R b/modules/Saving/Saving.R
index cf2ec1d0..ed0933f2 100644
--- a/modules/Saving/Saving.R
+++ b/modules/Saving/Saving.R
@@ -215,7 +215,8 @@ save_forecast <- function(data_cube,
   leadtimes <- as.numeric(dates - init_date)/3600
 
   syears <- seq(1:dim(data_cube$data)['syear'][[1]])
-  syears_val <- lubridate::year(data_cube$Dates$start[1, 1, , 1]) # expect dim = [sday = 1, sweek = 1, syear, time]
+  # expect dim = [sday = 1, sweek = 1, syear, time]
+  syears_val <- lubridate::year(data_cube$Dates$start[1, 1, , 1])
   for (i in syears) {
     # Select year from array and rearrange dimensions
     fcst <- ClimProjDiags::Subset(data_cube$data, 'syear', i, drop = T)
@@ -254,8 +255,9 @@ save_forecast <- function(data_cube,
     
     # Select start date
     if (fcst.horizon == 'decadal') {
-      #NOTE: Not good to use data_cube$load_parameters$dat1 because decadal data has been reshaped
-#      fcst.sdate <- format(as.Date(data_cube$Dates$start[i]), '%Y%m%d')
+      ## NOTE: Not good to use data_cube$load_parameters$dat1 because decadal
+      ## data has been reshaped
+      # fcst.sdate <- format(as.Date(data_cube$Dates$start[i]), '%Y%m%d')
 
       # init_date is like "1990-11-01"
       init_date <- as.POSIXct(init_date)
@@ -289,7 +291,7 @@ save_forecast <- function(data_cube,
       ArrayToNc(vars, outfile)
     }
   }
-  print("##### FCST SAVED TO NETCDF FILE #####")
+  info(recipe$Run$logger, "##### FCST SAVED TO NETCDF FILE #####")
 }
 
 
@@ -337,7 +339,8 @@ save_observations <- function(data_cube,
   leadtimes <- as.numeric(dates - init_date)/3600
 
   syears <- seq(1:dim(data_cube$data)['syear'][[1]])
-  syears_val <- lubridate::year(data_cube$Dates$start[1, 1, , 1]) # expect dim = [sday = 1, sweek = 1, syear, time]
+  ## expect dim = [sday = 1, sweek = 1, syear, time]
+  syears_val <- lubridate::year(data_cube$Dates$start[1, 1, , 1])
   for (i in syears) {
     # Select year from array and rearrange dimensions
     fcst <- ClimProjDiags::Subset(data_cube$data, 'syear', i, drop = T)
@@ -424,11 +427,11 @@ save_observations <- function(data_cube,
       ArrayToNc(vars, outfile)
     }
   }
-  print("##### OBS SAVED TO NETCDF FILE #####")
+  info(recipe$Run$logger, "##### OBS SAVED TO NETCDF FILE #####")
 }
 
 ## TODO: Place inside a function somewhere
-# if (tolower(agg) == "country"){
+# if (tolower(agg) == "country") {
 #   load(mask.path)
 #   grid <- europe.countries.iso
 # } else {
@@ -541,7 +544,7 @@ save_metrics <- function(skill,
     vars <- c(vars, skill)
     ArrayToNc(vars, outfile)
   }
-  print("##### SKILL METRICS SAVED TO NETCDF FILE #####")
+  info(recipe$Run$logger, "##### SKILL METRICS SAVED TO NETCDF FILE #####")
 }
 
 save_corr <- function(skill,
@@ -648,7 +651,8 @@ save_corr <- function(skill,
     vars <- c(vars, skill)
     ArrayToNc(vars, outfile)
   }
-  print("##### ENSEMBLE CORRELATION SAVED TO NETCDF FILE #####")
+  info(recipe$Run$logger,
+       "##### ENSEMBLE CORRELATION SAVED TO NETCDF FILE #####")
 }
 
 save_percentiles <- function(percentiles,
@@ -747,7 +751,7 @@ save_percentiles <- function(percentiles,
     vars <- c(vars, percentiles)
     ArrayToNc(vars, outfile)
   }
-  print("##### PERCENTILES SAVED TO NETCDF FILE #####")
+  info(recipe$Run$logger, "##### PERCENTILES SAVED TO NETCDF FILE #####")
 }
 
 save_probabilities <- function(probs,
@@ -793,7 +797,8 @@ save_probabilities <- function(probs,
   leadtimes <- as.numeric(dates - init_date)/3600
 
   syears <- seq(1:dim(data_cube$data)['syear'][[1]])
-  syears_val <- lubridate::year(data_cube$Dates$start[1, 1, , 1]) # expect dim = [sday = 1, sweek = 1, syear, time]
+  ## expect dim = [sday = 1, sweek = 1, syear, time]
+  syears_val <- lubridate::year(data_cube$Dates$start[1, 1, , 1])
   for (i in syears) {
     # Select year from array and rearrange dimensions
     probs_syear <- lapply(probs, ClimProjDiags::Subset, 'syear', i, drop = 'selected')
@@ -855,5 +860,5 @@ save_probabilities <- function(probs,
       ArrayToNc(vars, outfile)
     }
   }
-  print("##### PROBABILITIES SAVED TO NETCDF FILE #####")
+  info(recipe$Run$logger, "##### PROBABILITIES SAVED TO NETCDF FILE #####")
 }
diff --git a/modules/Skill/Skill.R b/modules/Skill/Skill.R
index 726cd605..11365ba8 100644
--- a/modules/Skill/Skill.R
+++ b/modules/Skill/Skill.R
@@ -186,7 +186,9 @@ compute_skill_metrics <- function(recipe, exp, obs) {
       metric_name <- (strsplit(metric, "_"))[[1]][1] # Get metric name
       if (!(metric_name %in% c('frpss', 'frps', 'bss10', 'bss90', 'enscorr',
 			       'rpss'))) {
-	stop("Some of the requested metrics are not available.")
+	## TODO: Test this scenario
+	warn(recipe$Run$logger,
+	     "Some of the requested metrics are not available.")
       }
       capture.output(
       skill <- Compute_verif_metrics(exp$data, obs$data,
@@ -203,7 +205,7 @@ compute_skill_metrics <- function(recipe, exp, obs) {
       skill_metrics[[ metric ]] <- skill
     }
   }
-  print("##### SKILL METRIC COMPUTATION COMPLETE #####")
+  info(recipe$Run$logger, "##### SKILL METRIC COMPUTATION COMPLETE #####")
   return(skill_metrics)
 }
 
@@ -224,8 +226,9 @@ compute_probabilities <- function(recipe, data) {
   named_probs <- list()
   named_quantiles <- list()
   if (is.null(recipe$Analysis$Workflow$Probabilities$percentiles)) {
-    stop("Quantiles and probability bins have been requested, but no ",
-	 "thresholds are provided in the recipe.")
+    error(recipe$Run$logger, "Quantiles and probability bins have been 
+	  requested, but no thresholds are provided in the recipe.")
+    stop()
   } else {
     for (element in recipe$Analysis$Workflow$Probabilities$percentiles) {
       # Parse thresholds in recipe
@@ -261,10 +264,12 @@ compute_probabilities <- function(recipe, data) {
     named_probs <- lapply(named_probs, function(x) {.drop_dims(x)})
     named_quantiles <- lapply(named_quantiles, function(x) {.drop_dims(x)})
   }
-  print("##### PERCENTILES AND PROBABILITY CATEGORIES COMPUTED #####")
+  info(recipe$Run$logger, 
+       "##### PERCENTILES AND PROBABILITY CATEGORIES COMPUTED #####")
   return(list(probs=named_probs, percentiles=named_quantiles))
 }
 
+## TODO: Replace with ClimProjDiags::Subset
 .drop_dims <- function(metric_array) {
   # Drop all singleton dimensions
   metric_array <- drop(metric_array)
diff --git a/modules/Visualization/Visualization.R b/modules/Visualization/Visualization.R
index 4b63de8c..28879742 100644
--- a/modules/Visualization/Visualization.R
+++ b/modules/Visualization/Visualization.R
@@ -53,8 +53,8 @@ plot_data <- function(recipe,
   if (!is.null(calibrated_data$fcst)) {
     plot_ensemble_mean(recipe, archive, calibrated_data$fcst, outdir)
   } else if (!is.null(data$fcst)) {
-    warning("Only the uncalibrated forecast was provided. Using this data ",
-	    "to plot the forecast ensemble mean.")
+    warn(recipe$Run$logger, "Only the uncalibrated forecast was provided. 
+	 Using this data to plot the forecast ensemble mean.")
     plot_ensemble_mean(recipe, archive, data$fcst, outdir)
   }
 
@@ -63,8 +63,8 @@ plot_data <- function(recipe,
     plot_most_likely_terciles(recipe, archive, calibrated_data$fcst,
 			      probabilities$percentiles, outdir)
   } else if ((!is.null(probabilities)) && (!is.null(data$fcst))) {
-    warning("Only the uncalibrated forecast was provided. Using this data ",
-	    "to plot the most likely terciles.")
+    warn(recipe$Run$logger, "Only the uncalibrated forecast was provided.
+	 Using this data to plot the most likely terciles.")
     plot_most_likely_terciles(recipe, archive, data$fcst,
 			      probabilities$percentiles, outdir)
   }
@@ -78,7 +78,7 @@ plot_skill_metrics <- function(recipe, archive, data_cube, skill_metrics,
     stop("Visualization functions not yet implemented for daily data.")
   }
   # Abort if skill_metrics is not list
-  if (!is.list(skill_metrics)) {
+  if (!is.list(skill_metrics) || is.null(names(skill_metrics))) {
     stop("The element 'skill_metrics' must be a list of named arrays.")
   }
   
@@ -190,7 +190,8 @@ plot_skill_metrics <- function(recipe, archive, data_cube, skill_metrics,
     }    
   }
 
-  print("##### SKILL METRIC PLOTS SAVED TO OUTPUT DIRECTORY #####")
+  info(recipe$Run$logger,
+       "##### SKILL METRIC PLOTS SAVED TO OUTPUT DIRECTORY #####")
 }
 
 plot_ensemble_mean <- function(recipe, archive, fcst, outdir) {
@@ -263,7 +264,8 @@ plot_ensemble_mean <- function(recipe, archive, fcst, outdir) {
                bar_label_digits = 4)
   }
 
-  print("##### FCST ENSEMBLE MEAN PLOT SAVED TO OUTPUT DIRECTORY #####")
+  info(recipe$Run$logger,
+       "##### FCST ENSEMBLE MEAN PLOT SAVED TO OUTPUT DIRECTORY #####")
 }
 
 plot_most_likely_terciles <- function(recipe, archive,
@@ -347,5 +349,6 @@ plot_most_likely_terciles <- function(recipe, archive,
     )
   }
 
-  print("##### MOST LIKELY TERCILE PLOT SAVED TO OUTPUT DIRECTORY #####")
+  info(recipe$Run$logger,
+       "##### MOST LIKELY TERCILE PLOT SAVED TO OUTPUT DIRECTORY #####")
 }
diff --git a/modules/test_seasonal.R b/modules/test_seasonal.R
index 436f8c9e..4071b648 100644
--- a/modules/test_seasonal.R
+++ b/modules/test_seasonal.R
@@ -1,16 +1,15 @@
-
 source("modules/Loading/Loading.R")
 source("modules/Calibration/Calibration.R")
 source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
-recipe_file <- "modules/Loading/testing_recipes/recipe_system7c3s-tas.yml"
-recipe <- read_yaml(recipe_file)
+recipe_file <- "modules/Loading/testing_recipes/recipe_test-logging.yml"
+recipe <- prepare_outputs(recipe_file)
 # archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 
 # Load datasets
-data <- load_datasets(recipe_file)
+data <- load_datasets(recipe)
 # Calibrate datasets
 calibrated_data <- calibrate_datasets(recipe, data)
 # Compute skill metrics
diff --git a/tools/data_summary.R b/tools/data_summary.R
index e211e202..34b6bd6e 100644
--- a/tools/data_summary.R
+++ b/tools/data_summary.R
@@ -4,27 +4,29 @@
 ## TODO: Adapt to daily/subseasonal cases
 ## TODO: Add check for missing files/NAs by dimension
 
-data_summary <- function(object, frequency) {
+data_summary <- function(data_cube, recipe) {
   # Get name, leadtime months and date range
-  object_name <- deparse(substitute(object))
-  if (tolower(frequency) == "monthly_mean") {
+  object_name <- deparse(substitute(data_cube))
+  if (recipe$Analysis$Variables$freq == "monthly_mean") {
     date_format <- '%b %Y'
-  } else if (tolower(frequency) == "daily_mean") {
+  } else if (recipe$Analysis$Variables$freq == "daily_mean") {
     date_format <- '%b %d %Y'
   }
-  months <- unique(format(as.Date(object$Dates[[1]]), format = '%B'))
+  months <- unique(format(as.Date(data_cube$Dates[[1]]), format = '%B'))
   months <- paste(as.character(months), collapse=", ")
-  sdate_min <- format(min(as.Date(object$Dates[[1]])), format = date_format)
-  sdate_max <- format(max(as.Date(object$Dates[[1]])), format = date_format)
+  sdate_min <- format(min(as.Date(data_cube$Dates[[1]])), format = date_format)
+  sdate_max <- format(max(as.Date(data_cube$Dates[[1]])), format = date_format)
   
-  print("DATA SUMMARY:")
+  # Create log instance and sink output to logfile and terminal
+  info(recipe$Run$logger, "DATA SUMMARY:")
+  sink(recipe$Run$logfile, append = TRUE, split = TRUE)
   print(paste0(object_name, " months: ", months))
   print(paste0(object_name, " range: ", sdate_min, " to ", sdate_max))
   print(paste0(object_name, " dimensions: "))
-  print(dim(object$data))
+  print(dim(data_cube$data))
   print(paste0("Statistical summary of the data in ", object_name, ":"))
-  print(summary(object$data))
+  print(summary(data_cube$data))
   print("---------------------------------------------")
-
+  sink()
 }
   
diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R
index 18cc2e58..9c557046 100644
--- a/tools/prepare_outputs.R
+++ b/tools/prepare_outputs.R
@@ -5,28 +5,30 @@
 #'the recipe. It returns an object of class logger that stores information on 
 #'the recipe configuration and errors.
 #'
-#'@param recipe Auto-S2S configuration recipe as returned by read_yaml()
+#'@param recipe_file path to a YAML file with Auto-S2S configuration recipe
 #'
-#'@return list contaning logger object, log filename and log directory name
+#'@return list contaning recipe with logger, log file name and log dir name
 #'
 #'@import log4r
+#'@import yaml
 #'
 #'@examples
 #'setwd("/esarchive/scratch/vagudets/repos/auto-s2s/")
 #'library(yaml)
-#'recipe <- read_yaml("modules/data_load/recipe_1.yml")
-#'logger <- prepare_outputs(recipe)
-#'folder <- logger$foldername
-#'log_file <- logger$logname
-#'logger <- logger$logger
+#'recipe <- prepare_outputs("modules/data_load/recipe_1.yml")
+#'info(recipe$Run$logger, "This is an info message")
 #'
 #'@export
 
-prepare_outputs <- function(recipe) {
+prepare_outputs <- function(recipe_file) {
 
 # recipe: the content of the readed recipe
 # file: the recipe file name
 
+  recipe <- read_yaml(recipe_file)
+  recipe$recipe_path <- recipe_file
+  recipe$name <- tools::file_path_sans_ext(basename(recipe_file))
+  
   output_dir = recipe$Run$output_dir
   # Create output folders:
   folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name)), "_",
@@ -43,6 +45,7 @@ prepare_outputs <- function(recipe) {
   file.copy(recipe$filepath, file.path(output_dir, folder_name, 'logs'))
 
   logfile <- file.path(output_dir, folder_name, 'logs', 'log.txt')
+  file.create(logfile)
 
   # Set default behaviour of log output file:
   if (is.null(recipe$Run)) {
@@ -51,6 +54,7 @@ prepare_outputs <- function(recipe) {
   if (is.null(recipe$Run$Loglevel)) { 
     recipe$Run$Loglevel <- 'INFO'
   }
+
   if (!is.logical(recipe$Run$Terminal)) {
     recipe$Run$Terminal <- TRUE
   }
@@ -61,9 +65,13 @@ prepare_outputs <- function(recipe) {
                                layout = default_log_layout())))
   } else {
     logger <- logger(threshold = recipe$Run$Loglevel,
-                appenders = list(file_appender(logfile, append = TRUE,             
+                appenders = list(file_appende(logfile, append = TRUE,             
                                layout = default_log_layout())))
   }
-  return(list(logger = logger, logname = logfile,
-              foldername = file.path(output_dir, folder_name)))
+
+  recipe$Run$output_dir <- file.path(output_dir, folder_name)
+  recipe$Run$logger <- logger
+  recipe$Run$logfile <- logfile
+
+  return(recipe)
 }
-- 
GitLab


From 9f25dac89166c146d8039de6c3f9c27e4175c765 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Mon, 24 Oct 2022 14:59:04 +0200
Subject: [PATCH 06/13] Modify decadal scripts, fix pipeline attempt

---
 modules/Loading/Loading_decadal.R       | 23 +++++++++--------------
 modules/test_decadal.R                  |  4 ++--
 modules/test_seasonal.R                 |  2 +-
 tests/testthat/test-decadal_daily_1.R   |  4 ++--
 tests/testthat/test-decadal_monthly_1.R |  4 ++--
 tests/testthat/test-decadal_monthly_2.R |  5 ++---
 tests/testthat/test-decadal_monthly_3.R |  4 ++--
 tests/testthat/test-seasonal_daily.R    |  7 +++----
 tests/testthat/test-seasonal_monthly.R  |  5 +++--
 tools/prepare_outputs.R                 |  9 +++++----
 10 files changed, 31 insertions(+), 36 deletions(-)

diff --git a/modules/Loading/Loading_decadal.R b/modules/Loading/Loading_decadal.R
index 9c4bb33d..2f6d0310 100644
--- a/modules/Loading/Loading_decadal.R
+++ b/modules/Loading/Loading_decadal.R
@@ -13,18 +13,14 @@ source("modules/Loading/check_latlon.R")
 source("tools/libs.R")
 ## TODO: Remove once the fun is included in CSTools
 source("tools/tmp/as.s2dv_cube.R")
-
+## TODO: Change stops to logger error messages
 
 #====================================================================
 
 # recipe_file <- "modules/Loading/testing_recipes/recipe_decadal.yml"
 # recipe_file <- "modules/Loading/testing_recipes/recipe_decadal_daily.yml"
 
-load_datasets <- function(recipe_file) {
-
-  recipe <- read_yaml(recipe_file)
-  recipe$filepath <- recipe_file
-  recipe$name <- tools::file_path_sans_ext(basename(recipe_file))
+load_datasets <- function(recipe) {
 
   archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
 
@@ -33,11 +29,7 @@ load_datasets <- function(recipe_file) {
 
   ## TODO: this should come from the main script 
   # Create output folder and log:
-  logger <- prepare_outputs(recipe = recipe) 
-  folder <- logger$foldername
-  log_file <- logger$logname
-  logger <- logger$logger
-
+  
   #-------------------------
   # Read from recipe:
   #-------------------------
@@ -413,7 +405,8 @@ load_datasets <- function(recipe_file) {
   # Remove negative values in accumulative variables
   dictionary <- read_yaml("conf/variable-dictionary.yml")
   if (dictionary$vars[[variable]]$accum) {
-    info(logger, " Accumulated variable: setting negative values to zero.")
+    info(recipe$Run$logger, 
+	 " Accumulated variable: setting negative values to zero.")
     obs$data[obs$data < 0] <- 0
     hcst$data[hcst$data < 0] <- 0
     if (!is.null(fcst)) {
@@ -428,7 +421,8 @@ load_datasets <- function(recipe_file) {
 	 attr(hcst$Variable, "variable")$units) &&
 	(attr(obs$Variable, "variable")$units == "m s-1")) {
       
-      info(logger, "Converting precipitation from m/s to mm/day.")
+      info(recipe$Run$logger,
+	   "Converting precipitation from m/s to mm/day.")
       obs$data <- obs$data*84000*1000
       attr(obs$Variable, "variable")$units <- "mm/day"
       hcst$data <- hcst$data*84000*1000
@@ -451,7 +445,8 @@ load_datasets <- function(recipe_file) {
     data_summary(fcst, store.freq)
   }
 
-  print("##### DATA LOADING COMPLETED SUCCESSFULLY #####")
+  info(recipe$Run$logger,
+       "##### DATA LOADING COMPLETED SUCCESSFULLY #####")
 
 
   return(list(hcst = hcst, fcst = fcst, obs = obs))
diff --git a/modules/test_decadal.R b/modules/test_decadal.R
index 686d91bb..80304f97 100644
--- a/modules/test_decadal.R
+++ b/modules/test_decadal.R
@@ -6,11 +6,11 @@ source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
 recipe_file <- "modules/Loading/testing_recipes/recipe_decadal.yml"
-recipe <- read_yaml(recipe_file)
+recipe <- prepare_outputs(recipe_file)
 # archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
 
 # Load datasets
-data <- load_datasets(recipe_file)
+data <- load_datasets(recipe)
 
 # Calibrate datasets
 calibrated_data <- calibrate_datasets(recipe, data)
diff --git a/modules/test_seasonal.R b/modules/test_seasonal.R
index 4071b648..d8eb5c4e 100644
--- a/modules/test_seasonal.R
+++ b/modules/test_seasonal.R
@@ -4,7 +4,7 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
-recipe_file <- "modules/Loading/testing_recipes/recipe_test-logging.yml"
+recipe_file <- "modules/Loading/testing_recipes/recipe_system7c3s-tas.yml"
 recipe <- prepare_outputs(recipe_file)
 # archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 
diff --git a/tests/testthat/test-decadal_daily_1.R b/tests/testthat/test-decadal_daily_1.R
index a78fd135..c9833d2b 100644
--- a/tests/testthat/test-decadal_daily_1.R
+++ b/tests/testthat/test-decadal_daily_1.R
@@ -8,12 +8,12 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 
 recipe_file <- "tests/recipes/recipe-decadal_daily_1.yml"
-recipe <- read_yaml(recipe_file)
+recipe <- prepare_outputs(recipe_file)
 archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
 
 # Load datasets
 suppressWarnings({invisible(capture.output(
-data <- load_datasets(recipe_file)
+data <- load_datasets(recipe)
 ))})
 
 ## Calibrate datasets
diff --git a/tests/testthat/test-decadal_monthly_1.R b/tests/testthat/test-decadal_monthly_1.R
index 39c8d900..5cf1922e 100644
--- a/tests/testthat/test-decadal_monthly_1.R
+++ b/tests/testthat/test-decadal_monthly_1.R
@@ -9,12 +9,12 @@ source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
 recipe_file <- "tests/recipes/recipe-decadal_monthly_1.yml"
-recipe <- read_yaml(recipe_file)
+recipe <- prepare_outputs(recipe_file)
 archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
 
 # Load datasets
 suppressWarnings({invisible(capture.output(
-data <- load_datasets(recipe_file)
+data <- load_datasets(recipe)
 ))})
 
 # Calibrate datasets
diff --git a/tests/testthat/test-decadal_monthly_2.R b/tests/testthat/test-decadal_monthly_2.R
index 98fa66cb..4dd72ebf 100644
--- a/tests/testthat/test-decadal_monthly_2.R
+++ b/tests/testthat/test-decadal_monthly_2.R
@@ -8,12 +8,11 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 
 recipe_file <- "tests/recipes/recipe-decadal_monthly_2.yml"
-recipe <- read_yaml(recipe_file)
-archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
+recipe <- prepare_outputs(recipe_file)
 
 # Load datasets
 suppressWarnings({invisible(capture.output(
-data <- load_datasets(recipe_file)
+data <- load_datasets(recipe)
 ))})
 
 # Calibrate datasets
diff --git a/tests/testthat/test-decadal_monthly_3.R b/tests/testthat/test-decadal_monthly_3.R
index 22f47d4c..2a5f7ef9 100644
--- a/tests/testthat/test-decadal_monthly_3.R
+++ b/tests/testthat/test-decadal_monthly_3.R
@@ -8,12 +8,12 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 
 recipe_file <- "tests/recipes/recipe-decadal_monthly_3.yml"
-recipe <- read_yaml(recipe_file)
+recipe <- prepate_outputs(recipe_file)
 archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
 
 # Load datasets
 suppressWarnings({invisible(capture.output(
-data <- load_datasets(recipe_file)
+data <- load_datasets(recipe)
 ))})
 
 # Calibrate datasets
diff --git a/tests/testthat/test-seasonal_daily.R b/tests/testthat/test-seasonal_daily.R
index 11c01f19..4fb4a71e 100644
--- a/tests/testthat/test-seasonal_daily.R
+++ b/tests/testthat/test-seasonal_daily.R
@@ -6,14 +6,13 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 
 recipe_file <- "tests/recipes/recipe-seasonal_daily_1.yml"
-
+recipe <- prepare_outputs(recipe_file)
 # Load datasets
 suppressWarnings({invisible(capture.output(
-data <- load_datasets(recipe_file)
+data <- load_datasets(recipe)
 ))})
 
-recipe <- read_yaml(recipe_file)
-
+# Calibrate data
 suppressWarnings({invisible(capture.output(
 calibrated_data <- calibrate_datasets(recipe, data)
 ))})
diff --git a/tests/testthat/test-seasonal_monthly.R b/tests/testthat/test-seasonal_monthly.R
index 9423cde9..86feedfb 100644
--- a/tests/testthat/test-seasonal_monthly.R
+++ b/tests/testthat/test-seasonal_monthly.R
@@ -7,14 +7,15 @@ source("modules/Saving/Saving.R")
 source("modules/Visualization/Visualization.R")
 
 recipe_file <- "tests/recipes/recipe-seasonal_monthly_1.yml"
-recipe <- read_yaml(recipe_file)
+recipe <- prepare_outputs(recipe_file)
 archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive
 
 # Load datasets
 suppressWarnings({invisible(capture.output(
-data <- load_datasets(recipe_file)
+data <- load_datasets(recipe)
 ))})
 
+# Calibrate data
 suppressWarnings({invisible(capture.output(
 calibrated_data <- calibrate_datasets(recipe, data)
 ))})
diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R
index 9c557046..8a683178 100644
--- a/tools/prepare_outputs.R
+++ b/tools/prepare_outputs.R
@@ -34,15 +34,16 @@ prepare_outputs <- function(recipe_file) {
   folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$name)), "_",
                         gsub(" ", "", gsub(":", "", gsub("-", "", Sys.time()))))
 
+  print("Saving all outputs to:")
   print(output_dir)
   print(folder_name)
 
-  dir.create(file.path(output_dir, folder_name, 'plots'), recursive = TRUE)
-  dir.create(file.path(output_dir, folder_name, 'outputs'))
+  dir.create(file.path(output_dir, folder_name, 'outputs'), recursive = TRUE)
   dir.create(file.path(output_dir, folder_name, 'logs'))
   dir.create(file.path(output_dir, folder_name, 'logs', 'recipes'))
 
-  file.copy(recipe$filepath, file.path(output_dir, folder_name, 'logs'))
+  file.copy(recipe$recipe_path, file.path(output_dir, folder_name, 'logs',
+				          'recipes'))
 
   logfile <- file.path(output_dir, folder_name, 'logs', 'log.txt')
   file.create(logfile)
@@ -51,7 +52,7 @@ prepare_outputs <- function(recipe_file) {
   if (is.null(recipe$Run)) {
     recipe$Run <- list(Loglevel = 'INFO', Terminal = TRUE)
   }
-  if (is.null(recipe$Run$Loglevel)) { 
+  if (is.null(recipe$Run$Loglevel)) {
     recipe$Run$Loglevel <- 'INFO'
   }
 
-- 
GitLab


From f78b9a769c5afb3de0b82a70bf68830656f1ccee Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 25 Oct 2022 09:39:43 +0200
Subject: [PATCH 07/13] fix decadal pipeline

---
 modules/Loading/Loading_decadal.R       | 6 +++---
 tests/testthat/test-decadal_monthly_3.R | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/Loading/Loading_decadal.R b/modules/Loading/Loading_decadal.R
index 2f6d0310..7b6b352f 100644
--- a/modules/Loading/Loading_decadal.R
+++ b/modules/Loading/Loading_decadal.R
@@ -439,10 +439,10 @@ load_datasets <- function(recipe) {
 #-------------------------------------------
 
   # Print a summary of the loaded data for the user, for each object
-  data_summary(hcst, store.freq)
-  data_summary(obs, store.freq)
+  data_summary(hcst, recipe)
+  data_summary(obs, recipe)
   if (!is.null(fcst)) {
-    data_summary(fcst, store.freq)
+    data_summary(fcst, recipe)
   }
 
   info(recipe$Run$logger,
diff --git a/tests/testthat/test-decadal_monthly_3.R b/tests/testthat/test-decadal_monthly_3.R
index 2a5f7ef9..7535e8dc 100644
--- a/tests/testthat/test-decadal_monthly_3.R
+++ b/tests/testthat/test-decadal_monthly_3.R
@@ -8,7 +8,7 @@ source("modules/Skill/Skill.R")
 source("modules/Saving/Saving.R")
 
 recipe_file <- "tests/recipes/recipe-decadal_monthly_3.yml"
-recipe <- prepate_outputs(recipe_file)
+recipe <- prepare_outputs(recipe_file)
 archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive
 
 # Load datasets
-- 
GitLab


From 1901998f50636690cb452dec9b904acd188f3f77 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 25 Oct 2022 09:46:24 +0200
Subject: [PATCH 08/13] Save outputs inside generated directory

---
 modules/Saving/paths2save.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/Saving/paths2save.R b/modules/Saving/paths2save.R
index 70f6cc92..f48ebe7b 100644
--- a/modules/Saving/paths2save.R
+++ b/modules/Saving/paths2save.R
@@ -39,7 +39,7 @@ get_dir <- function(recipe, agg = "global") {
   ## TODO: Get aggregation from recipe
   ## TODO: Add time frequency
 
-  outdir <- recipe$Run$output_dir
+  outdir <- paste0(recipe$Run$output_dir, "/outputs/")
   variable <- recipe$Analysis$Variables$name
   if (!is.null(recipe$Analysis$Time$fcst_year)) {
     if (tolower(recipe$Analysis$Horizon) == 'decadal') {
-- 
GitLab


From f8ac6fffd30965c85ff63006f9cf8da4670840f8 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 25 Oct 2022 10:29:22 +0200
Subject: [PATCH 09/13] Change stop() messages to log4r::error

---
 modules/Loading/Loading_decadal.R | 57 +++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 15 deletions(-)

diff --git a/modules/Loading/Loading_decadal.R b/modules/Loading/Loading_decadal.R
index 7b6b352f..e6f19fac 100644
--- a/modules/Loading/Loading_decadal.R
+++ b/modules/Loading/Loading_decadal.R
@@ -13,7 +13,6 @@ source("modules/Loading/check_latlon.R")
 source("tools/libs.R")
 ## TODO: Remove once the fun is included in CSTools
 source("tools/tmp/as.s2dv_cube.R")
-## TODO: Change stops to logger error messages
 
 #====================================================================
 
@@ -165,7 +164,9 @@ load_datasets <- function(recipe) {
   # dim(hcst) should be [dat, var, sday, sweek, syear, time, latitude, longitude, ensemble]
   dim(hcst) <- c(dim(hcst)[1:2], sday = 1, sweek = 1, dim(hcst)[3:7])
   if (!identical(dim(tmp_time_attr), dim(hcst)[c('syear', 'time')])) {
-    stop("hcst has problem in matching data and time attr dimension.")
+    error(recipe$Run$logger,
+	  "hcst has problem in matching data and time attr dimension.")
+    stop()
   }
   dim(attr(hcst, 'Variables')$common$time) <- c(sday = 1, sweek = 1, dim(tmp_time_attr))
 
@@ -237,7 +238,9 @@ load_datasets <- function(recipe) {
   # dim(fcst) should be [dat, var, sday, sweek, syear, time, latitude, longitude, ensemble]
   dim(fcst) <- c(dim(fcst)[1:2], sday = 1, sweek = 1, dim(fcst)[3:7])
   if (!identical(dim(tmp_time_attr), dim(fcst)[c('syear', 'time')])) {
-    stop("fcst has problem in matching data and time attr dimension.")
+    error(recipe$Run$logger,
+	  "fcst has problem in matching data and time attr dimension.")
+    stop()
   }
   dim(attr(fcst, 'Variables')$common$time) <- c(sday = 1, sweek = 1, dim(tmp_time_attr))
 
@@ -253,7 +256,9 @@ load_datasets <- function(recipe) {
 
   # Only syear could be different
   if (!identical(dim(hcst$data)[-5], dim(fcst$data)[-5])) {
-    stop("hcst and fcst do not share the same dimension structure.")
+    error(recipe$Run$logger,
+          "hcst and fcst do not share the same dimension structure.")
+    stop()
   }
 
   } else {
@@ -341,7 +346,9 @@ load_datasets <- function(recipe) {
 
   # Only ensemble dim could be different
   if (!identical(dim(obs), dim(hcst$data)[-9])) {
-    stop("obs and hcst dimensions do not match.")
+    error(recipe$Run$logger,
+	  "obs and hcst dimensions do not match.")
+    stop()
   }
   # Add ensemble dim to obs
   dim(obs) <- c(dim(obs), ensemble = 1)
@@ -357,45 +364,65 @@ load_datasets <- function(recipe) {
 #-------------------------------------------
   # dimension
   if (any(!names(dim(obs$data)) %in% names(dim(hcst$data)))) {
-    stop("hcst and obs don't share the same dimension names.")
+    error(recipe$Run$logger,
+	  "hcst and obs don't share the same dimension names.")
+    stop()
   } else {
     ens_ind <- which(names(dim(obs$data)) == 'ensemble')
     match_ind <- match(names(dim(obs$data))[-ens_ind], names(dim(hcst$data)))
-    if (!all(dim(hcst$data)[match_ind] == dim(obs$data)[-ens_ind])) stop("hcst and obs don't share the same dimension length.")
+    if (!all(dim(hcst$data)[match_ind] == dim(obs$data)[-ens_ind])) {
+      error(recipe$Run$logger,
+	    "hcst and obs don't share the same dimension length.")
+      stop()
+    }
   }
 
   # time attribute
   if (!identical(format(hcst$Dates$start, '%Y%m'),
                  format(obs$Dates$start, '%Y%m')))
-    stop("hcst and obs don't share the same time.")
+    error(recipe$Run$logger,
+	  "hcst and obs don't share the same time.")
+    stop()
 
   # lat and lon attributes
   if (!identical(as.vector(hcst$lat),
                  as.vector(obs$lat)))
-    stop("hcst and obs don't share the same latitude.")
+    error(recipe$Run$logger,
+	  "hcst and obs don't share the same latitude.")
+    stop()
   if (!identical(as.vector(hcst$lon),
                  as.vector(obs$lon)))
-    stop("hcst and obs don't share the same longitude.")
+    error(recipe$Run$logger,
+	  "hcst and obs don't share the same longitude.")
+    stop()
 
   # Check fcst
   if (!is.null(fcst)) {
     # dimension
     if (any(!names(dim(fcst$data)) %in% names(dim(hcst$data)))) {
-      stop("hcst and fcst don't share the same dimension names.")
+      error(recipe$Run$logger,
+	    "hcst and fcst don't share the same dimension names.")
+      stop()
     } else {
       ens_ind <- which(names(dim(fcst$data)) %in% c('ensemble', 'syear'))
       match_ind <- match(names(dim(fcst$data))[-ens_ind], names(dim(hcst$data)))
-    if (!all(dim(hcst$data)[match_ind] == dim(fcst$data)[-ens_ind])) 
-      stop("hcst and fcst don't share the same dimension length.")
+    if (!all(dim(hcst$data)[match_ind] == dim(fcst$data)[-ens_ind]))
+      error(recipe$Run$logger,
+	    "hcst and fcst don't share the same dimension length.")
+      stop()
     }
 
     # lat and lon attributes
     if (!identical(as.vector(hcst$lat),
                    as.vector(fcst$lat)))
-      stop("hcst and fcst don't share the same latitude.")
+      error(recipe$Run$logger,
+	    "hcst and fcst don't share the same latitude.")
+      stop()
     if (!identical(as.vector(hcst$lon),
                    as.vector(fcst$lon)))
-      stop("hcst and fcst don't share the same longitude.")
+      error(recipe$Run$logger,
+	    "hcst and fcst don't share the same longitude.")
+      stop()
   }
 
 
-- 
GitLab


From 5bf48230d504a47f0aac6dcb4b2e2d5bb95099a0 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 25 Oct 2022 11:56:06 +0200
Subject: [PATCH 10/13] Fix pipeline

---
 modules/Loading/Loading_decadal.R | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/modules/Loading/Loading_decadal.R b/modules/Loading/Loading_decadal.R
index e6f19fac..2aadfb95 100644
--- a/modules/Loading/Loading_decadal.R
+++ b/modules/Loading/Loading_decadal.R
@@ -379,22 +379,25 @@ load_datasets <- function(recipe) {
 
   # time attribute
   if (!identical(format(hcst$Dates$start, '%Y%m'),
-                 format(obs$Dates$start, '%Y%m')))
+                 format(obs$Dates$start, '%Y%m'))) {
     error(recipe$Run$logger,
 	  "hcst and obs don't share the same time.")
     stop()
+  }
 
   # lat and lon attributes
   if (!identical(as.vector(hcst$lat),
-                 as.vector(obs$lat)))
+                 as.vector(obs$lat))) {
     error(recipe$Run$logger,
 	  "hcst and obs don't share the same latitude.")
     stop()
+  }
   if (!identical(as.vector(hcst$lon),
-                 as.vector(obs$lon)))
+                 as.vector(obs$lon))) {
     error(recipe$Run$logger,
 	  "hcst and obs don't share the same longitude.")
     stop()
+  }
 
   # Check fcst
   if (!is.null(fcst)) {
@@ -406,23 +409,26 @@ load_datasets <- function(recipe) {
     } else {
       ens_ind <- which(names(dim(fcst$data)) %in% c('ensemble', 'syear'))
       match_ind <- match(names(dim(fcst$data))[-ens_ind], names(dim(hcst$data)))
-    if (!all(dim(hcst$data)[match_ind] == dim(fcst$data)[-ens_ind]))
-      error(recipe$Run$logger,
-	    "hcst and fcst don't share the same dimension length.")
-      stop()
+      if (!all(dim(hcst$data)[match_ind] == dim(fcst$data)[-ens_ind])) {
+        error(recipe$Run$logger,
+	      "hcst and fcst don't share the same dimension length.")
+        stop()
+      }
     }
 
     # lat and lon attributes
     if (!identical(as.vector(hcst$lat),
-                   as.vector(fcst$lat)))
+                   as.vector(fcst$lat))) {
       error(recipe$Run$logger,
 	    "hcst and fcst don't share the same latitude.")
       stop()
+    }
     if (!identical(as.vector(hcst$lon),
-                   as.vector(fcst$lon)))
+                   as.vector(fcst$lon))) {
       error(recipe$Run$logger,
 	    "hcst and fcst don't share the same longitude.")
       stop()
+    }
   }
 
 
-- 
GitLab


From d40b8d58b1463c1ae75f1c7c435692fdffcf54f5 Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Tue, 25 Oct 2022 16:34:04 +0200
Subject: [PATCH 11/13] Bugfix: change logger to recipe

---
 modules/Loading/Loading.R | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R
index 4436edc6..f78bd144 100644
--- a/modules/Loading/Loading.R
+++ b/modules/Loading/Loading.R
@@ -286,26 +286,26 @@ load_datasets <- function(recipe) {
       lat_error_msg <- paste("Latitude mismatch between hcst and obs.",
 			     "Please check the original grids and the",
 			     "regrid parameters in your recipe.")
-      error(logger, lat_error_msg)
+      error(recipe$Run$logger, lat_error_msg)
       hcst_lat_msg <- paste0("First hcst lat: ", hcst$lat[1],
 			     "; Last hcst lat: ", hcst$lat[length(hcst$lat)])
-      info(logger, hcst_lat_msg)
+      info(recipe$Run$logger, hcst_lat_msg)
       obs_lat_msg <- paste0("First obs lat: ", obs$lat[1],
 			     "; Last obs lat: ", obs$lat[length(obs$lat)])
-      info(logger, obs_lat_msg)
+      info(recipe$Run$logger, obs_lat_msg)
       stop("hcst and obs don't share the same latitudes.")
     }
     if (!identical(as.vector(hcst$lon), as.vector(obs$lon))) {
       lon_error_msg <- paste("Longitude mismatch between hcst and obs.",
 			     "Please check the original grids and the",
 			     "regrid parameters in your recipe.")
-      error(logger, lon_error_msg)
+      error(recipe$Run$logger, lon_error_msg)
       hcst_lon_msg <- paste0("First hcst lon: ", hcst$lon[1],
 			     "; Last hcst lon: ", hcst$lon[length(hcst$lon)])
-      info(logger, hcst_lon_msg)
+      info(recipe$Run$logger, hcst_lon_msg)
       obs_lon_msg <- paste0("First obs lon: ", obs$lon[1],
 			     "; Last obs lon: ", obs$lon[length(obs$lon)])
-      info(logger, obs_lon_msg)
+      info(recipe$Run$logger, obs_lon_msg)
       stop("hcst and obs don't share the same longitudes.")
 
     }
-- 
GitLab


From 73693d881b441dc46259c6c05ebb1b757f1a479e Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Wed, 26 Oct 2022 15:41:51 +0200
Subject: [PATCH 12/13] Refine lat/lon error message

---
 modules/Loading/Loading_decadal.R | 81 ++++++++++++++++++++++---------
 1 file changed, 58 insertions(+), 23 deletions(-)

diff --git a/modules/Loading/Loading_decadal.R b/modules/Loading/Loading_decadal.R
index 2aadfb95..e9f8b274 100644
--- a/modules/Loading/Loading_decadal.R
+++ b/modules/Loading/Loading_decadal.R
@@ -386,18 +386,35 @@ load_datasets <- function(recipe) {
   }
 
   # lat and lon attributes
-  if (!identical(as.vector(hcst$lat),
-                 as.vector(obs$lat))) {
-    error(recipe$Run$logger,
-	  "hcst and obs don't share the same latitude.")
-    stop()
-  }
-  if (!identical(as.vector(hcst$lon),
-                 as.vector(obs$lon))) {
-    error(recipe$Run$logger,
-	  "hcst and obs don't share the same longitude.")
-    stop()
-  }
+  if (!(recipe$Analysis$Regrid$type == 'none')) {
+    if (!identical(as.vector(hcst$lat), as.vector(obs$lat))) {
+      lat_error_msg <- paste("Latitude mismatch between hcst and obs.",
+			     "Please check the original grids and the",
+			     "regrid parameters in your recipe.")
+      error(recipe$Run$logger, lat_error_msg)
+      hcst_lat_msg <- paste0("First hcst lat: ", hcst$lat[1],
+			     "; Last hcst lat: ", hcst$lat[length(hcst$lat)])
+      info(recipe$Run$logger, hcst_lat_msg)
+      obs_lat_msg <- paste0("First obs lat: ", obs$lat[1],
+			     "; Last obs lat: ", obs$lat[length(obs$lat)])
+      info(recipe$Run$logger, obs_lat_msg)
+      stop("hcst and obs don't share the same latitudes.")
+    }
+
+    if (!identical(as.vector(hcst$lon), as.vector(obs$lon))) {
+      lon_error_msg <- paste("Longitude mismatch between hcst and obs.",
+			     "Please check the original grids and the",
+			     "regrid parameters in your recipe.")
+      error(recipe$Run$logger, lon_error_msg)
+      hcst_lon_msg <- paste0("First hcst lon: ", hcst$lon[1],
+			     "; Last hcst lon: ", hcst$lon[length(hcst$lon)])
+      info(recipe$Run$logger, hcst_lon_msg)
+      obs_lon_msg <- paste0("First obs lon: ", obs$lon[1],
+			     "; Last obs lon: ", obs$lon[length(obs$lon)])
+      info(recipe$Run$logger, obs_lon_msg)
+      stop("hcst and obs don't share the same longitudes.")
+    }
+ }
 
   # Check fcst
   if (!is.null(fcst)) {
@@ -417,18 +434,36 @@ load_datasets <- function(recipe) {
     }
 
     # lat and lon attributes
-    if (!identical(as.vector(hcst$lat),
-                   as.vector(fcst$lat))) {
-      error(recipe$Run$logger,
-	    "hcst and fcst don't share the same latitude.")
-      stop()
-    }
-    if (!identical(as.vector(hcst$lon),
-                   as.vector(fcst$lon))) {
-      error(recipe$Run$logger,
-	    "hcst and fcst don't share the same longitude.")
-      stop()
+    if (!(recipe$Analysis$Regrid$type == 'none')) {
+      if (!identical(as.vector(hcst$lat), as.vector(fcst$lat))) {
+        lat_error_msg <- paste("Latitude mismatch between hcst and fcst.",
+                               "Please check the original grids and the",
+                               "regrid parameters in your recipe.")
+        error(recipe$Run$logger, lat_error_msg)
+        hcst_lat_msg <- paste0("First hcst lat: ", hcst$lat[1],
+                               "; Last hcst lat: ", hcst$lat[length(hcst$lat)])
+        info(recipe$Run$logger, hcst_lat_msg)
+        fcst_lat_msg <- paste0("First fcst lat: ", fcst$lat[1],
+                               "; Last fcst lat: ", fcst$lat[length(fcst$lat)])
+        info(recipe$Run$logger, fcst_lat_msg)
+        stop("hcst and fcst don't share the same latitudes.")
+      }
+
+      if (!identical(as.vector(hcst$lon), as.vector(fcst$lon))) {
+        lon_error_msg <- paste("Longitude mismatch between hcst and fcst.",
+                               "Please check the original grids and the",
+                               "regrid parameters in your recipe.")
+        error(recipe$Run$logger, lon_error_msg)
+        hcst_lon_msg <- paste0("First hcst lon: ", hcst$lon[1],
+                               "; Last hcst lon: ", hcst$lon[length(hcst$lon)])
+        info(recipe$Run$logger, hcst_lon_msg)
+        fcst_lon_msg <- paste0("First fcst lon: ", fcst$lon[1],
+                               "; Last fcst lon: ", fcst$lon[length(fcst$lon)])
+        info(recipe$Run$logger, fcst_lon_msg)
+        stop("hcst and fcst don't share the same longitudes.")
+      }
     }
+
   }
 
 
-- 
GitLab


From b548380f61eb0ff2669cebd1594260c8433101db Mon Sep 17 00:00:00 2001
From: Victoria Agudetse Roures <victoria.agudetse@bsc.es>
Date: Thu, 27 Oct 2022 15:46:56 +0200
Subject: [PATCH 13/13] Fix pipeline

---
 modules/Calibration/Calibration.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/Calibration/Calibration.R b/modules/Calibration/Calibration.R
index df96789f..59e5451a 100644
--- a/modules/Calibration/Calibration.R
+++ b/modules/Calibration/Calibration.R
@@ -1,5 +1,5 @@
 
-calibrate_datasets <- function(data, recipe) {
+calibrate_datasets <- function(recipe, data) {
   # Function that calibrates the hindcast using the method stated in the
   # recipe. If the forecast is not null, it calibrates it as well.
   #
-- 
GitLab