From d51d89eb4a6df4de637fdb532f7cbe2a42d33f38 Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 22 May 2025 15:05:59 +0200 Subject: [PATCH 1/8] Remove recipe dependency --- modules/Loading/R/get_timeidx.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/Loading/R/get_timeidx.R b/modules/Loading/R/get_timeidx.R index 2796b202..c27353cf 100644 --- a/modules/Loading/R/get_timeidx.R +++ b/modules/Loading/R/get_timeidx.R @@ -12,7 +12,8 @@ #'@param time_freq time frequency ("monthly_mean" or "daily_mean" or "daily") get_timeidx <- function(sdates, ltmin, ltmax, - time_freq="monthly_mean") { + time_freq="monthly_mean", + ncores = 1) { if (time_freq %in% c("daily_mean", "daily")) { @@ -55,7 +56,7 @@ get_timeidx <- function(sdates, ltmin, ltmax, output_dims = c("time"), idx_min = ltmin - 1, idx_max = ltmax - 1, - ncores = recipe$Analysis$ncores + ncores = ncores )[[1]] indxs_dims <- dim(indxs) indxs <- as.POSIXct(indxs, origin = "1970-01-01", tz = "UTC") -- GitLab From eb1b7a919de3065fc1d3936473439dce0b47d7a8 Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 22 May 2025 15:06:11 +0200 Subject: [PATCH 2/8] Unit test for get_timeidx() --- tests/testthat/test-utils-get_timeidx.R | 115 ++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 tests/testthat/test-utils-get_timeidx.R diff --git a/tests/testthat/test-utils-get_timeidx.R b/tests/testthat/test-utils-get_timeidx.R new file mode 100644 index 00000000..2f0cb08a --- /dev/null +++ b/tests/testthat/test-utils-get_timeidx.R @@ -0,0 +1,115 @@ +context("get_plot_time_labels.R") + +source("tools/libs.R") +source("modules/Loading/R/get_timeidx.R") +source("modules/Loading/R/dates2load.R") +########################################### + +sdates <- list() +sdates$fcst <- "20250501" +dim(sdates$fcst) <- c(sday = 1, sweek = 1, syear = 1) +sdates$hcst <- paste0(1993:2016, "0501") +dim(sdates$hcst) <- c(sday = 1, sweek = 1, syear = length(1993:2016)) + +testthat("1. Seasonal daily, 6 months", { + ltmin <- 1 + ltmax <- 6 + time_freq <- "daily_mean" + # Step 1: generate sdates + + idxs <- list() + idxs$hcst <- get_timeidx(sdates = sdates$hcst, + ltmin = ltmin, + ltmax = ltmax, + time_freq = time_freq) + + idxs$fcst <- get_timeidx(sdates = sdates$fcst, + ltmin = ltmin, + ltmax = ltmax, + time_freq = time_freq) + # Results + expect_equal( + dim(idxs$hcst), + c(file_date = 24, time = 184) + ) + expect_equal( + idxs$hcst[1], + "1993-05-01 12:01:00 UTC" + ) + expect_equal( + idxs$hcst[24, 2], + "2016-05-02 12:01:00 UTC" + ) + expect_equal( + idxs$hcst[4, 91], + "1996-07-30 12:01:00 UTC" + ) + expect_equal( + dim(idxs$fcst), + c(file_date = 1, time = 184) + ) + expect_equal( + idxs$fcst[1], + "2025-05-01 12:01:00 UTC" + ) + expect_equal( + idxs$fcst[1, 2] + "2025-05-02 12:01:00 UTC" + ) + expect_equal( + idxs$fcst[1, 91], + "2025-07-30 12:01:00 UTC" + ) +}) + + +sdates <- list() +sdates$fcst <- "20231201" +dim(sdates$fcst) <- c(sday = 1, sweek = 1, syear = 1) +sdates$hcst <- paste0(1993:1996, "1201") +dim(sdates$hcst) <- c(sday = 1, sweek = 1, syear = length(1993:1996)) + +testthat("2. Seasonal daily, 6 months", { + ltmin <- 1 + ltmax <- 2 + time_freq <- "monthly_mean" + # Step 1: generate sdates + + idxs <- list() + idxs$hcst <- get_timeidx(sdates = sdates$hcst, + ltmin = ltmin, + ltmax = ltmax, + time_freq = time_freq) + + idxs$fcst <- get_timeidx(sdates = sdates$fcst, + ltmin = ltmin, + ltmax = ltmax, + time_freq = time_freq) + # Results + expect_equal( + dim(idxs$hcst), + c(time = 2, file_date = 4) + ) + expect_equal( + idxs$hcst[1, ], + c("1993-12-16 12:01:00 UTC", "1994-12-16 12:01:00 UTC", + "1995-12-16 12:01:00 UTC", "1996-12-16 12:01:00 UTC") + ) + expect_equal( + idxs$hcst[2, ], + c("1994-01-16 12:01:00 UTC", "1995-01-16 12:01:00 UTC", + "1996-01-16 12:01:00 UTC", "1997-01-16 12:01:00 UTC") + ) + expect_equal( + dim(idxs$fcst), + c(time = 2, file_date = 1) + ) + expect_equal( + idxs$fcst[1, ], + "2023-12-16 12:01:00 UTC" + ) + expect_equal( + idxs$fcst[2, ], + "2023-01-16 12:01:00 UTC" + ) +}) -- GitLab From 11b6c999624926312861527cbedb3cbc67b93f78 Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 22 May 2025 15:08:41 +0200 Subject: [PATCH 3/8] Comments --- tests/testthat/test-utils-get_timeidx.R | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/testthat/test-utils-get_timeidx.R b/tests/testthat/test-utils-get_timeidx.R index 2f0cb08a..6f9069a0 100644 --- a/tests/testthat/test-utils-get_timeidx.R +++ b/tests/testthat/test-utils-get_timeidx.R @@ -15,7 +15,6 @@ testthat("1. Seasonal daily, 6 months", { ltmin <- 1 ltmax <- 6 time_freq <- "daily_mean" - # Step 1: generate sdates idxs <- list() idxs$hcst <- get_timeidx(sdates = sdates$hcst, @@ -69,11 +68,10 @@ dim(sdates$fcst) <- c(sday = 1, sweek = 1, syear = 1) sdates$hcst <- paste0(1993:1996, "1201") dim(sdates$hcst) <- c(sday = 1, sweek = 1, syear = length(1993:1996)) -testthat("2. Seasonal daily, 6 months", { +testthat("2. Seasonal monthly, 2 months, end of year", { ltmin <- 1 ltmax <- 2 time_freq <- "monthly_mean" - # Step 1: generate sdates idxs <- list() idxs$hcst <- get_timeidx(sdates = sdates$hcst, -- GitLab From 06d90d9a1d5774c40f561e72437272f09a9687ce Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 22 May 2025 15:34:32 +0200 Subject: [PATCH 4/8] Fix pipeline --- tests/testthat/test-utils-get_timeidx.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-utils-get_timeidx.R b/tests/testthat/test-utils-get_timeidx.R index 6f9069a0..539fa8c8 100644 --- a/tests/testthat/test-utils-get_timeidx.R +++ b/tests/testthat/test-utils-get_timeidx.R @@ -52,7 +52,7 @@ testthat("1. Seasonal daily, 6 months", { "2025-05-01 12:01:00 UTC" ) expect_equal( - idxs$fcst[1, 2] + idxs$fcst[1, 2], "2025-05-02 12:01:00 UTC" ) expect_equal( -- GitLab From c29311de41ef626bb7a2f34414c414fcf91b513e Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 23 May 2025 10:40:31 +0200 Subject: [PATCH 5/8] Fix typo (test_that instead of testthat) --- tests/testthat/test-utils-get_timeidx.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/testthat/test-utils-get_timeidx.R b/tests/testthat/test-utils-get_timeidx.R index 539fa8c8..330845ba 100644 --- a/tests/testthat/test-utils-get_timeidx.R +++ b/tests/testthat/test-utils-get_timeidx.R @@ -1,4 +1,4 @@ -context("get_plot_time_labels.R") +context("get_timeidx.R") source("tools/libs.R") source("modules/Loading/R/get_timeidx.R") @@ -11,7 +11,7 @@ dim(sdates$fcst) <- c(sday = 1, sweek = 1, syear = 1) sdates$hcst <- paste0(1993:2016, "0501") dim(sdates$hcst) <- c(sday = 1, sweek = 1, syear = length(1993:2016)) -testthat("1. Seasonal daily, 6 months", { +test_that("1. Seasonal daily, 6 months", { ltmin <- 1 ltmax <- 6 time_freq <- "daily_mean" @@ -68,7 +68,7 @@ dim(sdates$fcst) <- c(sday = 1, sweek = 1, syear = 1) sdates$hcst <- paste0(1993:1996, "1201") dim(sdates$hcst) <- c(sday = 1, sweek = 1, syear = length(1993:1996)) -testthat("2. Seasonal monthly, 2 months, end of year", { +test_that("2. Seasonal monthly, 2 months, end of year", { ltmin <- 1 ltmax <- 2 time_freq <- "monthly_mean" -- GitLab From 5c36a073919a278e47a96073b95b40d61075d0e1 Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 23 May 2025 11:52:54 +0200 Subject: [PATCH 6/8] Fix pipeline --- tests/testthat/test-utils-get_timeidx.R | 44 ++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/testthat/test-utils-get_timeidx.R b/tests/testthat/test-utils-get_timeidx.R index 330845ba..7dc2d4ef 100644 --- a/tests/testthat/test-utils-get_timeidx.R +++ b/tests/testthat/test-utils-get_timeidx.R @@ -32,32 +32,32 @@ test_that("1. Seasonal daily, 6 months", { c(file_date = 24, time = 184) ) expect_equal( - idxs$hcst[1], - "1993-05-01 12:01:00 UTC" + as.character(idxs$hcst[1]), + "1993-05-01 12:01:00" ) expect_equal( - idxs$hcst[24, 2], - "2016-05-02 12:01:00 UTC" + as.character(idxs$hcst[24, 2]), + "2016-05-02 12:01:00" ) expect_equal( - idxs$hcst[4, 91], - "1996-07-30 12:01:00 UTC" + as.character(idxs$hcst[4, 91]), + "1996-07-30 12:01:00" ) expect_equal( dim(idxs$fcst), c(file_date = 1, time = 184) ) expect_equal( - idxs$fcst[1], - "2025-05-01 12:01:00 UTC" + as.character(idxs$fcst[1]), + "2025-05-01 12:01:00" ) expect_equal( - idxs$fcst[1, 2], - "2025-05-02 12:01:00 UTC" + as.character(idxs$fcst[1, 2]), + "2025-05-02 12:01:00" ) expect_equal( - idxs$fcst[1, 91], - "2025-07-30 12:01:00 UTC" + as.character(idxs$fcst[1, 91]), + "2025-07-30 12:01:00" ) }) @@ -89,25 +89,25 @@ test_that("2. Seasonal monthly, 2 months, end of year", { c(time = 2, file_date = 4) ) expect_equal( - idxs$hcst[1, ], - c("1993-12-16 12:01:00 UTC", "1994-12-16 12:01:00 UTC", - "1995-12-16 12:01:00 UTC", "1996-12-16 12:01:00 UTC") + as.character(idxs$hcst[1, ]), + c("1993-12-16 12:01:00", "1994-12-16 12:01:00", + "1995-12-16 12:01:00", "1996-12-16 12:01:00") ) expect_equal( - idxs$hcst[2, ], - c("1994-01-16 12:01:00 UTC", "1995-01-16 12:01:00 UTC", - "1996-01-16 12:01:00 UTC", "1997-01-16 12:01:00 UTC") + as.character(idxs$hcst[2, ]), + c("1994-01-16 12:01:00", "1995-01-16 12:01:00", + "1996-01-16 12:01:00", "1997-01-16 12:01:00") ) expect_equal( dim(idxs$fcst), c(time = 2, file_date = 1) ) expect_equal( - idxs$fcst[1, ], - "2023-12-16 12:01:00 UTC" + as.character(idxs$fcst[1, ]), + "2023-12-16 12:01:00" ) expect_equal( - idxs$fcst[2, ], - "2023-01-16 12:01:00 UTC" + as.character(idxs$fcst[2, ]), + "2024-01-16 12:01:00" ) }) -- GitLab From 217ea116a35ddb6e0de829fa2ece09ec4332b781 Mon Sep 17 00:00:00 2001 From: vagudets Date: Tue, 3 Jun 2025 09:17:51 +0200 Subject: [PATCH 7/8] Change get_timeidx() name to get_forecast_times() --- .../R/{get_timeidx.R => get_forecast_times.R} | 11 +++--- modules/Loading/R/load_decadal.R | 2 +- modules/Loading/R/load_seasonal.R | 18 +++++----- modules/Loading/R/load_subseasonal.R | 2 +- ...eidx.R => test-utils-get_forecast_times.R} | 36 +++++++++---------- 5 files changed, 35 insertions(+), 34 deletions(-) rename modules/Loading/R/{get_timeidx.R => get_forecast_times.R} (88%) rename tests/testthat/{test-utils-get_timeidx.R => test-utils-get_forecast_times.R} (68%) diff --git a/modules/Loading/R/get_timeidx.R b/modules/Loading/R/get_forecast_times.R similarity index 88% rename from modules/Loading/R/get_timeidx.R rename to modules/Loading/R/get_forecast_times.R index c27353cf..621c7d27 100644 --- a/modules/Loading/R/get_timeidx.R +++ b/modules/Loading/R/get_forecast_times.R @@ -2,18 +2,19 @@ #'to the start dates, min. and max. leadtimes and #'time frequency. # +#'The leadtimes are returned as dates in POSIXct format, +#'so that they can be read by Start(). #'The leadtimes are defined by months -#'Ex. 20201101 with leadtimes 1-4 corresponds to -#'the forecasting times covering November to february #' #'@param sdates vector containind the start dates #'@param ltmin first leadtime #'@param ltmax last leadtime #'@param time_freq time frequency ("monthly_mean" or "daily_mean" or "daily") +#'@param ncores number of cores to use for parallel computation -get_timeidx <- function(sdates, ltmin, ltmax, - time_freq="monthly_mean", - ncores = 1) { +get_forecast_times <- function(sdates, ltmin, ltmax, + time_freq="monthly_mean", + ncores = 1) { if (time_freq %in% c("daily_mean", "daily")) { diff --git a/modules/Loading/R/load_decadal.R b/modules/Loading/R/load_decadal.R index c55688e0..cdb3d339 100644 --- a/modules/Loading/R/load_decadal.R +++ b/modules/Loading/R/load_decadal.R @@ -3,7 +3,7 @@ source("modules/Loading/R/get_regrid_params.R") source("modules/Loading/R/helper_loading_decadal.R") source("modules/Loading/R/dates2load.R") source("modules/Loading/R/check_latlon.R") -source("modules/Loading/R/get_timeidx.R") +source("modules/Loading/R/get_forecast_times.R") source("modules/Loading/R/compare_exp_obs_grids.R") #==================================================================== diff --git a/modules/Loading/R/load_seasonal.R b/modules/Loading/R/load_seasonal.R index 1e6c9367..1824b7d5 100644 --- a/modules/Loading/R/load_seasonal.R +++ b/modules/Loading/R/load_seasonal.R @@ -1,7 +1,7 @@ # Load required libraries/funs source("modules/Loading/R/get_regrid_params.R") source("modules/Loading/R/dates2load.R") -source("modules/Loading/R/get_timeidx.R") +source("modules/Loading/R/get_forecast_times.R") source("modules/Loading/R/check_latlon.R") source("modules/Loading/R/compare_exp_obs_grids.R") @@ -27,16 +27,16 @@ load_seasonal <- function(recipe) { sdates <- dates2load(recipe, recipe$Run$logger) idxs <- NULL - idxs$hcst <- get_timeidx(sdates$hcst, - recipe$Analysis$Time$ftime_min, - recipe$Analysis$Time$ftime_max, - time_freq=store.freq) + idxs$hcst <- get_forecast_times(sdates$hcst, + recipe$Analysis$Time$ftime_min, + recipe$Analysis$Time$ftime_max, + time_freq=store.freq) if (!(is.null(sdates$fcst))) { - idxs$fcst <- get_timeidx(sdates$fcst, - recipe$Analysis$Time$ftime_min, - recipe$Analysis$Time$ftime_max, - time_freq=store.freq) + idxs$fcst <- get_forecast_times(sdates$fcst, + recipe$Analysis$Time$ftime_min, + recipe$Analysis$Time$ftime_max, + time_freq=store.freq) } # get datasets dict: diff --git a/modules/Loading/R/load_subseasonal.R b/modules/Loading/R/load_subseasonal.R index da316a61..dba21ee1 100644 --- a/modules/Loading/R/load_subseasonal.R +++ b/modules/Loading/R/load_subseasonal.R @@ -33,7 +33,7 @@ load_subseasonal <- function(recipe) { ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name # get datasets dict: - archive <-get_archive(recipe) # read_yaml("conf/archive_subseasonal.yml")[[recipe$Run$filesystem]] + archive <- get_archive(recipe) # read_yaml("conf/archive_subseasonal.yml")[[recipe$Run$filesystem]] exp_descrip <- archive$System[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable[1]]) diff --git a/tests/testthat/test-utils-get_timeidx.R b/tests/testthat/test-utils-get_forecast_times.R similarity index 68% rename from tests/testthat/test-utils-get_timeidx.R rename to tests/testthat/test-utils-get_forecast_times.R index 7dc2d4ef..89e6dbc6 100644 --- a/tests/testthat/test-utils-get_timeidx.R +++ b/tests/testthat/test-utils-get_forecast_times.R @@ -1,7 +1,7 @@ -context("get_timeidx.R") +context("get_forecast_times.R") source("tools/libs.R") -source("modules/Loading/R/get_timeidx.R") +source("modules/Loading/R/get_forecast_times.R") source("modules/Loading/R/dates2load.R") ########################################### @@ -17,15 +17,15 @@ test_that("1. Seasonal daily, 6 months", { time_freq <- "daily_mean" idxs <- list() - idxs$hcst <- get_timeidx(sdates = sdates$hcst, - ltmin = ltmin, - ltmax = ltmax, - time_freq = time_freq) + idxs$hcst <- get_forecast_times(sdates = sdates$hcst, + ltmin = ltmin, + ltmax = ltmax, + time_freq = time_freq) - idxs$fcst <- get_timeidx(sdates = sdates$fcst, - ltmin = ltmin, - ltmax = ltmax, - time_freq = time_freq) + idxs$fcst <- get_forecast_times(sdates = sdates$fcst, + ltmin = ltmin, + ltmax = ltmax, + time_freq = time_freq) # Results expect_equal( dim(idxs$hcst), @@ -74,15 +74,15 @@ test_that("2. Seasonal monthly, 2 months, end of year", { time_freq <- "monthly_mean" idxs <- list() - idxs$hcst <- get_timeidx(sdates = sdates$hcst, - ltmin = ltmin, - ltmax = ltmax, - time_freq = time_freq) + idxs$hcst <- get_forecast_times(sdates = sdates$hcst, + ltmin = ltmin, + ltmax = ltmax, + time_freq = time_freq) - idxs$fcst <- get_timeidx(sdates = sdates$fcst, - ltmin = ltmin, - ltmax = ltmax, - time_freq = time_freq) + idxs$fcst <- get_forecast_times(sdates = sdates$fcst, + ltmin = ltmin, + ltmax = ltmax, + time_freq = time_freq) # Results expect_equal( dim(idxs$hcst), -- GitLab From ad5561416e762fbc12d7e992aac579e3e14803d2 Mon Sep 17 00:00:00 2001 From: vagudets Date: Tue, 3 Jun 2025 09:57:53 +0200 Subject: [PATCH 8/8] Change get_timeidx() name --- modules/Loading/R/load_tas_tos.R | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/Loading/R/load_tas_tos.R b/modules/Loading/R/load_tas_tos.R index d035f05d..c19b7992 100644 --- a/modules/Loading/R/load_tas_tos.R +++ b/modules/Loading/R/load_tas_tos.R @@ -1,7 +1,7 @@ # Load required libraries/funs source("modules/Loading/R/get_regrid_params.R") source("modules/Loading/R/dates2load.R") -source("modules/Loading/R/get_timeidx.R") +source("modules/Loading/R/get_forecast_times.R") source("modules/Loading/R/check_latlon.R") source('modules/Loading/R/mask_tas_tos.R') source("modules/Loading/R/compare_exp_obs_grids.R") @@ -37,16 +37,16 @@ load_tas_tos <- function(recipe) { sdates <- dates2load(recipe, recipe$Run$logger) idxs <- NULL - idxs$hcst <- get_timeidx(sdates$hcst, - recipe$Analysis$Time$ftime_min, - recipe$Analysis$Time$ftime_max, - time_freq = store.freq) + idxs$hcst <- get_forecast_times(sdates$hcst, + recipe$Analysis$Time$ftime_min, + recipe$Analysis$Time$ftime_max, + time_freq = store.freq) if (!(is.null(sdates$fcst))) { - idxs$fcst <- get_timeidx(sdates$fcst, - recipe$Analysis$Time$ftime_min, - recipe$Analysis$Time$ftime_max, - time_freq = store.freq) + idxs$fcst <- get_forecast_times(sdates$fcst, + recipe$Analysis$Time$ftime_min, + recipe$Analysis$Time$ftime_max, + time_freq = store.freq) } # get esarchive datasets dict: -- GitLab