From 3aef74091d6add43f1b5c86689a37d744b7dc64c Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Tue, 1 Oct 2024 10:20:08 +0200 Subject: [PATCH 1/7] stop when PatternDim is chunked --- R/Compute.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/R/Compute.R b/R/Compute.R index 5a58abd..7f9c9c7 100644 --- a/R/Compute.R +++ b/R/Compute.R @@ -155,6 +155,10 @@ Compute <- function(workflow, chunks = 'auto', workflow_manager = 'ecFlow', stop("Workflows with only one step supported by now.") } + if (any(names(chunks)==attr(workflow$inputs$input1,"PatternDim"))) { + stop(paste0("Chunking of ",attr(workflow$inputs$input1,"PatternDim")," not allowed")) + } + # Run ByChunks with the chosen operation if (!is.null(cluster)) { if (is.null(workflow_manager)) { -- GitLab From b411bde13221c1ff340c3c0168b774471134239b Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Tue, 1 Oct 2024 13:58:44 +0200 Subject: [PATCH 2/7] added updatexample_dat_chunk_error.R --- tests/example_dat_chunk_error.R | 86 +++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 tests/example_dat_chunk_error.R diff --git a/tests/example_dat_chunk_error.R b/tests/example_dat_chunk_error.R new file mode 100644 index 0000000..1b73a5f --- /dev/null +++ b/tests/example_dat_chunk_error.R @@ -0,0 +1,86 @@ +# case 1 default variable name 'dat' + library(startR) + path1 <- "/esarchive/exp/ecmwf/system5c3s/monthly_mean/$var$_f6h/$var$_$sdate$.nc" + path2 <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc" + + data <- Start(dat = list(list(name = 'system5c3s', path = path1), + list(name = 'system5_m1', path = path2)), + var = c('tas'), + sdate = paste0(2017:2018, '0501'), + ensemble = 'all', + time = indices(1:3), + lat = values(list(20, 80)), lat_reorder = Sort(), + lon = values(list(-80, 40)), lon_reorder = CircularSort(-180, 180), + transform = CDORemapper, + transform_extra_cells = 2, + transform_params = list(grid = 'r360x181', + method = 'conservative'), + transform_vars = c('lat', 'lon'), + synonims = list(lat = c('lat', 'latitude'), lon = c('lon', 'longitude')), + return_vars = list(time = 'sdate', lon = 'dat', lat = 'dat'), + retrieve = FALSE) + func <- function(x) { + return(x) + } + step <- Step(func, target_dims = c('lat', 'lon'), output_dims = c('lat', 'lon')) + wf <- AddStep(data, step) + res <- Compute(wf, chunks = list(sdate = 2, dat = 2)) + +# case 2 variable name changed to 'dt' + library(startR) + path1 <- "/esarchive/exp/ecmwf/system5c3s/monthly_mean/$var$_f6h/$var$_$sdate$.nc" + path2 <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc" + + data <- Start(dt = list(list(name = 'system5c3s', path = path1), + list(name = 'system5_m1', path = path2)), + var = c('tas'), + sdate = paste0(2017:2018, '0501'), + ensemble = 'all', + time = indices(1:3), + lat = values(list(20, 80)), lat_reorder = Sort(), + lon = values(list(-80, 40)), lon_reorder = CircularSort(-180, 180), + transform = CDORemapper, + transform_extra_cells = 2, + transform_params = list(grid = 'r360x181', + method = 'conservative'), + transform_vars = c('lat', 'lon'), + synonims = list(lat = c('lat', 'latitude'), lon = c('lon', 'longitude')), + return_vars = list(time = 'sdate', lon = 'dt', lat = 'dt'), + retrieve = FALSE) + func <- function(x) { + return(x) + } + step <- Step(func, target_dims = c('lat', 'lon'), output_dims = c('lat', 'lon')) + wf <- AddStep(data, step) + res <- Compute(wf, chunks = list(sdate = 2, dt = 2)) + +# case 3 variable name and input order changed + library(startR) + path1 <- "/esarchive/exp/ecmwf/system5c3s/monthly_mean/$var$_f6h/$var$_$sdate$.nc" + path2 <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc" + + data <- Start(var = c('tas'), + sdate = paste0(2017:2018, '0501'), + ensemble = 'all', + time = indices(1:3), + lat = values(list(20, 80)), lat_reorder = Sort(), + lon = values(list(-80, 40)), lon_reorder = CircularSort(-180, 180), + transform = CDORemapper, + transform_extra_cells = 2, + transform_params = list(grid = 'r360x181', + method = 'conservative'), + set = list(list(name = 'system5c3s', path = path1), + list(name = 'system5_m1', path = path2)), + transform_vars = c('lat', 'lon'), + pattern_dims = "set", + synonims = list(lat = c('lat', 'latitude'), lon = c('lon', 'longitude')), + return_vars = list(time = 'sdate', lon = 'set', lat = 'set'), + retrieve = FALSE) + func <- function(x) { + return(x) + } + step <- Step(func, target_dims = c('lat', 'lon'), output_dims = c('lat', 'lon')) + wf <- AddStep(data, step) + res <- Compute(wf, chunks = list(sdate = 2, set = 2)) + + -- GitLab From 5acf7814c9c851a1a8ab0732448c6b221fd65e2d Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Tue, 1 Oct 2024 15:33:02 +0200 Subject: [PATCH 3/7] style correction Compute.R --- R/Compute.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/Compute.R b/R/Compute.R index 7f9c9c7..2f0a301 100644 --- a/R/Compute.R +++ b/R/Compute.R @@ -155,8 +155,8 @@ Compute <- function(workflow, chunks = 'auto', workflow_manager = 'ecFlow', stop("Workflows with only one step supported by now.") } - if (any(names(chunks)==attr(workflow$inputs$input1,"PatternDim"))) { - stop(paste0("Chunking of ",attr(workflow$inputs$input1,"PatternDim")," not allowed")) + if (any(names(chunks) == attr(workflow$inputs$input1, "PatternDim"))) { + stop(paste0("Chunking of ", attr(workflow$inputs$input1, "PatternDim"), " not allowed")) } # Run ByChunks with the chosen operation -- GitLab From 9148920bb9418c092355d7cf22ef3c9e19b066f8 Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Tue, 1 Oct 2024 15:43:44 +0200 Subject: [PATCH 4/7] moved example_dat_chunk_error.R from tests --- tests/example_dat_chunk_error.R | 86 --------------------------------- 1 file changed, 86 deletions(-) delete mode 100644 tests/example_dat_chunk_error.R diff --git a/tests/example_dat_chunk_error.R b/tests/example_dat_chunk_error.R deleted file mode 100644 index 1b73a5f..0000000 --- a/tests/example_dat_chunk_error.R +++ /dev/null @@ -1,86 +0,0 @@ -# case 1 default variable name 'dat' - library(startR) - path1 <- "/esarchive/exp/ecmwf/system5c3s/monthly_mean/$var$_f6h/$var$_$sdate$.nc" - path2 <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc" - - data <- Start(dat = list(list(name = 'system5c3s', path = path1), - list(name = 'system5_m1', path = path2)), - var = c('tas'), - sdate = paste0(2017:2018, '0501'), - ensemble = 'all', - time = indices(1:3), - lat = values(list(20, 80)), lat_reorder = Sort(), - lon = values(list(-80, 40)), lon_reorder = CircularSort(-180, 180), - transform = CDORemapper, - transform_extra_cells = 2, - transform_params = list(grid = 'r360x181', - method = 'conservative'), - transform_vars = c('lat', 'lon'), - synonims = list(lat = c('lat', 'latitude'), lon = c('lon', 'longitude')), - return_vars = list(time = 'sdate', lon = 'dat', lat = 'dat'), - retrieve = FALSE) - func <- function(x) { - return(x) - } - step <- Step(func, target_dims = c('lat', 'lon'), output_dims = c('lat', 'lon')) - wf <- AddStep(data, step) - res <- Compute(wf, chunks = list(sdate = 2, dat = 2)) - -# case 2 variable name changed to 'dt' - library(startR) - path1 <- "/esarchive/exp/ecmwf/system5c3s/monthly_mean/$var$_f6h/$var$_$sdate$.nc" - path2 <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc" - - data <- Start(dt = list(list(name = 'system5c3s', path = path1), - list(name = 'system5_m1', path = path2)), - var = c('tas'), - sdate = paste0(2017:2018, '0501'), - ensemble = 'all', - time = indices(1:3), - lat = values(list(20, 80)), lat_reorder = Sort(), - lon = values(list(-80, 40)), lon_reorder = CircularSort(-180, 180), - transform = CDORemapper, - transform_extra_cells = 2, - transform_params = list(grid = 'r360x181', - method = 'conservative'), - transform_vars = c('lat', 'lon'), - synonims = list(lat = c('lat', 'latitude'), lon = c('lon', 'longitude')), - return_vars = list(time = 'sdate', lon = 'dt', lat = 'dt'), - retrieve = FALSE) - func <- function(x) { - return(x) - } - step <- Step(func, target_dims = c('lat', 'lon'), output_dims = c('lat', 'lon')) - wf <- AddStep(data, step) - res <- Compute(wf, chunks = list(sdate = 2, dt = 2)) - -# case 3 variable name and input order changed - library(startR) - path1 <- "/esarchive/exp/ecmwf/system5c3s/monthly_mean/$var$_f6h/$var$_$sdate$.nc" - path2 <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc" - - data <- Start(var = c('tas'), - sdate = paste0(2017:2018, '0501'), - ensemble = 'all', - time = indices(1:3), - lat = values(list(20, 80)), lat_reorder = Sort(), - lon = values(list(-80, 40)), lon_reorder = CircularSort(-180, 180), - transform = CDORemapper, - transform_extra_cells = 2, - transform_params = list(grid = 'r360x181', - method = 'conservative'), - set = list(list(name = 'system5c3s', path = path1), - list(name = 'system5_m1', path = path2)), - transform_vars = c('lat', 'lon'), - pattern_dims = "set", - synonims = list(lat = c('lat', 'latitude'), lon = c('lon', 'longitude')), - return_vars = list(time = 'sdate', lon = 'set', lat = 'set'), - retrieve = FALSE) - func <- function(x) { - return(x) - } - step <- Step(func, target_dims = c('lat', 'lon'), output_dims = c('lat', 'lon')) - wf <- AddStep(data, step) - res <- Compute(wf, chunks = list(sdate = 2, set = 2)) - - -- GitLab From f11d42393f62fb6b748a297c6345d69dfc29f97c Mon Sep 17 00:00:00 2001 From: vagudets Date: Tue, 1 Oct 2024 16:41:08 +0200 Subject: [PATCH 5/7] Make chunking along dat error message more explicit --- R/Compute.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/R/Compute.R b/R/Compute.R index 2f0a301..1c787f4 100644 --- a/R/Compute.R +++ b/R/Compute.R @@ -156,8 +156,10 @@ Compute <- function(workflow, chunks = 'auto', workflow_manager = 'ecFlow', } if (any(names(chunks) == attr(workflow$inputs$input1, "PatternDim"))) { - stop(paste0("Chunking of ", attr(workflow$inputs$input1, "PatternDim"), " not allowed")) - } + stop(paste0("Chunking along the pattern dimension ", + attr(workflow$inputs$input1, "PatternDim"), + " is not allowed for now.")) + } # Run ByChunks with the chosen operation if (!is.null(cluster)) { -- GitLab From 57ca8938cabe09463d9eaba89a8e58b727ecaf62 Mon Sep 17 00:00:00 2001 From: vagudets Date: Tue, 1 Oct 2024 16:42:17 +0200 Subject: [PATCH 6/7] Fix indentation issue --- R/Compute.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/Compute.R b/R/Compute.R index 1c787f4..5fe4b73 100644 --- a/R/Compute.R +++ b/R/Compute.R @@ -155,11 +155,11 @@ Compute <- function(workflow, chunks = 'auto', workflow_manager = 'ecFlow', stop("Workflows with only one step supported by now.") } - if (any(names(chunks) == attr(workflow$inputs$input1, "PatternDim"))) { + if (any(names(chunks) == attr(workflow$inputs$input1, "PatternDim"))) { stop(paste0("Chunking along the pattern dimension ", attr(workflow$inputs$input1, "PatternDim"), " is not allowed for now.")) - } + } # Run ByChunks with the chosen operation if (!is.null(cluster)) { -- GitLab From 32aac586d94f94dda66ba385593c563a91ab489b Mon Sep 17 00:00:00 2001 From: vagudets Date: Tue, 1 Oct 2024 16:42:48 +0200 Subject: [PATCH 7/7] Run unit tests --- .Rbuildignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.Rbuildignore b/.Rbuildignore index aa7059a..98316cc 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -9,7 +9,7 @@ ^inst/doc$ ^\.gitlab-ci\.yml$ ## unit tests should be ignored when building the package for CRAN -^tests$ +#^tests$ ^inst/PlotProfiling\.R$ ^.gitlab$ # Suggested by http://r-pkgs.had.co.nz/package.html -- GitLab