From ff039c9af1d9fd6e91009fa7aa95c7a750275d09 Mon Sep 17 00:00:00 2001 From: nperez Date: Wed, 30 Oct 2019 18:21:26 +0100 Subject: [PATCH 1/6] sub_array_indices is a list of 1 element --- R/Start.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/Start.R b/R/Start.R index 21def44..66a5b01 100644 --- a/R/Start.R +++ b/R/Start.R @@ -2120,8 +2120,10 @@ print(str(tvi)) if (goes_across_prime_meridian) { #sub_array_of_fri <- 1:n sub_array_of_fri <- c(1:sub_array_of_indices[[2]], sub_array_of_indices[[1]]:n) - } else if (is.list(sub_array_of_indices)) { + } else if (is.list(sub_array_of_indices) & length(list(sub_array_of_indices)) > 1) { sub_array_of_fri <- sub_array_of_indices[[1]]:sub_array_of_indices[[2]] + } else if (is.list(sub_array_of_indices) & length(list(sub_array_of_indices)) == 1) { + sub_array_of_fri <- sub_array_of_indices[[1]] } else { sub_array_of_fri <- sub_array_of_indices } -- GitLab From 7c15c3bdd4c46dcce4e24aa4448a761712c66da0 Mon Sep 17 00:00:00 2001 From: nperez Date: Wed, 30 Oct 2019 18:35:30 +0100 Subject: [PATCH 2/6] removing list not required --- R/Start.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/Start.R b/R/Start.R index 66a5b01..eaaacc5 100644 --- a/R/Start.R +++ b/R/Start.R @@ -2120,9 +2120,9 @@ print(str(tvi)) if (goes_across_prime_meridian) { #sub_array_of_fri <- 1:n sub_array_of_fri <- c(1:sub_array_of_indices[[2]], sub_array_of_indices[[1]]:n) - } else if (is.list(sub_array_of_indices) & length(list(sub_array_of_indices)) > 1) { + } else if (is.list(sub_array_of_indices) & length(sub_array_of_indices) > 1) { sub_array_of_fri <- sub_array_of_indices[[1]]:sub_array_of_indices[[2]] - } else if (is.list(sub_array_of_indices) & length(list(sub_array_of_indices)) == 1) { + } else if (is.list(sub_array_of_indices) & length(sub_array_of_indices) == 1) { sub_array_of_fri <- sub_array_of_indices[[1]] } else { sub_array_of_fri <- sub_array_of_indices -- GitLab From 1d66d5d4c012d8e4c240664062d118adc3dacaba Mon Sep 17 00:00:00 2001 From: nperez Date: Wed, 6 Nov 2019 11:48:52 +0100 Subject: [PATCH 3/6] Use case added to check Start modification in StartIndices branch --- inst/doc/usecase.md | 1 + inst/doc/usecase/ex2_8_calibration.R | 85 ++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 inst/doc/usecase/ex2_8_calibration.R diff --git a/inst/doc/usecase.md b/inst/doc/usecase.md index a584f1e..2bfb989 100644 --- a/inst/doc/usecase.md +++ b/inst/doc/usecase.md @@ -17,3 +17,4 @@ In this document, you can link to the example scripts for various demands. For t 5. [Using experimental and (date-corresponding) observational data](inst/doc/usecase/ex2_5_exp_and_obs.R) 6. [Use external parameters in atomic function](inst/doc/usecase/ex2_6_ext_param_func.R) 7. [Seasonal forecast verification on cca](inst/doc/usecase/ex2_7_seasonal_forecast_verification.R) + 8. [Use CSTools Calibration function](inst/doc/usecase/ex2_8_calibration.R) diff --git a/inst/doc/usecase/ex2_8_calibration.R b/inst/doc/usecase/ex2_8_calibration.R new file mode 100644 index 0000000..13b0739 --- /dev/null +++ b/inst/doc/usecase/ex2_8_calibration.R @@ -0,0 +1,85 @@ +# Load startR +library(startR) + +# Define a region +lons.min <- 0 +lons.max <- 10 +lats.min <- 0 +lats.max <- 10 + +# Declaration of data sources +exp <- Start(dat = '/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc', + var = 'tas', + sdate = paste0(2000:2010, '0101'), + ensemble = 'all', + time = indices(1), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(), + longitude = values(list(lons.min, lons.max)), + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = c('sdate')), + retrieve = FALSE) + +obs <- Start(dat = '/esarchive/recon/ecmwf/erainterim/monthly_mean/$var$_f6h/$var$_$sdate$.nc', + var = 'tas', + sdate = paste0(2000:2010, '02'), + time = indices(1), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(), + longitude = values(list(lons.min, lons.max)), + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = c('sdate')), + split_multiselected_dims = TRUE, + merge_across_dims = TRUE, + retrieve = FALSE) + +# Define of the workflow + +# Function +wrap_cal <- function(obs, exp) { + obs <- s2dverification::InsertDim(obs, 1, 1) + names(dim(obs))<- c('member', 'sdate') + exp <- t(exp) + names(dim(exp))<- c('member', 'sdate') + calibrated <- CSTools:::.cal(var_obs = obs, var_exp = exp) # CSTools version 1.0.1 or earlier + # calibrated <- CSTools:::.cal(mod = exp, obs = obs, + # cal.method = "mse_min", + # eval.method = "leave-one-out", + # multi.model = FALSE) + return(calibrated) +} + +step <- Step(wrap_cal, + target_dims = list(obs = c('sdate'), exp = c('sdate', 'ensemble')), + output_dims = c('ensemble', 'sdate')) + +# workflow of operations +wf <- AddStep(list(obs = obs, exp = exp), step) + + +# Execution +res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), + threads_load = 2, threads_compute = 4) +## Check output: +#summary(res$output1) +# Min. 1st Qu. Median Mean 3rd Qu. Max. +# 294.1 298.8 300.5 300.6 302.2 306.7 + + +# Declaration of HPC and execution +## ECFlow is required +res_fat2 <- Compute(wf,chunks = list(latitude = 2, longitude = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = "bsceslogin01.bsc.es", + cores_per_job = 2, + max_jobs = 4, job_wallclock = '00:10:00'), + ecflow_suite_dir = "/esarchive/scratch/nperez/ecflow") # your path! +## Check output: +#summary(res_fat2$output1) +# Min. 1st Qu. Median Mean 3rd Qu. Max. +# 294.1 298.8 300.5 300.6 302.2 306.7 + + -- GitLab From ed4a71883dabb3008fab95b65bc67fdf32b963eb Mon Sep 17 00:00:00 2001 From: aho Date: Fri, 15 Nov 2019 16:26:02 +0100 Subject: [PATCH 4/6] Modify the usage of chunk_indices() when 'sub_array_of_indices' is list --- R/Start.R | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/R/Start.R b/R/Start.R index b26170f..ad57b66 100644 --- a/R/Start.R +++ b/R/Start.R @@ -1983,10 +1983,19 @@ print("-> THE INNER DIMENSION DOES NOT GO ACROSS ANY FILE DIMENSION OR IT DOES B tolerance_params[[inner_dim]] }) } - sub_array_of_indices <- sub_array_of_indices[chunk_indices(length(sub_array_of_indices), - chunks[[inner_dim]]['chunk'], - chunks[[inner_dim]]['n_chunks'], - inner_dim)] + if (!is.list(sub_array_of_indices)) { + sub_array_of_indices <- sub_array_of_indices[chunk_indices(length(sub_array_of_indices), + chunks[[inner_dim]]["chunk"], + chunks[[inner_dim]]["n_chunks"], + inner_dim)] + } else { + tmp <- chunk_indices(length(sub_array_of_indices[[1]]:sub_array_of_indices[[2]]), + chunks[[inner_dim]]["chunk"], chunks[[inner_dim]]["n_chunks"], + inner_dim) + start_pt <- sub_array_of_indices[[1]] + sub_array_of_indices[[1]] <- start_pt + tmp[1] - 1 + sub_array_of_indices[[2]] <- start_pt + tmp[length(tmp)] - 1 + } # The sub_array_of_indices now contains numeric indices of the values to be taken. #Check if all the files have the selectors assigned (e.g., region = 'Grnland') _20191015 @@ -2128,10 +2137,8 @@ print(str(tvi)) if (goes_across_prime_meridian) { #sub_array_of_fri <- 1:n sub_array_of_fri <- c(1:sub_array_of_indices[[2]], sub_array_of_indices[[1]]:n) - } else if (is.list(sub_array_of_indices) & length(sub_array_of_indices) > 1) { + } else if (is.list(sub_array_of_indices)) { sub_array_of_fri <- sub_array_of_indices[[1]]:sub_array_of_indices[[2]] - } else if (is.list(sub_array_of_indices) & length(sub_array_of_indices) == 1) { - sub_array_of_fri <- sub_array_of_indices[[1]] } else { sub_array_of_fri <- sub_array_of_indices } -- GitLab From 75bfef441eee2f5f24ce13b8aa4375d86461c3ff Mon Sep 17 00:00:00 2001 From: aho Date: Fri, 15 Nov 2019 16:34:22 +0100 Subject: [PATCH 5/6] Fix syntax error --- R/Start.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/Start.R b/R/Start.R index ad57b66..eec20ac 100644 --- a/R/Start.R +++ b/R/Start.R @@ -2045,7 +2045,7 @@ print("-> SELECTORS REQUESTED BEFORE TRANSFORM.") if (first_index - beta <= 0 | last_index + beta > n) { sub_array_of_fri <- 1:n .warning(paste0("Adding the parameter transform_extra_cells = ", - transform_extra_cells, " to the transformed index excesses " + transform_extra_cells, " to the transformed index excesses ", "the border. Use the whole index instead.")) } else { sub_array_of_fri <- (first_index - beta):(last_index + beta) -- GitLab From eb38291795ee218787d3080a769bf51965a8fdf7 Mon Sep 17 00:00:00 2001 From: aho Date: Tue, 19 Nov 2019 15:52:54 +0100 Subject: [PATCH 6/6] Small bugfix in SelectorChecker --- R/SelectorChecker.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/SelectorChecker.R b/R/SelectorChecker.R index 1bf2681..141cae2 100644 --- a/R/SelectorChecker.R +++ b/R/SelectorChecker.R @@ -212,7 +212,7 @@ SelectorChecker <- function(selectors, var = NULL, return_indices = TRUE, } else { if (!is.null(var)) { if (is.list(selectors)) { - if (length(selectors != 2)) { + if (length(selectors) != 2) { stop("'selectors' provided in a wrong format.") } else { var[selectors[[1]]:selectors[[2]]] -- GitLab