From 56c1cf8b300de51e70fcea11c1aacc950946864c Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Tue, 13 May 2025 10:59:06 +0200 Subject: [PATCH 01/11] modified ex2_2:14 --- inst/doc/usecase/ex2_10_existing_mask.R | 54 ++++++++++++++++++++++++- inst/doc/usecase/ex2_2_attr.R | 27 +++++++++++++ inst/doc/usecase/ex2_4_two_func.R | 24 +++++++++++ inst/doc/usecase/ex2_6_ext_param_func.R | 24 +++++++++++ inst/doc/usecase/ex2_8_calibration.R | 28 +++++++++++++ 5 files changed, 156 insertions(+), 1 deletion(-) diff --git a/inst/doc/usecase/ex2_10_existing_mask.R b/inst/doc/usecase/ex2_10_existing_mask.R index 8e70712..0f45f41 100644 --- a/inst/doc/usecase/ex2_10_existing_mask.R +++ b/inst/doc/usecase/ex2_10_existing_mask.R @@ -118,6 +118,30 @@ res <- Compute(workflow = wf_mask, ecflow_suite_dir = ecflow_suite_dir, wait = TRUE ) + +#-------------------nord4 with autosubmit--------------------- + res <- Compute(wf_mask, + chunks = list(lat = 2, + lon = 2),#$output1 + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = queue_host, + expid = "a96t", + hpc_user = "bsc979986", + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 2, + job_wallclock = '05:00', + max_jobs = 4, + bidirectional = FALSE, + polling_period = 10, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = "/home/Earth/tkariyat/startR_local_autosubmit/", # Change to yours + autosubmit_server = NULL, + wait = TRUE) #-------- See the result ----------- str(res$output1) @@ -246,13 +270,41 @@ res <- Compute(workflow = wf_mask, job_wallclock = '05:00', max_jobs = 4, bidirectional = FALSE, - polling_period = 10 + polling_period = 10, special_setup = 'nord4', extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') ), ecflow_suite_dir = ecflow_suite_dir, wait = TRUE ) +#-------------------user-defined--------------------- + queue_host = 'nord4' #your own host name for nord4 + temp_dir = '/gpfs/scratch/bsc32/bsc979986/startR_hpc/' + ecflow_suite_dir = '/home/Earth/tkariyat/startR_local/' #your own local directory +##---------------------------------------------------- + res <- Compute(wf_mask, + chunks = list(lat = 2, + lon = 2),#$output1 + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = queue_host, + expid = "a96t", + hpc_user = "bsc979986", + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 2, + job_wallclock = '05:00', + max_jobs = 4, + bidirectional = FALSE, + polling_period = 10, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = "/home/Earth/tkariyat/startR_local_autosubmit/", # Change to yours + autosubmit_server = NULL, + wait = TRUE) + #--------Check the result ------------ str(res$output1) diff --git a/inst/doc/usecase/ex2_2_attr.R b/inst/doc/usecase/ex2_2_attr.R index 2968e93..2b6dc7a 100644 --- a/inst/doc/usecase/ex2_2_attr.R +++ b/inst/doc/usecase/ex2_2_attr.R @@ -72,6 +72,33 @@ library(multiApply) ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) +#------------nord4 with autosubmit------------------------------------------------ + + res2_Nord <- Compute(wf2, + chunks = list(ensemble = 20, + sdate = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = 'nord4', # your own host name for nord4 + expid = "a96t", # your own expid + hpc_user = "bsc979986", # change to yours + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 2, + polling_period = 10, + bidirectional = FALSE, + job_wallclock = '01:00', + max_jobs = 40, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') + ), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = "/home/Earth/tkariyat/startR_local_autosubmit/", # change to yours + autosubmit_server = NULL, + wait = TRUE) + + ## on Power9 #-----------modify according to your personal info--------- queue_host = 'cte-power' #your own host name for power9 diff --git a/inst/doc/usecase/ex2_4_two_func.R b/inst/doc/usecase/ex2_4_two_func.R index e8411bc..2c91bf2 100644 --- a/inst/doc/usecase/ex2_4_two_func.R +++ b/inst/doc/usecase/ex2_4_two_func.R @@ -71,3 +71,27 @@ summary(res4$output1) extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) + +#------------nord 4 with autosubmit------------------------------------------------ + res4_N <- Compute(wf4, + chunks = list(ensemble = 2, + sdate = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = 'nord4', # your own host name for nord4 + expid = "a96t", # your own expid + hpc_user = "bsc979986", # change to yours + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 1, + job_wallclock = '01:00', + max_jobs = 6, + polling_period = 10, + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = "/home/Earth/tkariyat/startR_local_autosubmit/", # change to yours + autosubmit_server = NULL, + wait = TRUE) diff --git a/inst/doc/usecase/ex2_6_ext_param_func.R b/inst/doc/usecase/ex2_6_ext_param_func.R index 63987d5..c2ba560 100644 --- a/inst/doc/usecase/ex2_6_ext_param_func.R +++ b/inst/doc/usecase/ex2_6_ext_param_func.R @@ -131,3 +131,27 @@ res$strat[1:5, 1:2, 1] ecflow_suite_dir = ecflow_suite_dir, #user-specific wait = TRUE) +#--------------nord4 with autosubmit-------------------------------------- + res <- Compute(workflow$strat, + chunks = list(latitude = 2, longitude = 2), + threads_load = 2, + threads_compute = 2, + cluster = list(queue_host = 'nord4', # your own host name for nord4 + expid = "a96t", # your own expid + hpc_user = "bsc979986", # change to yours + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 2, + job_wallclock = '10:00', + max_jobs = 4, + bidirectional = FALSE, + polling_period = 10, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = "/home/Earth/tkariyat/startR_local_autosubmit/", # change to yours + autosubmit_server = NULL, + wait = TRUE) + + diff --git a/inst/doc/usecase/ex2_8_calibration.R b/inst/doc/usecase/ex2_8_calibration.R index 94cc429..50b35e2 100644 --- a/inst/doc/usecase/ex2_8_calibration.R +++ b/inst/doc/usecase/ex2_8_calibration.R @@ -92,6 +92,34 @@ res <- Compute(wf, ecflow_suite_dir = ecflow_suite_dir, wait = T) +#---------------------------------------------------- + queue_host = 'nord4' #your own host name for nord4 + temp_dir = '/gpfs/scratch/bsc32/bsc979986/startR_hpc/' + ecflow_suite_dir = '/home/Earth/tkariyat/startR_local/' #your own local directory +#---------------------------------------------------- + res <- Compute(wf, + chunks = list(latitude = 2, longitude = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = 'nord4', + expid = "a96t", + hpc_user = "bsc979986", + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 2, + extra_queue_params = list("#SBATCH -A bsc32", + "#SBATCH -q bsc_es", + "#SBATCH --constraint=highmem"), + polling_period = 10, + bidirectional = FALSE, + job_wallclock = '01:00', + max_jobs = 4, + special_setup = 'nord4'), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = "/home/Earth/tkariyat/startR_local_autosubmit/", # Change to yours + autosubmit_server = NULL, + wait = TRUE) # Results dim(res$output1) # ensemble sdate dat var time latitude longitude -- GitLab From ca16649f79056067a3420d16d4c6b56851b987d0 Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Fri, 23 May 2025 09:33:03 +0200 Subject: [PATCH 02/11] ex2_6:10 modified --- inst/doc/usecase/ex2_10_existing_mask.R | 15 +++++++-------- inst/doc/usecase/ex2_6_ext_param_func.R | 2 +- inst/doc/usecase/ex2_8_calibration.R | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/inst/doc/usecase/ex2_10_existing_mask.R b/inst/doc/usecase/ex2_10_existing_mask.R index 0f45f41..f9ef241 100644 --- a/inst/doc/usecase/ex2_10_existing_mask.R +++ b/inst/doc/usecase/ex2_10_existing_mask.R @@ -13,15 +13,14 @@ # have matched already. # A) Load mask file -mask_path <- '/esarchive/autosubmit/con_files/land_sea_mask_1x1_c3s.nc' - +mask_path <- "/esarchive/exp/ecmwf/system5c3s/constant/lsm/$var$.nc" mask <- Start(dat = mask_path, - var = 'LSM', + var = 'lsm', lat = 'all', - lon = 'all', - return_vars = list(var_names = NULL, - lat = NULL, lon = NULL), - var_var = 'var_names', + lon = 'all', + synonims = list(lat = c('lat', 'latitude'), + lon = c('lon', 'longitude')), + return_vars = list(lat = NULL, lon = NULL), retrieve = F) @@ -49,7 +48,7 @@ reg_mask <- function(data, mask) { if (mask == 1) { ind <- s2dv::MeanDims(data, c('dat', 'ensemble', 'time')) } else { - ind <- array(rep(NA, dim(data)[2]), dim = c(sdate = dim(data)[2])) + ind <- array(rep(NA, dim(data)[['sdate']]), dim = c(sdate = dim(data)[['sdate']])) } return(ind) diff --git a/inst/doc/usecase/ex2_6_ext_param_func.R b/inst/doc/usecase/ex2_6_ext_param_func.R index c2ba560..79d2dfa 100644 --- a/inst/doc/usecase/ex2_6_ext_param_func.R +++ b/inst/doc/usecase/ex2_6_ext_param_func.R @@ -135,7 +135,7 @@ res$strat[1:5, 1:2, 1] res <- Compute(workflow$strat, chunks = list(latitude = 2, longitude = 2), threads_load = 2, - threads_compute = 2, + threads_compute = 4, cluster = list(queue_host = 'nord4', # your own host name for nord4 expid = "a96t", # your own expid hpc_user = "bsc979986", # change to yours diff --git a/inst/doc/usecase/ex2_8_calibration.R b/inst/doc/usecase/ex2_8_calibration.R index 50b35e2..6a52097 100644 --- a/inst/doc/usecase/ex2_8_calibration.R +++ b/inst/doc/usecase/ex2_8_calibration.R @@ -41,8 +41,8 @@ obs <- Start(dat = '/esarchive/recon/ecmwf/erainterim/monthly_mean/$var$_f6h/$va # Function wrap_cal <- function(obs, exp) { - obs <- s2dv::InsertDim(obs, 1, 1, name = 'ensemble') - #calibrated <- CSTools:::.cal(obs = obs, var_exp = exp) # CSTools version 1.0.1 or earlier + # obs <- s2dv::InsertDim(obs, 1, 1, name = 'ensemble') + # calibrated <- CSTools:::.cal(obs = obs, var_exp = exp) # CSTools version 1.0.1 or earlier calibrated <- CSTools:::.cal(exp = exp, obs = obs, cal.method = "mse_min", eval.method = "leave-one-out", -- GitLab From d8d92013efc96039d18d6e68884a7d0a94e3f0ab Mon Sep 17 00:00:00 2001 From: ARIADNA BATALLA FERRES Date: Tue, 27 May 2025 12:53:20 +0200 Subject: [PATCH 03/11] Update usecase ex2_1_timedim.R --- inst/doc/usecase/ex2_1_timedim.R | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/inst/doc/usecase/ex2_1_timedim.R b/inst/doc/usecase/ex2_1_timedim.R index f9872af..a8da4f7 100644 --- a/inst/doc/usecase/ex2_1_timedim.R +++ b/inst/doc/usecase/ex2_1_timedim.R @@ -77,20 +77,21 @@ library(startR) autosubmit_suite_dir <- "/home/Earth//startR_local_autosubmit/" #------------------------------------------------------------ res <- Compute(wf1, - chunks = list(ensemble = 20, sdate = 2), + chunks = list(ensemble = 2, sdate = 2), threads_load = 2, threads_compute = 4, cluster = list( queue_host = 'nord4', r_module = "R/4.1.2-foss-2019b", - autosubmit_module = 'autosubmit/4.0.0b-foss-2015a-Python-3.7.3', + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', cores_per_job = 2, job_wallclock = '01:00:00', - max_jobs = 40, + max_jobs = 4, polling_period = 10, - extra_queue_params = list('#SBATCH --constraint=medmem', '#SBATCH --exclusive'), + extra_queue_params = list('#SBATCH -N 1'), special_setup = 'nord4', - expid = NULL, + expid = expid, hpc_user = hpc_user ), workflow_manager = 'autosubmit', -- GitLab From caaf2b8919e3b67092b642467a240e3057f2127f Mon Sep 17 00:00:00 2001 From: ARIADNA BATALLA FERRES Date: Tue, 27 May 2025 17:44:43 +0200 Subject: [PATCH 04/11] Update usecase ex2_3_cdo.R --- inst/doc/usecase/ex2_3_cdo.R | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/inst/doc/usecase/ex2_3_cdo.R b/inst/doc/usecase/ex2_3_cdo.R index bc9d1d6..ec7b269 100644 --- a/inst/doc/usecase/ex2_3_cdo.R +++ b/inst/doc/usecase/ex2_3_cdo.R @@ -50,8 +50,8 @@ library(startR) # -------------------------------------------------------------------- -## on Nord4 -#-----------modify according to your personal info--------- +## on Nord4 with ecFlow +#-----------modify according to your personal info----------- queue_host = 'n4login1.bsc.es' temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory @@ -74,3 +74,33 @@ library(startR) extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) + + + ## on Nord4 with Autosubmit +#-----------modify according to your personal info----------- + hpc_user <- "bsc032xxx" + expid <- "xxxx" # autosubmit exp id; can be NULL + autosubmit_suite_dir <- "/home/Earth//startR_local_autosubmit/" +#------------------------------------------------------------ + res3_N <- Compute(wf3, + chunks = list(ensemble = 2, sdate = 2), + threads_compute = 4, + threads_load = 2, + cluster = list( + queue_host = 'nord4', + expid = expid, + hpc_user = hpc_user, + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 2, + job_wallclock = '01:00:00', + max_jobs = 4, + polling_period = 100, + extra_queue_params = list('#SBATCH -N 1'), + special_setup = 'nord4' + ), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = autosubmit_suite_dir, + autosubmit_server = NULL, #'bscesautosubmit01', + wait = TRUE) -- GitLab From 826b3a8c8fd575a77027021e27bdd0eca6029d41 Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Thu, 29 May 2025 15:25:35 +0200 Subject: [PATCH 05/11] update comment ex2_10 --- inst/doc/usecase/ex2_10_existing_mask.R | 1 + 1 file changed, 1 insertion(+) diff --git a/inst/doc/usecase/ex2_10_existing_mask.R b/inst/doc/usecase/ex2_10_existing_mask.R index f9ef241..aec259c 100644 --- a/inst/doc/usecase/ex2_10_existing_mask.R +++ b/inst/doc/usecase/ex2_10_existing_mask.R @@ -157,6 +157,7 @@ s2dv::PlotLayout(PlotEquiMap, c('lat', 'lon'), res$output1[ , 1, , ], #================= Case 2 ================== +# Data is not available for this case; included here for illustrative purposes only. # Goal: Apply analysis only on Baffin Bay and Greenland region. # The mask of the two regions is from the same file, while we have to read them # seperately because the dimensions have to be in line with the data. That is, -- GitLab From 4811b748f0a4fb22246587ede37a3d582eb9652c Mon Sep 17 00:00:00 2001 From: ARIADNA BATALLA FERRES Date: Fri, 30 May 2025 16:43:17 +0200 Subject: [PATCH 06/11] Update usecase ex2_7_seasonal_forecast_verification.R --- .../ex2_7_seasonal_forecast_verification.R | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R b/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R index 3d48bf1..6c66ec8 100644 --- a/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R +++ b/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R @@ -77,7 +77,7 @@ wait = TRUE ) -# Compute() on Nord4 +# Compute() on Nord4 with ecflow res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), @@ -95,6 +95,36 @@ polling_period = 50), ecflow_suite_dir = '/home/Earth/nperez/startR_local/', # your own path wait = TRUE) + +# Compute() on Nord4 wiht Autosubmit + #-----------modify according to your personal info--------- + hpc_user <- "bsc032xxx" + expid <- "xxxx" # autosubmit exp id; can be NULL + autosubmit_suite_dir <- "/home/Earth//startR_local_autosubmit/" + #------------------------------------------------------------ + res <- Compute(wf, + chunks = list(latitude = 2, longitude = 2), + threads_compute = 4, + threads_load = 2, + cluster = list( + queue_host = 'nord4', + expid = expid, + hpc_user = hpc_user, + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 4, + job_wallclock = '00:30', + max_jobs = 4, + special_setup = 'nord4' + ), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = autosubmit_suite_dir, + autosubmit_server = NULL, + wait = TRUE) + + + # Results dim(res$output1) # dat var time latitude longitude -- GitLab From 9dd5b838fb4cc18b41884afaa0a18e6f1551feff Mon Sep 17 00:00:00 2001 From: ARIADNA BATALLA FERRES Date: Thu, 12 Jun 2025 15:11:14 +0200 Subject: [PATCH 07/11] Update usecase ex2_9_mask.R --- inst/doc/usecase/ex2_9_mask.R | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/inst/doc/usecase/ex2_9_mask.R b/inst/doc/usecase/ex2_9_mask.R index aca6162..6537551 100644 --- a/inst/doc/usecase/ex2_9_mask.R +++ b/inst/doc/usecase/ex2_9_mask.R @@ -50,6 +50,7 @@ dims_var[[2]] <- dim_lat datanc <- ncvar_def(name = 'mask', units = 'adimensional', dim = dims_var, missval = -99999) +# NOTE: Replace the following path with a path to your own scratch directory. file_nc <- nc_create("/esarchive/scratch/nperez/mask_system5_m1_harvestmonth.nc", datanc) ncvar_put(file_nc, datanc, mask) @@ -75,6 +76,7 @@ data <- Start(dat = repos, retrieve = FALSE) # Read mask: +# NOTE: Replace the following path with the one used to save the mask. path <- '/esarchive/scratch/nperez/$var$_system5_m1_harvestmonth.nc' mask <- Start(dat = path, var = 'mask', @@ -108,6 +110,34 @@ res <- Compute(workflow = wf_mask, chunks = list(latitude = 2, longitude = 2)) +## alternatively compute on Nord4 with Autosubmit +#-----------modify according to your personal info--------- +hpc_user <- "bsc032424" +expid <- "a9e1" +autosubmit_suite_dir <- "/home/Earth/abatalla/startR_local_autosubmit/" +#------------------------------------------------------------ +res <- Compute(wf_mask, + chunks = list(latitude = 2, longitude = 2), + threads_compute = 4, + threads_load = 2, + cluster = list( + queue_host = 'nord4', + expid = expid, + hpc_user = hpc_user, + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 4, + job_wallclock = '01:00:00', + max_jobs = 4, + special_setup = 'nord4' + ), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = autosubmit_suite_dir, + autosubmit_server = NULL, + wait = TRUE) + + ##################################################################### # -------------------------------------------------------------------------------- ##################################################################### -- GitLab From 1f100adb5b81df6fb319eb4eb2333de77ed4af12 Mon Sep 17 00:00:00 2001 From: ARIADNA BATALLA FERRES Date: Fri, 20 Jun 2025 17:09:54 +0200 Subject: [PATCH 08/11] Update ex2_11_two_dat_inconsistent_target_dim.R --- .../ex2_11_two_dat_inconsistent_target_dim.R | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/inst/doc/usecase/ex2_11_two_dat_inconsistent_target_dim.R b/inst/doc/usecase/ex2_11_two_dat_inconsistent_target_dim.R index 69267b0..b236081 100644 --- a/inst/doc/usecase/ex2_11_two_dat_inconsistent_target_dim.R +++ b/inst/doc/usecase/ex2_11_two_dat_inconsistent_target_dim.R @@ -112,14 +112,43 @@ workflow <- AddStep(list(exp = exp, obs = obs), step, lons_exp = lons_exp, lats_exp = lats_exp, lons_obs = lons_obs, lats_obs = lats_obs) +#========= OPTION 1: Compute locally ============ res <- Compute(workflow$ind_exp, chunks = list(var = 1)) + +#========= OPTION 2: Compute ON NORD4 with AUTOSUBMIT ============ +#-----------modify according to your personal info--------- +hpc_user <- "bsc032xxx" +expid <- "xxxx" # autosubmit exp id; can be NULL +autosubmit_suite_dir <- "/home/Earth//startR_local_autosubmit/" +#------------------------------------------------------------ +res <- Compute(workflow$ind_exp, + chunks = list(var = 1), + threads_compute = 4, + threads_load = 2, + cluster = list( + queue_host = 'nord4', + expid = expid, + hpc_user = hpc_user, + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 4, + job_wallclock = '01:00:00', + max_jobs = 4, + special_setup = 'nord4' + ), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = autosubmit_suite_dir, + autosubmit_server = NULL, + wait = TRUE) + + + str(res) #List of 2 # $ ind_exp: num [1:4, 1, 1] 1.195 0.422 -0.6 -1.017 # $ ind_obs: num [1:4, 1, 1] 0.4642 0.0206 0.9123 -1.3971 # ... # ... - - -- GitLab From 6b8b79deb9306eb981f0fe816b9b996e1e425333 Mon Sep 17 00:00:00 2001 From: ARIADNA BATALLA FERRES Date: Wed, 25 Jun 2025 13:07:40 +0200 Subject: [PATCH 09/11] Update ex2_13_irregular_regrid.R --- inst/doc/usecase/ex2_13_irregular_regrid.R | 59 +++++++++++++++++----- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/inst/doc/usecase/ex2_13_irregular_regrid.R b/inst/doc/usecase/ex2_13_irregular_regrid.R index 3fe3fc1..af29ef6 100644 --- a/inst/doc/usecase/ex2_13_irregular_regrid.R +++ b/inst/doc/usecase/ex2_13_irregular_regrid.R @@ -11,7 +11,7 @@ library(startR) library(s2dv) path <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/CMCC-CM2-SR5/', - 'DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/$member$/Omon/$var$/gn/v20200101/', + 'DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/$member$/Omon/$var$/gn/v20210719/', '$var$_*_s$sdate$-$member$_gn_$aux$.nc') data <- Start(dataset = path, @@ -19,26 +19,26 @@ data <- Start(dataset = path, sdate = c('1960', '1961'), aux = 'all', aux_depends = 'sdate', - x = indices(2:361), # remove two indices to avoid white strips - y = indices(2:291), # remove two indices to avoid white strips + i = indices(2:291), # remove two indices to avoid white strips + j = indices(2:361), # remove two indices to avoid white strips time = indices(1:12), member = 'r1i1p1f1', - return_vars = list(nav_lat = NULL, nav_lon = NULL), + return_vars = list(latitude = NULL, longitude = NULL), retrieve = F) attr(data, 'Dimensions') -#dataset var sdate aux x y time member -# 1 1 2 1 360 290 12 1 -dim(attr(data, 'Variables')$common$nav_lon) -# x y +#dataset var sdate aux i j time member +# 1 1 2 1 290 360 12 1 +dim(attr(data, 'Variables')$common$longitude) +# j i #362 292 -dim(attr(data, 'Variables')$common$nav_lat) -# x y +dim(attr(data, 'Variables')$common$latitude) +# j i #362 292 func_regrid <- function(data) { - lons <- attr(data, 'Variables')$common$nav_lon - lats <- attr(data, 'Variables')$common$nav_lat + lons <- attr(data, 'Variables')$common$longitude + lats <- attr(data, 'Variables')$common$latitude data <- s2dv::CDORemap(data, lons[2:361, 2:291], lats[2:361, 2:291], grid = 'r360x180', method = 'bil', crop = FALSE) lons_reg <- data[['lons']] @@ -47,15 +47,48 @@ func_regrid <- function(data) { } step <- Step(fun = func_regrid, - target_dims = list(data = c('x', 'y')), + target_dims = list(data = c('i', 'j')), output_dims = list(data = c('lon', 'lat'), lats = 'lat', lons = 'lon'), use_attributes = list(data = "Variables")) wf <- AddStep(data, step) + + +#========= OPTION 1: Compute locally ============================= res <- Compute(workflow = wf$data, chunks = list(sdate = 2, time = 2)) + +#========= OPTION 2: Compute ON NORD4 with AUTOSUBMIT ============ +#-----------modify according to your personal info----------- +hpc_user <- "bsc032xxx" +expid <- "xxxx" # autosubmit exp id; can be NULL +autosubmit_suite_dir <- "/home/Earth//startR_local_autosubmit/" +#------------------------------------------------------------ +res <- Compute(workflow = wf$data, + chunks = list(sdate = 2, time = 2), + threads_compute = 4, + threads_load = 2, + cluster = list( + queue_host = 'nord4', + expid = expid, + hpc_user = hpc_user, + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 4, + job_wallclock = '01:00:00', + max_jobs = 4, + special_setup = 'nord4' + ), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = autosubmit_suite_dir, + autosubmit_server = NULL, + wait = TRUE) + + + names(res) #[1] "data" "lats" "lons" dim(res$data) -- GitLab From dc374416b71e6d8f69c6be99d211af0302428eba Mon Sep 17 00:00:00 2001 From: ARIADNA BATALLA FERRES Date: Thu, 26 Jun 2025 09:50:30 +0200 Subject: [PATCH 10/11] Update ex3_1_SubseasonalECVHindcast.md --- .../usecase/ex3_1_SubseasonalECVHindcast.md | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md b/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md index 21262ce..ec2f9ef 100644 --- a/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md +++ b/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md @@ -18,7 +18,7 @@ After loading startR package, the paths to the hindcast should be defined, inclu library(startR) ecmwf_path <- paste0('/esarchive/exp/ecmwf/s2s-monthly_ensforhc/', - 'weekly_mean/$var$_f6h/$sdate$/$var$_$syear$.nc') + 'weekly_mean/$var$_f24h/$sdate$/$var$_$syear$.nc') ``` Now, create the sequence of start dates in 2016 following the scheme in this figure: @@ -191,30 +191,34 @@ step <- Step(fun = score_calc, wf <- AddStep(list(exp, obs), step, sdates.seq = sdates.seq, scores = 'all') ``` -Finally, execute the analysis, for instance, in Nord3v2: +Finally, execute the analysis, for instance, in Nord4: ```r #-----------modify according to your personal info--------- - queue_host = 'nord4' #your own host name for Nord4 - temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' - ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory +hpc_user <- "bsc032xxx" +expid <- "xxxx" # autosubmit exp id; can be NULL +autosubmit_suite_dir <- "/home/Earth//startR_local_autosubmit/" #------------------------------------------------------------ result <- Compute(wf, chunks = list(time = 4, longitude = 2, latitude = 2), threads_load = 2, threads_compute = 12, - cluster = list(queue_host = queue_host, - queue_type = 'slurm', - cores_per_job = 12, - temp_dir = temp_dir, - polling_period = 10, - job_wallclock = '03:00:00', - max_jobs = 16, - bidirectional = FALSE, - special_setup = 'nord4', - extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), - ecflow_suite_dir = ecflow_suite_dir, + cluster = list( + queue_host = "nord4", + expid = expid, + hpc_user = hpc_user, + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = "autosubmit/4.1.13-foss-2023b-Python-3.11.5", + cores_per_job = 12, + job_wallclock = "03:00:00", + max_jobs = 16, + special_setup = "nord4" + ), + workflow_manager = "autosubmit", + autosubmit_suite_dir = autosubmit_suite_dir, + autosubmit_server = NULL, wait = TRUE) ``` *Notice that the execution of `Compute` may last for ~2 hours each chunk. Consider set `wait` as false (see [practical guide](https://earth.bsc.es/gitlab/es/startR/-/blob/master/inst/doc/practical_guide.md#collect-and-the-ec-flow-gui)).* -- GitLab From 710ef7e6e361ad57b1f6717f7ec067ef2be28532 Mon Sep 17 00:00:00 2001 From: ARIADNA BATALLA FERRES Date: Thu, 26 Jun 2025 12:48:59 +0200 Subject: [PATCH 11/11] Update ex2_5_rainFARM.R --- inst/doc/usecase/ex2_5_rainFARM.R | 59 ++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/inst/doc/usecase/ex2_5_rainFARM.R b/inst/doc/usecase/ex2_5_rainFARM.R index 379e81c..8f2006a 100644 --- a/inst/doc/usecase/ex2_5_rainFARM.R +++ b/inst/doc/usecase/ex2_5_rainFARM.R @@ -58,23 +58,22 @@ Chunk_RF <- function(x, nf, destination, startdates, nchunks = chunk_indices) { down_data <- RainFARM(x, lon = lon, lat = lat, drop_realization_dim = TRUE, nf, lon_dim = 'longitude', lat_dim = 'latitude', time_dim = 'time') # detect the dates of forecast time for different start dates - time <- attributes(x)$Variables$common$time + time <- attributes(x)$Variables$common$time dates <- lapply(startdates, function(x) {seq(as.Date(x, format = "%Y-%m-%d"), x + length(time) - 1, 'day')}) - dimname <- names(dim(down_data$data)) - var_dims <- list(ncdim_def(name = 'lon', units = 'degrees', - vals = as.vector(down_data$lon), longname = 'longitude'), - ncdim_def(name = 'lat', units = 'degrees', - vals = as.vector(down_data$lat), longname = 'longitude'), - ncdim_def(name = 'ensemble', units = 'adim', - vals = 1 : dim(down_data$data)[which(dimname == 'member')], - longname = 'ensemble', create_dimvar = TRUE)) metadata_var <- list(units = 'm s-1', cdo_grid_name = paste0('r', length(lon), 'x', length(lat)), projection = 'none') + coords <- list(longitude = array(down_data$longitude, + dim = c(longitude = length(down_data$longitude))), + latitude = array(down_data$latitude, + dim = c(latitude = length(down_data$latitude))), + member = array(as.numeric(1:dim(down_data$data)[["member"]]), + dim = c(member = dim(down_data$data)[["member"]]))) + # startdates and dates depends on the chunk: - CSTools:::.saveExp(down_data$data, startdates = startdates[nchunks['sdate']], - dates = dates[[nchunks['sdate']]], defined_dims = var_dims, + CSTools:::.saveexp(down_data$data, startdates = startdates[nchunks['sdate']], + dates = dates[[nchunks['sdate']]], coords = coords, varname = 'prlr', metadata_var = metadata_var, destination = destination) down_data_mean <- s2dv::MeanDims(down_data$data, c('member', 'longitude', 'latitude')) @@ -87,6 +86,9 @@ step <- Step(Chunk_RF, use_libraries = c('CSTools', 'ncdf4'), use_attributes = list(data = "Variables")) +# NOTE: the directory defined in destination must be created beforehand +if (!dir.exists("./test_RF_start")) dir.create("./test_RF_start") + workflow <- AddStep(list(data = data), step, nf = 4, destination = "./test_RF_start/", startdates = as.Date(sdates, format = "%Y%m%d")) @@ -101,12 +103,12 @@ res <- Compute(workflow, #========= OPTION 2: Compute ON NORD4 ============ -#-----------modify according to your personal info--------- +## on Nord4 with ecFlow +#-----------modify according to your personal info----------- queue_host = 'nord4' # your own host name for nord4 temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' # your own local directory #------------------------------------------------------------ - res <- Compute(workflow, chunks = list(sdate = 4), threads_load = 1, @@ -125,5 +127,36 @@ res <- Compute(workflow, ecflow_suite_dir = ecflow_suite_dir, wait = TRUE) + +## on Nord4 with Autosubmit +#-----------modify according to your personal info----------- +hpc_user <- "bsc032xxx" +expid <- "xxxx" # autosubmit exp id; can be NULL +autosubmit_suite_dir <- "/home/Earth//startR_local_autosubmit/" +#------------------------------------------------------------ +res <- Compute(workflow, + chunks = list(sdate = 4), + threads_load = 1, + threads_compute = 1, + cluster = list( + queue_host = 'nord4', + expid = expid, + hpc_user = hpc_user, + r_module = "R/4.1.2-foss-2019b", + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.1.13-foss-2023b-Python-3.11.5', + cores_per_job = 16, + job_wallclock = '01:00:00', + max_jobs = 4, + extra_queue_params = list('#SBATCH -N 1'), + special_setup = 'nord4' + ), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = autosubmit_suite_dir, + autosubmit_server = NULL, + wait = TRUE) +#------------------------------------------------------------ + + # Visualize the temporal evolution of the result simultaneously for all sdates: matplot(1:215, res$output1[, 1, 1, ], type = "l") -- GitLab