From 6da1ccdfb8f52f981e7585d63682905ffc54e023 Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 22 Nov 2024 16:32:15 +0100 Subject: [PATCH 01/19] Add MN5 and CTE-AMD to platforms.yml --- inst/chunking/Autosubmit/platforms.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/inst/chunking/Autosubmit/platforms.yml b/inst/chunking/Autosubmit/platforms.yml index f8d8f70..b0ce9bc 100644 --- a/inst/chunking/Autosubmit/platforms.yml +++ b/inst/chunking/Autosubmit/platforms.yml @@ -12,3 +12,23 @@ Platforms: CUSTOM_DIRECTIVES: # "['#SBATCH --exclusive']" "['#SBATCH --constraint=medmem']" # MAX_WALLCLOCK: '48:00' #Q: ARE THESE SETTING CORRECT? + mn5: + TYPE: slurm + HOST: glogin1.bsc.es + PROJECT: bsc32 + USER: + SERIAL_QUEUE: gp_debug + QUEUE: gp_bsces + SCRATCH_DIR: /gpfs/scratch/ + TEMP_DIR: '' + ADD_PROJECT_TO_HOST: "false" + CUSTOM_DIRECTIVES: + amd: + TYPE: slurm + HOST: amdlogin1.bsc.es + PROJECT: bsc32 + USER: + SCRATCH_DIR: /gpfs/scratch/ + TEMP_DIR: '' + ADD_PROJECT_TO_HOST: "false" + CUSTOM_DIRECTIVES: -- GitLab From 99422a93025aee7d99de15c54acb04bd86603c60 Mon Sep 17 00:00:00 2001 From: vagudets Date: Mon, 25 Nov 2024 16:09:45 +0100 Subject: [PATCH 02/19] Add checks and script lines for conda_env in ByChunks() (WIP) --- R/ByChunks_autosubmit.R | 31 ++++++++++++++++++++++--------- R/ByChunks_ecflow.R | 28 +++++++++++++++++++++++----- R/Utils.R | 14 ++++++++++---- 3 files changed, 55 insertions(+), 18 deletions(-) diff --git a/R/ByChunks_autosubmit.R b/R/ByChunks_autosubmit.R index ec33632..e66ec20 100644 --- a/R/ByChunks_autosubmit.R +++ b/R/ByChunks_autosubmit.R @@ -199,6 +199,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', r_module = 'R', CDO_module = NULL, autosubmit_module = 'autosubmit', + conda_env = NULL, node_memory = NULL, # not used cores_per_job = NULL, job_wallclock = '01:00:00', @@ -216,7 +217,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', if (any(!(names(cluster) %in% c('queue_host', 'queue_type', 'data_dir', 'temp_dir', 'lib_dir', 'init_commands', 'r_module', 'CDO_module', 'autosubmit_module', - 'ecflow_module', 'node_memory', + 'ecflow_module', 'conda_env', 'node_memory', 'cores_per_job', 'job_wallclock', 'max_jobs', 'extra_queue_params', 'bidirectional', 'polling_period', 'special_setup', 'expid', 'hpc_user', @@ -238,7 +239,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', ### queue_host support_hpcs <- c('local', 'nord3') # names in platforms.yml if (is.null(cluster$queue_host) || !cluster$queue_host %in% support_hpcs) { - stop("Cluster component 'queue_host' must be one of the follows: ", + stop("Cluster component 'queue_host' must be one of the following: ", paste(support_hpcs, collapse = ','), '.') } @@ -266,12 +267,14 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', "character strings.") } ### r_module - if (!is.character(cluster[['r_module']])) { - stop("The component 'r_module' of the parameter 'cluster' must be a character string.") - } - if ((nchar(cluster[['r_module']]) < 1) || (grepl(' ', cluster[['r_module']]))) { - stop("The component 'r_module' of the parameter 'cluster' must have at least one character ", - "and contain no blank spaces.") + if (!is.null(cluster[['r_module']])) { + if (!is.character(cluster[['r_module']])) { + stop("The component 'r_module' of the parameter 'cluster' must be a character string.") + } + if ((nchar(cluster[['r_module']]) < 1) || (grepl(' ', cluster[['r_module']]))) { + stop("The component 'r_module' of the parameter 'cluster' must have at least one character ", + "and contain no blank spaces.") + } } ### CDO_module if (!is.null(cluster[['CDO_module']])) { @@ -279,11 +282,21 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', stop("The component 'CDO_module' of the parameter 'cluster' must be a character string.") } if (nchar(cluster[['CDO_module']]) < 1 || grepl(' ', cluster[['CDO_module']])) { - warning("The component 'CDO_module' of parameter 'cluster' must have ", + warning("The component 'CDO_module' of parameter 'cluster' is longer ", " than 1 and only the first element will be used.") } cluster[['r_module']] <- paste(cluster[['r_module']], cluster[['CDO_module']]) } + ### conda environment + if (!is.null(cluster[['conda_env']])) { + if (!is.character(cluster[['conda_env']])) { + stop("The component 'conda_env' of the parameter 'cluster' must be a character string.") + } + if (nchar(cluster[['conda_env']]) < 1 || grepl(' ', cluster[['conda_env']])) { + warning("The component 'conda_env' of parameter 'cluster' must have at ", + "least one character and contain no blank spaces.") + } + } ### autosubmit_module if (!is.character(cluster[['autosubmit_module']])) { stop("The component 'autosubmit_module' of the parameter 'cluster' must be a character string.") diff --git a/R/ByChunks_ecflow.R b/R/ByChunks_ecflow.R index 6292448..1334b01 100644 --- a/R/ByChunks_ecflow.R +++ b/R/ByChunks_ecflow.R @@ -166,6 +166,7 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', init_commands = list(''), r_module = 'R', CDO_module = NULL, + conda_env = NULL, ecflow_module = 'ecFlow', node_memory = NULL, cores_per_job = NULL, @@ -184,8 +185,8 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', } if (any(!(names(cluster) %in% c('queue_host', 'queue_type', 'data_dir', 'temp_dir', 'lib_dir', 'init_commands', - 'r_module', 'CDO_module', 'autosubmit_module', - 'ecflow_module', 'node_memory', + 'r_module', 'CDO_module', 'conda_env', + 'autosubmit_module', 'ecflow_module', 'node_memory', 'cores_per_job', 'job_wallclock', 'max_jobs', 'extra_queue_params', 'bidirectional', 'polling_period', 'special_setup', 'expid', 'hpc_user')))) { @@ -272,6 +273,16 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', } cluster[['r_module']] <- paste(cluster[['r_module']], cluster[['CDO_module']]) } + ### conda environment + if (!is.null(cluster[['conda_env']])) { + if (!is.character(cluster[['conda_env']])) { + stop("The component 'conda_env' of the parameter 'cluster' must be a character string.") + } + if (nchar(cluster[['conda_env']]) < 1 || grepl(' ', cluster[['conda_env']])) { + warning("The component 'conda_env' of parameter 'cluster' must have at ", + "least one character and contain no blank spaces.") + } + } if (!is.character(cluster[['ecflow_module']])) { stop("The component 'ecflow_module' of the parameter 'cluster' must be a character string.") } @@ -508,9 +519,16 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', chunk_ecf_script_lines <- gsub('^include_init_commands', paste0(paste0(cluster[['init_commands']], collapse = '\n'), '\n'), chunk_ecf_script_lines) - chunk_ecf_script_lines <- gsub('^include_module_load', - paste0('module load ', cluster[['r_module']]), - chunk_ecf_script_lines) + if ('conda_env' %in% names(cluster)) { + chunk_ecf_script_lines <- gsub('^include_module_load', + paste0('eval "$(conda shell.bash hook)" \n', + 'conda activate ', cluster[['conda_env']]), + chunk_ecf_script_lines) + } else { + chunk_ecf_script_lines <- gsub('^include_module_load', + paste0('module load ', cluster[['r_module']]), + chunk_ecf_script_lines) + } ecf_vars <- paste0('%', as.vector(sapply(chunked_dims, function(x) { c(toupper(x), paste0(toupper(x), '_N')) diff --git a/R/Utils.R b/R/Utils.R index e440dde..2e4fb21 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -896,12 +896,18 @@ write_autosubmit_bash <- function(chunks, cluster, autosubmit_suite_dir) { # Include init commands bash_script_lines <- gsub('^include_init_commands', paste0(paste0(cluster[['init_commands']], collapse = '\n'), '\n'), - bash_script_lines) # Rewrite include_module_load - bash_script_lines <- gsub('^include_module_load', - paste0('module load ', cluster[['r_module']]), - bash_script_lines) + if ('conda_env' %in% names(cluster)) { + bash_script_lines <- gsub('^include_module_load', + paste0('eval "$(conda shell.bash hook)" \n', + 'conda activate ', cluster[['conda_env']]), + bash_script_lines) + } else { + bash_script_lines <- gsub('^include_module_load', + paste0('module load ', cluster[['r_module']]), + bash_script_lines) + } # Rewrite cd run_dir # If run_dir is not specified, the script will run under ${proj_dir} if (!is.null(cluster[['run_dir']])) { -- GitLab From edb2b8d851b9c0168faf181c278d137b8788f9be Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 29 Nov 2024 16:50:17 +0100 Subject: [PATCH 03/19] Remove hard-coded version --- inst/chunking/Autosubmit/autosubmit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/chunking/Autosubmit/autosubmit.yml b/inst/chunking/Autosubmit/autosubmit.yml index 8b129a0..7693d65 100644 --- a/inst/chunking/Autosubmit/autosubmit.yml +++ b/inst/chunking/Autosubmit/autosubmit.yml @@ -1,5 +1,5 @@ config: - AUTOSUBMIT_VERSION: 4.0.0b0 + AUTOSUBMIT_VERSION: MAXWAITINGJOBS: # Should it be the total amount of chunk? TOTALJOBS: SAFETYSLEEPTIME: 10 -- GitLab From 6adfe5d44e5a593754148a44aceecdfa85ab3548 Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 29 Nov 2024 16:52:05 +0100 Subject: [PATCH 04/19] Add conva environment + MN5/CTE-AMD support (WIP) --- R/ByChunks_autosubmit.R | 41 +++++++++++++++++++++++++++-------------- R/ByChunks_ecflow.R | 20 +++++++++++++++++--- R/Utils.R | 9 +++++++-- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/R/ByChunks_autosubmit.R b/R/ByChunks_autosubmit.R index e66ec20..430f55a 100644 --- a/R/ByChunks_autosubmit.R +++ b/R/ByChunks_autosubmit.R @@ -237,7 +237,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', cluster <- default_cluster ### queue_host - support_hpcs <- c('local', 'nord3') # names in platforms.yml + support_hpcs <- c('local', 'nord3', 'mn5', 'cte-amd') # names in platforms.yml if (is.null(cluster$queue_host) || !cluster$queue_host %in% support_hpcs) { stop("Cluster component 'queue_host' must be one of the following: ", paste(support_hpcs, collapse = ','), '.') @@ -296,6 +296,8 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', warning("The component 'conda_env' of parameter 'cluster' must have at ", "least one character and contain no blank spaces.") } + } else { + cluster$conda_env <- NULL } ### autosubmit_module if (!is.character(cluster[['autosubmit_module']])) { @@ -335,7 +337,11 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', } cluster[['polling_period']] <- round(cluster[['polling_period']]) ### special_setup - if (!(cluster[['special_setup']] %in% c('none', 'marenostrum4'))) { + SPECIAL_SETUP_MACHINES <- c('marenostrum5', 'cte-amd') + if (is.null(cluster[['special_setup']])) { + cluster[['special_setup']] <- 'none' + } + if (!(cluster[['special_setup']] %in% c('none', SPECIAL_SETUP_MACHINES))) { stop("The value provided for the component 'special_setup' of the parameter ", "'cluster' is not recognized.") } @@ -494,7 +500,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', chunk_script_lines) deparsed_calls <- paste0('start_calls <- list(') extra_path <- '' - if (cluster[['special_setup']] == 'marenostrum4') { + if (cluster[['special_setup']] %in% SPECIAL_SETUP_MACHINES) { extra_path <- '/gpfs/archive/bsc32/' } for (cube_header in 1:length(cube_headers)) { @@ -582,19 +588,26 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', #which_files_exist <- sapply(files_to_check, file.exists) which_files_exist <- sapply(files_to_send, file.exists) files_to_send <- files_to_send[which_files_exist] - if (cluster[['special_setup']] == 'marenostrum4') { + if (cluster[['special_setup']] %in% SPECIAL_SETUP_MACHINES) { file_spec <- paste(paste0("/gpfs/archive/bsc32/", files_to_send), collapse = ' ') - system(paste0("ssh ", cluster[['queue_host']], " 'mkdir -p ", remote_data_dir, - ' ; module load transfer ; cd ', remote_autosubmit_suite_dir_suite, - ' ; dtrsync -Rrav ', '\'', file_spec, '\' "', remote_data_dir, '/"', - " ; sleep 1 ; ", - "while [[ ! $(ls dtrsync_*.out 2>/dev/null | wc -l) -ge 1 ]] ; ", - "do sleep 2 ; done", - " ; sleep 1 ; ", - 'while [[ ! $(grep "total size is" dtrsync_*.out | ', - "wc -l) -ge 1 ]] ; ", - "do sleep 5 ; done", "'")) + ## TODO: Set up user dynamically + system(paste0("ssh bsc032762@transfer1.bsc.es 'mkdir -p remote_data_dir,", + ' ; rsync -Rrav ', '\'', file_spec, '\' "', remote_data_dir, '/"', + " ; sleep 10 '")) + + # ## TODO: + # ## This bit most likely needs to be changed + # system(paste0("ssh ", cluster[['queue_host']], " 'mkdir -p ", remote_data_dir, + # ' ; module load transfer ; cd ', remote_autosubmit_suite_dir_suite, + # ' ; dtrsync -Rrav ', '\'', file_spec, '\' "', remote_data_dir, '/"', + # " ; sleep 1 ; ", + # "while [[ ! $(ls dtrsync_*.out 2>/dev/null | wc -l) -ge 1 ]] ; ", + # "do sleep 2 ; done", + # " ; sleep 1 ; ", + # 'while [[ ! $(grep "total size is" dtrsync_*.out | ', + # "wc -l) -ge 1 ]] ; ", + # "do sleep 5 ; done", "'")) } else { file_spec <- paste(files_to_send, collapse = ' :') system(paste0("ssh ", cluster[['queue_host']], ' "mkdir -p ', diff --git a/R/ByChunks_ecflow.R b/R/ByChunks_ecflow.R index 1334b01..95a0a9c 100644 --- a/R/ByChunks_ecflow.R +++ b/R/ByChunks_ecflow.R @@ -274,6 +274,18 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', cluster[['r_module']] <- paste(cluster[['r_module']], cluster[['CDO_module']]) } ### conda environment + if (!is.null(cluster[['conda_env']])) { + if (!is.character(cluster[['conda_env']])) { + stop("The component 'conda_env' of the parameter 'cluster' must be a character string.") + } + if (nchar(cluster[['conda_env']]) < 1 || grepl(' ', cluster[['conda_env']])) { + warning("The component 'conda_env' of parameter 'cluster' must have at ", + "least one character and contain no blank spaces.") + } + } else { + cluster$conda_env <- NULL + } + if (!is.null(cluster[['conda_env']])) { if (!is.character(cluster[['conda_env']])) { stop("The component 'conda_env' of the parameter 'cluster' must be a character string.") @@ -521,8 +533,10 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', chunk_ecf_script_lines) if ('conda_env' %in% names(cluster)) { chunk_ecf_script_lines <- gsub('^include_module_load', - paste0('eval "$(conda shell.bash hook)" \n', - 'conda activate ', cluster[['conda_env']]), + paste0('set +eu \n', + 'source /gpfs/projects/bsc32/software/suselinux/11/software/Miniconda3/4.7.10/etc/profile.d/conda.sh \n', + 'conda activate ', cluster[['conda_env']], '\n', + 'set -eu'), chunk_ecf_script_lines) } else { chunk_ecf_script_lines <- gsub('^include_module_load', @@ -657,7 +671,7 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', suite <- add_line(suite, paste0('edit ', toupper(chunked_dims[i]), ' ', 1), tabs) suite <- add_line(suite, paste0('edit ', toupper(chunked_dims[i]), '_N ', chunks[[chunked_dims[i]]]), tabs) } - + # Iterate through chunks chunk_array <- array(1:prod(unlist(chunks)), dim = (unlist(chunks))) arrays_of_results <- vector('list', length(attr(step_fun, 'OutputDims'))) diff --git a/R/Utils.R b/R/Utils.R index 2e4fb21..bf3286b 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -900,8 +900,10 @@ write_autosubmit_bash <- function(chunks, cluster, autosubmit_suite_dir) { # Rewrite include_module_load if ('conda_env' %in% names(cluster)) { bash_script_lines <- gsub('^include_module_load', - paste0('eval "$(conda shell.bash hook)" \n', - 'conda activate ', cluster[['conda_env']]), + paste0('set +eu \n', + 'source /gpfs/projects/bsc32/software/suselinux/11/software/Miniconda3/4.7.10/etc/profile.d/conda.sh \n', + 'conda activate ', cluster[['conda_env']], '\n', + 'set -eu'), bash_script_lines) } else { bash_script_lines <- gsub('^include_module_load', @@ -950,6 +952,9 @@ write_autosubmit_confs <- function(chunks, cluster, autosubmit_suite_dir) { if (conf_type == "autosubmit") { #Q: Should it be the total amount of chunk? + ## TODO: Fix this; it is not generalizable + conf$config$AUTOSUBMIT_VERSION <- stringr::str_extract(cluster['autosubmit_module'], + "(?<=/)(.+)(?=\\-foss)") conf$config$MAXWAITINGJOBS <- as.integer(prod(unlist(chunks))) # total amount of chunk #NOTE: Nord3 max. amount of queued jobs is 366 if (conf$config$MAXWAITINGJOBS > 366) conf$config$MAXWAITINGJOBS <- 366 -- GitLab From a2cf24386bdbb54d2fe4ddf1c58097daef234e9e Mon Sep 17 00:00:00 2001 From: vagudets Date: Tue, 4 Mar 2025 10:47:01 +0100 Subject: [PATCH 05/19] Add Nord4 as platform --- inst/chunking/Autosubmit/platforms.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/inst/chunking/Autosubmit/platforms.yml b/inst/chunking/Autosubmit/platforms.yml index b0ce9bc..44d5f6a 100644 --- a/inst/chunking/Autosubmit/platforms.yml +++ b/inst/chunking/Autosubmit/platforms.yml @@ -31,4 +31,13 @@ Platforms: SCRATCH_DIR: /gpfs/scratch/ TEMP_DIR: '' ADD_PROJECT_TO_HOST: "false" + CUSTOM_DIRECTIVES: + nord4: + TYPE: SLURM + HOST: n4login1.bsc.es + PROJECT: bsc32 + ADD_PROJECT_TO_HOST: "false" + USER: + SCRATCH_DIR: /gpfs/scratch/ + QUEUE: bsc_es CUSTOM_DIRECTIVES: -- GitLab From 662a7ce4c9870d69b821f8f35614b80c086f1b0c Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 5 Mar 2025 12:14:19 +0100 Subject: [PATCH 06/19] Add special_setup nord4 to cluster parameters --- R/ByChunks_autosubmit.R | 48 +++++++++++++++++++++-------------------- R/ByChunks_ecflow.R | 23 +++++++++++++++++++- R/Utils.R | 25 ++++++++++++++++++++- 3 files changed, 71 insertions(+), 25 deletions(-) diff --git a/R/ByChunks_autosubmit.R b/R/ByChunks_autosubmit.R index 75ffd25..5f1544f 100644 --- a/R/ByChunks_autosubmit.R +++ b/R/ByChunks_autosubmit.R @@ -236,7 +236,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', cluster <- default_cluster ### queue_host - support_hpcs <- c('local', 'nord3', 'mn5', 'cte-amd') # names in platforms.yml + support_hpcs <- c('local', 'nord3', 'mn5', 'cte-amd', 'nord4') # names in platforms.yml if (is.null(cluster$queue_host) || !cluster$queue_host %in% support_hpcs) { stop("Cluster component 'queue_host' must be one of the following: ", paste(support_hpcs, collapse = ','), '.') @@ -295,8 +295,12 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', warning("The component 'conda_env' of parameter 'cluster' must have at ", "least one character and contain no blank spaces.") } + if (!is.null(cluster[['r_module']])) { + warning("Both 'conda_env' and 'r_module' cluster components have been ", + "specified. 'conda_env' will be used.") + } } else { - cluster$conda_env <- NULL + cluster[['conda_env']] <- NULL } ### autosubmit_module if (!is.character(cluster[['autosubmit_module']])) { @@ -346,12 +350,10 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', stop("The component 'polling_period' of the parameter 'cluster' must be numeric.") } cluster[['polling_period']] <- round(cluster[['polling_period']]) - ### special_setup - SPECIAL_SETUP_MACHINES <- c('marenostrum5', 'cte-amd') if (is.null(cluster[['special_setup']])) { cluster[['special_setup']] <- 'none' } - if (!(cluster[['special_setup']] %in% c('none', SPECIAL_SETUP_MACHINES))) { + if (!(cluster[['special_setup']] %in% c('none', 'marenostrum4', 'nord4'))) { stop("The value provided for the component 'special_setup' of the parameter ", "'cluster' is not recognized.") } @@ -514,7 +516,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', chunk_script_lines) deparsed_calls <- paste0('start_calls <- list(') extra_path <- '' - if (cluster[['special_setup']] %in% SPECIAL_SETUP_MACHINES) { + if (cluster[['special_setup']] == 'marenostrum4') { extra_path <- '/gpfs/archive/bsc32/' } for (cube_header in 1:length(cube_headers)) { @@ -602,26 +604,26 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', #which_files_exist <- sapply(files_to_check, file.exists) which_files_exist <- sapply(files_to_send, file.exists) files_to_send <- files_to_send[which_files_exist] - if (cluster[['special_setup']] %in% SPECIAL_SETUP_MACHINES) { + if (cluster[['special_setup']] == 'marenostrum4') { file_spec <- paste(paste0("/gpfs/archive/bsc32/", files_to_send), collapse = ' ') - ## TODO: Set up user dynamically - system(paste0("ssh bsc032762@transfer1.bsc.es 'mkdir -p remote_data_dir,", - ' ; rsync -Rrav ', '\'', file_spec, '\' "', remote_data_dir, '/"', - " ; sleep 10 '")) + ## TODO: Use transfer machine? + # system(paste0("ssh bsc032762@transfer1.bsc.es 'mkdir -p remote_data_dir,", + # ' ; rsync -Rrav ', '\'', file_spec, '\' "', remote_data_dir, '/"', + # " ; sleep 10 '")) - # ## TODO: - # ## This bit most likely needs to be changed - # system(paste0("ssh ", cluster[['queue_host']], " 'mkdir -p ", remote_data_dir, - # ' ; module load transfer ; cd ', remote_autosubmit_suite_dir_suite, - # ' ; dtrsync -Rrav ', '\'', file_spec, '\' "', remote_data_dir, '/"', - # " ; sleep 1 ; ", - # "while [[ ! $(ls dtrsync_*.out 2>/dev/null | wc -l) -ge 1 ]] ; ", - # "do sleep 2 ; done", - # " ; sleep 1 ; ", - # 'while [[ ! $(grep "total size is" dtrsync_*.out | ', - # "wc -l) -ge 1 ]] ; ", - # "do sleep 5 ; done", "'")) + ## TODO: + ## This bit may need to be changed + system(paste0("ssh ", cluster[['queue_host']], " 'mkdir -p ", remote_data_dir, + ' ; module load transfer ; cd ', remote_autosubmit_suite_dir_suite, + ' ; dtrsync -Rrav ', '\'', file_spec, '\' "', remote_data_dir, '/"', + " ; sleep 1 ; ", + "while [[ ! $(ls dtrsync_*.out 2>/dev/null | wc -l) -ge 1 ]] ; ", + "do sleep 2 ; done", + " ; sleep 1 ; ", + 'while [[ ! $(grep "total size is" dtrsync_*.out | ', + "wc -l) -ge 1 ]] ; ", + "do sleep 5 ; done", "'")) } else { file_spec <- paste(files_to_send, collapse = ' :') system(paste0("ssh ", cluster[['queue_host']], ' "mkdir -p ', diff --git a/R/ByChunks_ecflow.R b/R/ByChunks_ecflow.R index 95a0a9c..4c5c592 100644 --- a/R/ByChunks_ecflow.R +++ b/R/ByChunks_ecflow.R @@ -318,7 +318,7 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', stop("The component 'extra_queue_params' of the parameter 'cluster' must be a list of ", "character strings.") } - if (!(cluster[['special_setup']] %in% c('none', 'marenostrum4'))) { + if (!(cluster[['special_setup']] %in% c('none', 'marenostrum4', 'nord4'))) { stop("The value provided for the component 'special_setup' of the parameter ", "'cluster' is not recognized.") } @@ -528,6 +528,22 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', paste0('%include "./', cluster[['queue_type']], '.h"'), chunk_ecf_script_lines) } + # Special setup: Nord4 singularity wrapper + if (cluster[['special_setup']] == "nord4") { + nord4_commands <- list(paste0("module load nord3-singu \n", + "module load bsc/current \n", + "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", + "unset PYTHONSTARTUP \n", + "alias cdo='nord3_singu_es cdo' \n", + "alias Rscript='nord3_singu_es Rscript' \n", + "alias ecflow_client='nord3_singu_es ecflow_client' \n", + "shopt -s expand_aliases \n", + "include_init_commands")) + chunk_ecf_script_lines <- gsub('^include_init_commands', + paste0(paste0(nord4_commands, collapse = '\n'), '\n'), + chunk_ecf_script_lines) + } + # Init commands chunk_ecf_script_lines <- gsub('^include_init_commands', paste0(paste0(cluster[['init_commands']], collapse = '\n'), '\n'), chunk_ecf_script_lines) @@ -599,6 +615,11 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', writeLines(chunk_queue_header_lines, paste0(ecflow_suite_dir_suite, '/', cluster[['queue_type']], '.h')) # Copy headers + ## TODO: remove + # file.copy("/esarchive/scratch/vagudets/repos/startR/inst/chunking/ecFlow/head.h", + # to = ecflow_suite_dir_suite) + # file.copy("/esarchive/scratch/vagudets/repos/startR/inst/chunking/ecFlow/tail.h", + # to = ecflow_suite_dir_suite) file.copy(system.file('chunking/ecFlow/head.h', package = 'startR'), ecflow_suite_dir_suite) file.copy(system.file('chunking/ecFlow/tail.h', package = 'startR'), diff --git a/R/Utils.R b/R/Utils.R index 38ff387..e4ffe5c 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -893,7 +893,22 @@ write_autosubmit_bash <- function(chunks, cluster, autosubmit_suite_dir) { # Rewrite chunk_args= bash_script_lines <- gsub('^chunk_args=*', paste0('chunk_args=', chunk_args), bash_script_lines) - # Include init commands + # Include Nord4 Singularity wrapper special setup + if (cluster[['special_setup']] == "nord4") { + nord4_commands <- list(paste0("module load nord3-singu \n", + "module load bsc/current \n", + "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", + "unset PYTHONSTARTUP \n", + "alias cdo='nord3_singu_es cdo' \n", + "alias Rscript='nord3_singu_es Rscript' \n", + "alias ecflow_client='nord3_singu_es ecflow_client' \n", + "shopt -s expand_aliases \n", + "include_init_commands")) + bash_script_lines <- gsub('^include_init_commands', + paste0(paste0(nord4_commands, collapse = '\n'), '\n'), + bash_script_lines) + } + # Include init commands bash_script_lines <- gsub('^include_init_commands', paste0(paste0(cluster[['init_commands']], collapse = '\n'), '\n'), bash_script_lines) @@ -908,6 +923,12 @@ write_autosubmit_bash <- function(chunks, cluster, autosubmit_suite_dir) { } else { bash_script_lines <- gsub('^include_module_load', paste0('module load ', cluster[['r_module']]), + paste0('if [ $BSC_MACHINE == "nord4" ]; then \n', + ' unalias Rscript 2>/dev/null \n', + ' unalias cdo 2>/dev/null \n', + ' alias cdo="nord3_singu cdo" \n', + ' alias Rscript="nord3_singu Rscript" \n', + 'fi'), bash_script_lines) } # Rewrite cd run_dir @@ -940,6 +961,8 @@ write_autosubmit_confs <- function(chunks, cluster, autosubmit_suite_dir) { # "autosubmit_suite_dir" should be the local folder that has R script, like ecflow_suite_dir in Compute() # Get config template files from package + ## TODO: Change + # template_dir <- "/esarchive/scratch/vagudets/repos/startR/inst/chunking/Autosubmit/" template_dir <- system.file('chunking/Autosubmit/', package = 'startR') config_files <- list.files(template_dir, pattern = "*\\.yml$") -- GitLab From d3c1399ae2c2abf997f1a19695c28a1ecc51b0b3 Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 5 Mar 2025 13:13:58 +0100 Subject: [PATCH 07/19] Update practical guide with Nord4 informatioN --- inst/doc/practical_guide.md | 89 ++++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 17 deletions(-) diff --git a/inst/doc/practical_guide.md b/inst/doc/practical_guide.md index 637474e..11c2338 100644 --- a/inst/doc/practical_guide.md +++ b/inst/doc/practical_guide.md @@ -589,10 +589,10 @@ You will need to add two parameters to your `Compute()` call: `cluster` and `ecf The parameter `ecflow_suite_dir` expects a path to a folder in the workstation where to store temporary files generated for the automatic management of the workflow. As you will see later, the EC-Flow workflow manager is used transparently for this purpose. -The parameter `cluster` expects a list with a number of components that will have to be provided a bit differently depending on the HPC you want to run on. You can see next an example cluster configuration that will execute the previously defined workflow on Nord3-v2. +The parameter `cluster` expects a list with a number of components that will have to be provided a bit differently depending on the HPC you want to run on. You can see next an example cluster configuration that will execute the previously defined workflow on Nord4. ```r # user-defined - temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' + temp_dir <- '/gpfs/scratch/bsc32/bsc032734/startR_hpc/' ecflow_suite_dir <- '/home/Earth/aho/startR_local/' res <- Compute(wf, @@ -600,15 +600,16 @@ The parameter `cluster` expects a list with a number of components that will hav longitude = 2), threads_load = 2, threads_compute = 4, - cluster = list(queue_host = 'nord4.bsc.es', + cluster = list(queue_host = 'n4login1.bsc.es', queue_type = 'slurm', temp_dir = temp_dir, r_module = 'R/4.1.2-foss-2019b' cores_per_job = 4, job_wallclock = '00:10:00', max_jobs = 4, - extra_queue_params = list('#SBATCH --constraint=medmem'), + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es'), bidirectional = FALSE, + special_setup = 'nord4', polling_period = 10 ), ecflow_suite_dir = ecflow_suite_dir, @@ -622,13 +623,15 @@ The cluster components and options are explained next: - `temp_dir`: directory on the HPC where to store temporary files. Must be accessible from the HPC login node and all HPC nodes. - `lib_dir`: directory on the HPC where the startR R package and other required R packages are installed, accessible from all HPC nodes. These installed packages must be compatible with the R module specified in `r_module`. This parameter is optional; only required when the libraries are not installed in the R module. - `r_module`: name of the UNIX environment module to be used for R. If not specified, 'module load R' will be used. +- `cdo_module`: name of the UNIX environment module to be used for CDO. If not specified, no CDO module will be loaded. +- `conda_env`: path to the conda environment containing R (and CDO if needed). If both `r_module` and `conda_env` are specified, the conda environment will take preference. - `cores_per_job`: number of computing cores to be requested when submitting the job for each chunk to the HPC queue. Each node (should be core?) may be capable of supporting more than one computing thread. - `job_wallclock`: amount of time to reserve the resources when submitting the job for each chunk. Must follow the specific format required by the specified `queue_type`. - `max_jobs`: maximum number of jobs (chunks) to be queued simultaneously onto the HPC queue. Submitting too many jobs could overload the bandwidth between the HPC nodes and the storage system, or could overload the queue system. - `extra_queue_params`: list of character strings with additional queue headers for the jobs to be submitted to the HPC. Mainly used to specify the amount of memory to book for each job (e.g. '#SBATCH --mem-per-cpu=30000'; __NOTE: this line does not work on Nord3v2__), to request special queuing (e.g. '#SBATCH --qos=bsc_es'), or to request use of specific software (e.g. '#SBATCH --reservation=test-rhel-7.5'). - `bidirectional`: whether the connection between the R workstation and the HPC login node is bidirectional (TRUE) or unidirectional from the workstation to the login node (FALSE). - `polling_period`: when the connection is unidirectional, the workstation will ask the HPC login node for results each `polling_period` seconds. An excessively small value can overload the login node or result in temporary banning. -- `special_setup`: name of the machine if the computation requires an special setup. Only Marenostrum 4 needs this parameter (e.g. special_setup = 'marenostrum4'). +- `special_setup`: name of the machine if the computation requires an special setup. Nord4 requires this parameter (special_setup = 'nord4'). The parameter for the old Marenostrum 4 machine (i.e. special_setup = 'marenostrum4') will be deprecated in a future release. After the `Compute()` call is executed, an EC-Flow server is automatically started on your workstation, which will orchestrate the work and dispatch jobs onto the HPC. Thanks to the use of EC-Flow, you will also be able to monitor visually the progress of the execution. See the "Collect and the EC-Flow GUI" section. @@ -722,7 +725,7 @@ You can see one example of cluster configuration below. res <- Compute(wf, chunks = list(sdate = 2), threads_compute = 4, threads_load = 2, cluster = list( - queue_host = 'nord3', + queue_host = 'nord4', expid = , hpc_user = "bsc32xxx", r_module = "R/4.1.2-foss-2019b", @@ -731,7 +734,8 @@ You can see one example of cluster configuration below. autosubmit_version = '4.0.0b', cores_per_job = 4, job_wallclock = '01:00:00', - max_jobs = 4 + max_jobs = 4, + special_setup = 'nord4' ), workflow_manager = 'autosubmit', autosubmit_suite_dir = "/home/Earth//startR_local_autosubmit/", @@ -743,7 +747,7 @@ You can see one example of cluster configuration below. res <- Compute(wf, chunks = list(sdate = 2), threads_compute = 4, threads_load = 2, cluster = list( - queue_host = 'nord3', + queue_host = 'nord4', expid = , hpc_user = "bsc32xxx", r_module = "R/4.1.2-foss-2019b", @@ -752,7 +756,8 @@ You can see one example of cluster configuration below. autosubmit_version = '4.0.98', cores_per_job = 4, job_wallclock = '01:00:00', - max_jobs = 4 + max_jobs = 4, + special_setup = 'nord4' ), workflow_manager = 'autosubmit', autosubmit_suite_dir = "/home/Earth//startR_local_autosubmit/", @@ -773,7 +778,8 @@ To have the good practice, note down the expid if it is automatically created by - `run_dir`: The directory to run the startR jobs. It is useful when the self-defined function has relative path. - `init_commands`: The initial commands in bash script before R script runs. For example, the modules required by computation can be loaded here. - `r_module`: Name of the UNIX environment module to be used for R. If not specified, `module load R` will be used. -- `CDO_module`: Name of the UNIX environment module to be used for CDO. If not specified, it is NULL and no CDO module will be loaded. Make sure to assign it if `tranform` is required in Start(). +- `CDO_module`: Name of the UNIX environment module to be used for CDO. If not specified, it is NULL and no CDO module will be loaded. Make sure to assign it if `transform` is required in Start(). +- `conda_env`: path to the conda environment containing R (and CDO if needed). If both `r_module` and `conda_env` are specified, the conda environment will take preference. - `autosubmit_module`: The name of the Autosubmit module. If not specified, `module load autosubmit` will be used. - `autosubmit_version`: The Autosubmit version (e.g. '4.0.0b'). If not specified, it will be parsed from the `autosubmit_module`. If it cannot be parsed from this parameter, it will be set to '4.0.0b' by default. - `cores_per_job`: Number of computing cores to be requested when submitting the job for each chunk to the HPC queue. @@ -783,7 +789,7 @@ It is corresponded to the parameter "THREADS" in _jobs.yml_ and "PROCESSORS_PER_ - `extra_queue_params`: list of character strings for additional queue headers for the jobs to be submitted to the HPC. For example, to constraint using medmem node ('#SBATCH --constraint=medmem'); to request an exclusive mode ('#SBATCH --exclusive'). - `polling_period`: when the connection is unidirectional, the workstation will ask the HPC login node for results each `polling_period` seconds. An excessively small value can overload the login node or result in temporary banning. -- `special_setup`: name of the machine if the computation requires an special setup. Only Marenostrum 4 needs this parameter (e.g. special_setup = 'marenostrum4'). +- `special_setup`: name of the machine if the computation requires an special setup. Nord4 requires this parameter (special_setup = 'nord4'). The parameter for the old Marenostrum 4 machine (i.e. special_setup = 'marenostrum4') will be deprecated in a future release. After the Compute() call is executed, you can monitor the status on [Autosubmit GUI](https://earth.bsc.es/autosubmitapp/). @@ -798,15 +804,16 @@ Usually, in use cases where large data inputs are involved, it is convenient to longitude = 2), threads_load = 2, threads_compute = 4, - cluster = list(queue_host = 'nord4.bsc.es', + cluster = list(queue_host = 'n4login1.bsc.es', queue_type = 'slurm', temp_dir = '/gpfs/scratch/bsc32/bsc32734/startR_hpc/', cores_per_job = 4, job_wallclock = '00:10:00', max_jobs = 4, - extra_queue_params = list('#SBATCH --constraint=medmem'), + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es'), bidirectional = FALSE, - polling_period = 10 + polling_period = 10, + special_setup = 'nord4' ), ecflow_suite_dir = '/home/Earth/aho/startR_local/', wait = FALSE @@ -1127,7 +1134,55 @@ r <- Compute(wf, ## 7. Compute() cluster templates -### Nord3-v2 + +### Nord4 + +```r +# Using Autosubmit +res <- Compute(wf, chunks = list(lat = 2, lon = 2), + threads_compute = 4, threads_load = 2, + cluster = list( + queue_host = 'n4login1.bsc.es', + expid = NULL, # fill with your own expid if you already have one + hpc_user = 'bsc032762', + r_module = 'R/4.1.2-foss-2019b', + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.0.98-foss-2015a-Python-3.7.3', + # autosubmit_module = 'autosubmit/4.0.98-foss-2021b-Python-3.9.6', + cores_per_job = 4, + job_wallclock = '01:00:00', + max_jobs = 4, + special_setup = 'nord4' + ), + workflow_manager = 'autosubmit', + autosubmit_suite_dir = "/home/Earth/vagudets/startR_local_autosubmit/", + autosubmit_server = 'bscesautosubmit01', + wait = TRUE + ) + + +# Using ecFlow +res <- Compute(wf, chunks = list(lat = 2, lon = 2), + threads_compute = 4, threads_load = 2, + cluster = list( + queue_host = 'n4login1.bsc.es', + r_module = "R/4.1.2-foss-2019b", + temp_dir = '/gpfs/scratch/bsc32/bsc032762/startR_hpc/', + CDO_module = "CDO/1.9.8-foss-2019b", + ecflow_module = 'ecFlow', + cores_per_job = 4, + job_wallclock = '01:00:00', + max_jobs = 4, + special_setup = "nord4", + bidirectional = FALSE, + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') + ), + ecflow_suite_dir = '/home/Earth/vagudets/startR_local/', + wait = TRUE + ) +``` + +### Nord3-v2 (deprecated) ```r # Using ecFlow @@ -1160,7 +1215,7 @@ cluster = list(queue_host = 'nord1.bsc.es', ) ``` -### CTE-Power9 +### CTE-Power9 (deprecated) ```r cluster = list(queue_host = 'p9login1.bsc.es', @@ -1186,7 +1241,7 @@ cluster = list(queue_host = 'bsceslogin01.bsc.es', ) ``` -### Marenostrum 4 +### Marenostrum 4 (deprecated) ```r cluster = list(queue_host = 'mn2.bsc.es', -- GitLab From 910d934987f91fc7a74fd5a69b0d5ded2a9bdbb4 Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 5 Mar 2025 15:09:54 +0100 Subject: [PATCH 08/19] Update Use Cases to use Nord4 --- inst/doc/usecase/ex2_10_existing_mask.R | 34 +++++++++++++++- inst/doc/usecase/ex2_1_timedim.R | 38 +++++------------- inst/doc/usecase/ex2_2_attr.R | 30 +++++++------- inst/doc/usecase/ex2_3_cdo.R | 35 +++-------------- inst/doc/usecase/ex2_4_two_func.R | 35 +++-------------- inst/doc/usecase/ex2_5_rainFARM.R | 10 +++-- inst/doc/usecase/ex2_6_ext_param_func.R | 25 +++--------- .../ex2_7_seasonal_forecast_verification.R | 6 ++- inst/doc/usecase/ex2_8_calibration.R | 39 ++++++++++--------- .../usecase/ex3_1_SubseasonalECVHindcast.md | 32 ++++++++------- 10 files changed, 123 insertions(+), 161 deletions(-) diff --git a/inst/doc/usecase/ex2_10_existing_mask.R b/inst/doc/usecase/ex2_10_existing_mask.R index 93c1aba..8e70712 100644 --- a/inst/doc/usecase/ex2_10_existing_mask.R +++ b/inst/doc/usecase/ex2_10_existing_mask.R @@ -67,6 +67,34 @@ res <- Compute(workflow = wf_mask, chunks = list(lat = 2, lon = 2)) + + +# Submit to Nord4 +#-------------------user-defined--------------------- + queue_host <- 'n4login1.bsc.es' + temp_dir <- '/gpfs/scratch/bsc32/bsc032734/startR_hpc/' + ecflow_suite_dir <- '/home/Earth/aho/startR_local/' +#---------------------------------------------------- + res <- Compute(wf_mask, + chunks = list(lat = 2, + lon = 2),#$output1 + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = queue_host, + queue_type = 'slurm', + temp_dir = temp_dir, + cores_per_job = 2, + job_wallclock = '05:00', + max_jobs = 4, + bidirectional = FALSE, + polling_period = 10, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') + ), + ecflow_suite_dir = ecflow_suite_dir, + wait = TRUE + ) + # Submit to Nord3v2 #-------------------user-defined--------------------- queue_host <- 'nord4' @@ -200,10 +228,10 @@ res <- Compute(workflow = wf_mask, chunks = list(lat = 2, lon = 2)) -# Submit to Nord3v2 +# Submit to Nord4 #-------------------user-defined--------------------- queue_host <- 'nord4' - temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' + temp_dir <- '/gpfs/scratch/bsc32/bsc032734/startR_hpc/' ecflow_suite_dir <- '/home/Earth/aho/startR_local/' #---------------------------------------------------- res <- Compute(wf_mask, @@ -219,6 +247,8 @@ res <- Compute(workflow = wf_mask, max_jobs = 4, bidirectional = FALSE, polling_period = 10 + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') ), ecflow_suite_dir = ecflow_suite_dir, wait = TRUE diff --git a/inst/doc/usecase/ex2_1_timedim.R b/inst/doc/usecase/ex2_1_timedim.R index 0c0f36c..f9872af 100644 --- a/inst/doc/usecase/ex2_1_timedim.R +++ b/inst/doc/usecase/ex2_1_timedim.R @@ -43,31 +43,8 @@ library(startR) # 220.5 273.3 282.7 279.9 295.6 306.8 # ---------------------------------------------------------------- -## on Power9 -#-----------modify according to your personal info--------- - queue_host <- 'cte-power' #your own host name for power9 - temp_dir <- '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' - ecflow_suite_dir <- '/home/Earth/nperez/startR_local/' #your own local directory -#------------------------------------------------------------ - res <- Compute(wf1, - chunks = list(ensemble = 20, - sdate = 2), - threads_load = 2, - threads_compute = 4, - cluster = list(queue_host = queue_host, - queue_type = 'slurm', - cores_per_job = 2, - temp_dir = temp_dir, - r_module = 'R/3.6.1-foss-2018b', - polling_period = 10, - job_wallclock = '01:00:00', - max_jobs = 40, - bidirectional = FALSE), - ecflow_suite_dir = ecflow_suite_dir, - wait = TRUE) - -## on Nord3 with ecFlow +## on Nord4 with ecFlow #-----------modify according to your personal info--------- queue_host <- 'nord4' temp_dir <- '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' @@ -85,15 +62,17 @@ library(startR) polling_period = 10, job_wallclock = '01:00', max_jobs = 40, - bidirectional = FALSE), + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') + ), ecflow_suite_dir = ecflow_suite_dir, wait = TRUE) - -## on Nord3 with Autosubmit +## on Nord4 with Autosubmit #-----------modify according to your personal info--------- - hpc_user <- "bsc32xxx" + hpc_user <- "bsc032xxx" expid <- "xxxx" # autosubmit exp id; can be NULL autosubmit_suite_dir <- "/home/Earth//startR_local_autosubmit/" #------------------------------------------------------------ @@ -102,7 +81,7 @@ library(startR) threads_load = 2, threads_compute = 4, cluster = list( - queue_host = 'nord3', + queue_host = 'nord4', r_module = "R/4.1.2-foss-2019b", autosubmit_module = 'autosubmit/4.0.0b-foss-2015a-Python-3.7.3', cores_per_job = 2, @@ -110,6 +89,7 @@ library(startR) max_jobs = 40, polling_period = 10, extra_queue_params = list('#SBATCH --constraint=medmem', '#SBATCH --exclusive'), + special_setup = 'nord4', expid = NULL, hpc_user = hpc_user ), diff --git a/inst/doc/usecase/ex2_2_attr.R b/inst/doc/usecase/ex2_2_attr.R index fbd2eac..2968e93 100644 --- a/inst/doc/usecase/ex2_2_attr.R +++ b/inst/doc/usecase/ex2_2_attr.R @@ -47,13 +47,13 @@ library(multiApply) # 13.19 169.60 237.60 217.90 284.10 305.10 # ----------------------------------------------------------------- -## on Power9 +## on Nord4 #-----------modify according to your personal info--------- - queue_host = 'cte-power' #your own host name for power9 - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + queue_host = 'nord4' #your own host name for nord4 + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ - res2_P <- Compute(wf2, + res2_Nord <- Compute(wf2, chunks = list(ensemble = 20, sdate = 2), threads_load = 2, @@ -62,22 +62,23 @@ library(multiApply) queue_type = 'slurm', cores_per_job = 2, temp_dir = temp_dir, - r_module = 'R/3.6.1-foss-2018b', - #extra_queue_params = list('#SBATCH --mem-per-cpu=3000'), polling_period = 10, - job_wallclock = '01:00:00', + job_wallclock = '01:00', max_jobs = 40, - bidirectional = FALSE), + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') + ), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) - -## on Nord3v2 + +## on Power9 #-----------modify according to your personal info--------- - queue_host = 'nord4' #your own host name for power9 + queue_host = 'cte-power' #your own host name for power9 temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ - res2_Nord <- Compute(wf2, + res2_P <- Compute(wf2, chunks = list(ensemble = 20, sdate = 2), threads_load = 2, @@ -86,9 +87,12 @@ library(multiApply) queue_type = 'slurm', cores_per_job = 2, temp_dir = temp_dir, + r_module = 'R/3.6.1-foss-2018b', + #extra_queue_params = list('#SBATCH --mem-per-cpu=3000'), polling_period = 10, - job_wallclock = '01:00', + job_wallclock = '01:00:00', max_jobs = 40, bidirectional = FALSE), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) + diff --git a/inst/doc/usecase/ex2_3_cdo.R b/inst/doc/usecase/ex2_3_cdo.R index 9dca045..bc9d1d6 100644 --- a/inst/doc/usecase/ex2_3_cdo.R +++ b/inst/doc/usecase/ex2_3_cdo.R @@ -49,36 +49,11 @@ library(startR) # 206.3 269.2 283.3 278.8 296.4 314.2 # -------------------------------------------------------------------- -## on Power9 -#-----------modify according to your personal info--------- - queue_host = 'cte-power' #your own host name for power9 - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' - ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory -#------------------------------------------------------------ - res3_P <- Compute(wf3, - chunks = list(ensemble = 2, - sdate = 2), - threads_load = 2, - threads_compute = 4, - cluster = list(queue_host = queue_host, #your own host name for power9 - queue_type = 'slurm', - cores_per_job = 1, - temp_dir = temp_dir, - r_module = 'R/3.5.0-foss-2018b', - CDO_module = 'CDO/1.9.5-foss-2018b', - extra_queue_params = list('#SBATCH --mem-per-cpu=3000'), - polling_period = 100, - job_wallclock = '01:00:00', - max_jobs = 4, - bidirectional = FALSE), - ecflow_suite_dir = ecflow_suite_dir, #your own local directory - wait = TRUE) - -## on Nord3v2 +## on Nord4 #-----------modify according to your personal info--------- - queue_host = 'nord4.bsc.es' - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + queue_host = 'n4login1.bsc.es' + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ res3_N <- Compute(wf3, @@ -94,6 +69,8 @@ library(startR) polling_period = 100, job_wallclock = '01:00', max_jobs = 4, - bidirectional = FALSE), + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) diff --git a/inst/doc/usecase/ex2_4_two_func.R b/inst/doc/usecase/ex2_4_two_func.R index c76eda9..9d4d2d3 100644 --- a/inst/doc/usecase/ex2_4_two_func.R +++ b/inst/doc/usecase/ex2_4_two_func.R @@ -47,35 +47,10 @@ summary(res4$output1) # 227.3 259.6 280.8 277.1 296.2 306.7 # ------------------------------------------------------------------ -## on Power9 -#-----------modify according to your personal info----------- - queue_host = 'cte-power' #your own host name for power9 - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' - ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory -#------------------------------------------------------------ - res4_P <- Compute(wf4, - chunks = list(ensemble = 2, - sdate = 2), - threads_load = 2, - threads_compute = 4, - cluster = list(queue_host = queue_host, #your own host name for power9 - queue_type = 'slurm', - cores_per_job = 1, - temp_dir = temp_dir, - r_module = 'R/3.6.1-foss-2018b', - CDO_module = 'CDO/1.9.5-foss-2018b', - extra_queue_params = list('#SBATCH --mem-per-cpu=3000'), - polling_period = 10, - job_wallclock = '01:00:00', - max_jobs = 6, - bidirectional = FALSE), - ecflow_suite_dir = ecflow_suite_dir, #your own local directory - wait = TRUE) - ## on Nord3v2 #-----------modify according to your personal info----------- - queue_host = 'nord4.bsc.es' - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + queue_host = 'n4login1.bsc.es' + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ res4_N <- Compute(wf4, @@ -83,7 +58,7 @@ summary(res4$output1) sdate = 2), threads_load = 2, threads_compute = 4, - cluster = list(queue_host = queue_host, #your own host name for power9 + cluster = list(queue_host = queue_host, #your own host name for nord4 queue_type = 'slurm', cores_per_job = 1, temp_dir = temp_dir, @@ -91,6 +66,8 @@ summary(res4$output1) polling_period = 10, job_wallclock = '01:00', max_jobs = 6, - bidirectional = FALSE), + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, #your own local directory wait = TRUE) diff --git a/inst/doc/usecase/ex2_5_rainFARM.R b/inst/doc/usecase/ex2_5_rainFARM.R index 8b58901..379e81c 100644 --- a/inst/doc/usecase/ex2_5_rainFARM.R +++ b/inst/doc/usecase/ex2_5_rainFARM.R @@ -99,11 +99,11 @@ res <- Compute(workflow, threads_compute = 4) -#========= OPTION 2: Compute ON NORD3 ============ +#========= OPTION 2: Compute ON NORD4 ============ #-----------modify according to your personal info--------- - queue_host = 'nord3' # your own host name for nord3v2 - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + queue_host = 'nord4' # your own host name for nord4 + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' # your own local directory #------------------------------------------------------------ @@ -119,7 +119,9 @@ res <- Compute(workflow, polling_period = 10, job_wallclock = '01:00:00', max_jobs = 4, - bidirectional = FALSE), + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, wait = TRUE) diff --git a/inst/doc/usecase/ex2_6_ext_param_func.R b/inst/doc/usecase/ex2_6_ext_param_func.R index 760e30d..970c338 100644 --- a/inst/doc/usecase/ex2_6_ext_param_func.R +++ b/inst/doc/usecase/ex2_6_ext_param_func.R @@ -106,27 +106,10 @@ res$strat[1:5, 1:2, 1] # ------------------------------------------------------------- -#Power9 - res <- Compute(workflow$strat, - chunks = list(latitude = 2, longitude = 2), - threads_load = 2, - threads_compute = 2, - cluster = list(queue_host = 'cte-power', #user-specific - queue_type = 'slurm', - temp_dir = '/gpfs/scratch/bsc32/bsc32339/', #user-specific - r_module = 'R/3.6.1-foss-2018b', - cores_per_job = 2, - job_wallclock = '10:00:00', - max_jobs = 4, - bidirectional = FALSE, - polling_period = 10), - ecflow_suite_dir = '/esarchive/scratch/nperez/ecFlow', #user-specific - wait = TRUE) - -# Nord3-v2 +# Nord4 #-------------------user-defined--------------------- queue_host <- 'nord4' - temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' + temp_dir <- '/gpfs/scratch/bsc32/bsc032734/startR_hpc/' ecflow_suite_dir <- '/home/Earth/aho/startR_local/' #---------------------------------------------------- res <- Compute(workflow$strat, @@ -140,7 +123,9 @@ res$strat[1:5, 1:2, 1] job_wallclock = '10:00', max_jobs = 4, bidirectional = FALSE, - polling_period = 10), + polling_period = 10, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, #user-specific wait = TRUE) diff --git a/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R b/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R index b752383..3d48bf1 100644 --- a/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R +++ b/inst/doc/usecase/ex2_7_seasonal_forecast_verification.R @@ -77,7 +77,7 @@ wait = TRUE ) -# Compute() on Nord3v2 +# Compute() on Nord4 res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), @@ -85,11 +85,13 @@ threads_compute = 4, cluster = list(queue_host = 'nord4', # your alias queue_type = 'slurm', - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/', # your own path + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/', # your own path job_wallclock = '00:30', cores_per_job = 4, max_jobs = 4, bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es'), polling_period = 50), ecflow_suite_dir = '/home/Earth/nperez/startR_local/', # your own path wait = TRUE) diff --git a/inst/doc/usecase/ex2_8_calibration.R b/inst/doc/usecase/ex2_8_calibration.R index a36f349..5f21e7e 100644 --- a/inst/doc/usecase/ex2_8_calibration.R +++ b/inst/doc/usecase/ex2_8_calibration.R @@ -67,27 +67,30 @@ res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), # Declaration of HPC and execution ## ECFlow is required -# On Nord3v2 +# On Nord4 #-------------------user-defined--------------------- - queue_host <- 'nord4.bsc.es' - temp_dir <- '/gpfs/scratch/bsc32/bsc32734/startR_hpc/' + queue_host <- 'n4login1.bsc.es' + temp_dir <- '/gpfs/scratch/bsc32/bsc032734/startR_hpc/' ecflow_suite_dir <- '/home/Earth/aho/startR_local/' #---------------------------------------------------- -res_nord3 <- Compute(wf, - chunks = list(latitude = 2, longitude = 2), - threads_load = 2, - threads_compute = 4, - cluster = list(queue_host = queue_host, - queue_type = 'slurm', - extra_queue_params = list("#SBATCH --constraint=medmem"), - cores_per_job = 2, - temp_dir = temp_dir, - polling_period = 10, - job_wallclock = '01:00', - max_jobs = 4, - bidirectional = FALSE), - ecflow_suite_dir = ecflow_suite_dir, - wait = T) +res <- Compute(wf, + chunks = list(latitude = 2, longitude = 2), + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = queue_host, + queue_type = 'slurm', + extra_queue_params = list("#SBATCH -A bsc32", + "#SBATCH -q bsc_es", + "#SBATCH --constraint=medmem"), + cores_per_job = 2, + temp_dir = temp_dir, + polling_period = 10, + job_wallclock = '01:00', + max_jobs = 4, + bidirectional = FALSE, + special_setup = 'nord4'), + ecflow_suite_dir = ecflow_suite_dir, + wait = T) # Results dim(res$output1) diff --git a/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md b/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md index e554c66..0c39f24 100644 --- a/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md +++ b/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md @@ -196,24 +196,26 @@ Finally, execute the analysis, for instance, in Nord3v2: ```r #-----------modify according to your personal info--------- - queue_host = 'nord4' #your own host name for Nord3v2 - temp_dir = '/gpfs/scratch/bsc32/bsc32339/startR_hpc/' + queue_host = 'nord4' #your own host name for Nord4 + temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' ecflow_suite_dir = '/home/Earth/nperez/startR_local/' #your own local directory #------------------------------------------------------------ result <- Compute(wf, - chunks = list(time = 4, longitude = 2, latitude = 2), - threads_load = 2, - threads_compute = 12, - cluster = list(queue_host = queue_host, - queue_type = 'slurm', - cores_per_job = 12, - temp_dir = temp_dir, - polling_period = 10, - job_wallclock = '03:00:00', - max_jobs = 16, - bidirectional = FALSE), - ecflow_suite_dir = ecflow_suite_dir, - wait = TRUE) + chunks = list(time = 4, longitude = 2, latitude = 2), + threads_load = 2, + threads_compute = 12, + cluster = list(queue_host = queue_host, + queue_type = 'slurm', + cores_per_job = 12, + temp_dir = temp_dir, + polling_period = 10, + job_wallclock = '03:00:00', + max_jobs = 16, + bidirectional = FALSE, + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), + ecflow_suite_dir = ecflow_suite_dir, + wait = TRUE) ``` *Notice that the execution of `Compute` may last for ~2 hours each chunk. Consider set `wait` as false (see [practical guide](https://earth.bsc.es/gitlab/es/startR/-/blob/master/inst/doc/practical_guide.md#collect-and-the-ec-flow-gui)).* -- GitLab From 843a5e63e95b36e94af7b112e8c7b258f547ea6e Mon Sep 17 00:00:00 2001 From: vagudets Date: Wed, 5 Mar 2025 15:10:23 +0100 Subject: [PATCH 09/19] Fix typos, update README to use Nord4 --- README.md | 35 ++++++++++--------- inst/doc/practical_guide.md | 67 ++++++++++++++----------------------- 2 files changed, 44 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 010fafa..dd4344d 100644 --- a/README.md +++ b/README.md @@ -97,25 +97,28 @@ wf <- AddStep(data, step) #### 3. Declaration of the HPC platform and execution ```r + res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), - threads_load = 1, - threads_compute = 2, - cluster = list(queue_host = 'p9login1.bsc.es', - queue_type = 'slurm', - temp_dir = '/gpfs/scratch/bsc32/bsc32473/startR_tests/', - lib_dir = '/gpfs/projects/bsc32/share/R_libs/3.5/', - r_module = 'R/3.5.0-foss-2018b', - cores_per_job = 2, - job_wallclock = '00:10:00', - max_jobs = 4, - extra_queue_params = list('#SBATCH --qos=bsc_es'), - bidirectional = FALSE, - polling_period = 10 - ), - ecflow_suite_dir = '/home/Earth/nmanuben/test_startR/', - wait = TRUE) + threads_load = 2, + threads_compute = 4, + cluster = list(queue_host = 'n4login1.bsc.es', + queue_type = 'slurm', + temp_dir = /gpfs/scratch/bsc32/bsc032473/startR_tests/', + r_module = 'R/4.1.2-foss-2019b' + cores_per_job = 4, + job_wallclock = '00:10:00', + max_jobs = 4, + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es'), + bidirectional = FALSE, + special_setup = 'nord4', + polling_period = 10 + ), + ecflow_suite_dir = ecflow_suite_dir, + wait = TRUE + ) + ``` #### 4. Monitoring the execution diff --git a/inst/doc/practical_guide.md b/inst/doc/practical_guide.md index 11c2338..811d8d3 100644 --- a/inst/doc/practical_guide.md +++ b/inst/doc/practical_guide.md @@ -727,7 +727,7 @@ You can see one example of cluster configuration below. cluster = list( queue_host = 'nord4', expid = , - hpc_user = "bsc32xxx", + hpc_user = "bsc032xxx", r_module = "R/4.1.2-foss-2019b", CDO_module = "CDO/1.9.8-foss-2019b", autosubmit_module = 'autosubmit/4.0.0b-foss-2015a-Python-3.7.3', @@ -749,7 +749,7 @@ You can see one example of cluster configuration below. cluster = list( queue_host = 'nord4', expid = , - hpc_user = "bsc32xxx", + hpc_user = "bsc032xxx", r_module = "R/4.1.2-foss-2019b", CDO_module = "CDO/1.9.8-foss-2019b", autosubmit_module = 'autosubmit/4.0.98-foss-2021b-Python-3.9.6', @@ -806,7 +806,7 @@ Usually, in use cases where large data inputs are involved, it is convenient to threads_compute = 4, cluster = list(queue_host = 'n4login1.bsc.es', queue_type = 'slurm', - temp_dir = '/gpfs/scratch/bsc32/bsc32734/startR_hpc/', + temp_dir = '/gpfs/scratch/bsc32/bsc032734/startR_hpc/', cores_per_job = 4, job_wallclock = '00:10:00', max_jobs = 4, @@ -1139,47 +1139,30 @@ r <- Compute(wf, ```r # Using Autosubmit -res <- Compute(wf, chunks = list(lat = 2, lon = 2), - threads_compute = 4, threads_load = 2, - cluster = list( - queue_host = 'n4login1.bsc.es', - expid = NULL, # fill with your own expid if you already have one - hpc_user = 'bsc032762', - r_module = 'R/4.1.2-foss-2019b', - CDO_module = "CDO/1.9.8-foss-2019b", - autosubmit_module = 'autosubmit/4.0.98-foss-2015a-Python-3.7.3', - # autosubmit_module = 'autosubmit/4.0.98-foss-2021b-Python-3.9.6', - cores_per_job = 4, - job_wallclock = '01:00:00', - max_jobs = 4, - special_setup = 'nord4' - ), - workflow_manager = 'autosubmit', - autosubmit_suite_dir = "/home/Earth/vagudets/startR_local_autosubmit/", - autosubmit_server = 'bscesautosubmit01', - wait = TRUE - ) - +cluster = list(queue_host = 'n4login1.bsc.es', + expid = NULL, # NOTE: fill with your own expid if you already have one + hpc_user = 'bsc032762', + r_module = 'R/4.1.2-foss-2019b', + CDO_module = "CDO/1.9.8-foss-2019b", + autosubmit_module = 'autosubmit/4.0.98-foss-2015a-Python-3.7.3', + cores_per_job = 4, + job_wallclock = '01:00:00', + max_jobs = 4, + special_setup = 'nord4' + ) # Using ecFlow -res <- Compute(wf, chunks = list(lat = 2, lon = 2), - threads_compute = 4, threads_load = 2, - cluster = list( - queue_host = 'n4login1.bsc.es', - r_module = "R/4.1.2-foss-2019b", - temp_dir = '/gpfs/scratch/bsc32/bsc032762/startR_hpc/', - CDO_module = "CDO/1.9.8-foss-2019b", - ecflow_module = 'ecFlow', - cores_per_job = 4, - job_wallclock = '01:00:00', - max_jobs = 4, - special_setup = "nord4", - bidirectional = FALSE, - extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') - ), - ecflow_suite_dir = '/home/Earth/vagudets/startR_local/', - wait = TRUE - ) +cluster = list(queue_host = 'n4login1.bsc.es', + r_module = "R/4.1.2-foss-2019b", + temp_dir = '/gpfs/scratch/bsc32/bsc032762/startR_hpc/', + CDO_module = "CDO/1.9.8-foss-2019b", + cores_per_job = 4, + job_wallclock = '01:00:00', + max_jobs = 4, + special_setup = "nord4", + bidirectional = FALSE, + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es') + ) ``` ### Nord3-v2 (deprecated) -- GitLab From 1100b50613e954731038bd3abc94761b95addf7e Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 6 Mar 2025 10:39:46 +0100 Subject: [PATCH 10/19] Remove duplicated sanity check; add warning about 'conda_env' preceding 'r_module' --- R/ByChunks_ecflow.R | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/R/ByChunks_ecflow.R b/R/ByChunks_ecflow.R index 4c5c592..11d39c7 100644 --- a/R/ByChunks_ecflow.R +++ b/R/ByChunks_ecflow.R @@ -282,19 +282,14 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', warning("The component 'conda_env' of parameter 'cluster' must have at ", "least one character and contain no blank spaces.") } + if (!is.null(cluster[['r_module']])) { + warning("Both 'conda_env' and 'r_module' cluster components have been ", + "specified. 'conda_env' will be used.") + } } else { cluster$conda_env <- NULL } - if (!is.null(cluster[['conda_env']])) { - if (!is.character(cluster[['conda_env']])) { - stop("The component 'conda_env' of the parameter 'cluster' must be a character string.") - } - if (nchar(cluster[['conda_env']]) < 1 || grepl(' ', cluster[['conda_env']])) { - warning("The component 'conda_env' of parameter 'cluster' must have at ", - "least one character and contain no blank spaces.") - } - } if (!is.character(cluster[['ecflow_module']])) { stop("The component 'ecflow_module' of the parameter 'cluster' must be a character string.") } -- GitLab From 6f108d6c29c84599a83adfc33840057bd220517e Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 6 Mar 2025 10:57:32 +0100 Subject: [PATCH 11/19] Fix typo and update first example in practical guide --- README.md | 3 +-- inst/doc/practical_guide.md | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dd4344d..c8cf874 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,6 @@ wf <- AddStep(data, step) #### 3. Declaration of the HPC platform and execution ```r - res <- Compute(wf, chunks = list(latitude = 2, longitude = 2), @@ -106,7 +105,7 @@ res <- Compute(wf, cluster = list(queue_host = 'n4login1.bsc.es', queue_type = 'slurm', temp_dir = /gpfs/scratch/bsc32/bsc032473/startR_tests/', - r_module = 'R/4.1.2-foss-2019b' + r_module = 'R/4.1.2-foss-2019b', cores_per_job = 4, job_wallclock = '00:10:00', max_jobs = 4, diff --git a/inst/doc/practical_guide.md b/inst/doc/practical_guide.md index 811d8d3..691d657 100644 --- a/inst/doc/practical_guide.md +++ b/inst/doc/practical_guide.md @@ -58,7 +58,7 @@ Afterwards, you will need to understand and use five functions, all of them incl - **Compute()**, for specifying the HPC to be employed, the execution parameters (e.g. number of chunks and cores), and to trigger the computation - **Collect()** and the **EC-Flow graphical user interface**, for monitoring of the progress and collection of results -Next, you can see an example startR script performing the ensemble mean of a small data set on an HPC cluster such as Nord3-v2 or CTE-Power9, for you to get a broad picture of how the startR functions interact and the information that is represented in a startR script. Note that the `queue_host`, `temp_dir` and `ecflow_suite_dir` parameters in the `Compute()` call are user-specific. +Next, you can see an example startR script performing the ensemble mean of a small data set on an HPC cluster such as Nord4, for you to get a broad picture of how the startR functions interact and the information that is represented in a startR script. Note that the `queue_host`, `temp_dir` and `ecflow_suite_dir` parameters in the `Compute()` call are user-specific. ```r library(startR) @@ -89,15 +89,16 @@ wf <- AddStep(data, step) longitude = 2), threads_load = 2, threads_compute = 4, - cluster = list(queue_host = 'nord4.bsc.es', + cluster = list(queue_host = 'n4login1.bsc.es', queue_type = 'slurm', temp_dir = '/gpfs/scratch/bsc32/bsc32734/startR_hpc/', cores_per_job = 4, job_wallclock = '00:10:00', max_jobs = 4, - extra_queue_params = list('#SBATCH --constraint=medmem'), + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es'), bidirectional = FALSE, - polling_period = 10 + polling_period = 10, + special_setup = 'nord4' ), ecflow_suite_dir = '/home/Earth/aho/startR_local/', wait = TRUE @@ -130,7 +131,16 @@ After following these steps for the connections in both directions (although fro Do not forget adding the following lines in your .bashrc on the HPC machine. -If you are planning to run it on Nord3-v2, you have to add: +If you are planning to run it on Nord4, you have to add: +elif [ $BSC_MACHINE == "nord4" ]; then + module load nord3-singu + module load bsc/current + module use /gpfs/projects/bsc32/software/suselinux/11/modules/all + unset PYTHONSTARTUP +fi + + +If you are using Nord3v2: ``` if [ $BSC_MACHINE == "nord3v2" ]; then module purge -- GitLab From a4f1b7c851d107841dd1decad1d83fdf658738a3 Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 6 Mar 2025 12:37:42 +0100 Subject: [PATCH 12/19] Fix typos in usecases --- inst/doc/usecase/ex2_4_two_func.R | 2 +- inst/doc/usecase/ex2_8_calibration.R | 2 +- inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/inst/doc/usecase/ex2_4_two_func.R b/inst/doc/usecase/ex2_4_two_func.R index 9d4d2d3..e8411bc 100644 --- a/inst/doc/usecase/ex2_4_two_func.R +++ b/inst/doc/usecase/ex2_4_two_func.R @@ -47,7 +47,7 @@ summary(res4$output1) # 227.3 259.6 280.8 277.1 296.2 306.7 # ------------------------------------------------------------------ -## on Nord3v2 +## on Nord4 #-----------modify according to your personal info----------- queue_host = 'n4login1.bsc.es' temp_dir = '/gpfs/scratch/bsc32/bsc032339/startR_hpc/' diff --git a/inst/doc/usecase/ex2_8_calibration.R b/inst/doc/usecase/ex2_8_calibration.R index 5f21e7e..94cc429 100644 --- a/inst/doc/usecase/ex2_8_calibration.R +++ b/inst/doc/usecase/ex2_8_calibration.R @@ -81,7 +81,7 @@ res <- Compute(wf, queue_type = 'slurm', extra_queue_params = list("#SBATCH -A bsc32", "#SBATCH -q bsc_es", - "#SBATCH --constraint=medmem"), + "#SBATCH --constraint=highmem"), cores_per_job = 2, temp_dir = temp_dir, polling_period = 10, diff --git a/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md b/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md index 0c39f24..21262ce 100644 --- a/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md +++ b/inst/doc/usecase/ex3_1_SubseasonalECVHindcast.md @@ -212,8 +212,8 @@ result <- Compute(wf, job_wallclock = '03:00:00', max_jobs = 16, bidirectional = FALSE, - special_setup = 'nord4', - extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), + special_setup = 'nord4', + extra_queue_params = list('#SBATCH -A bsc32', '#SBATCH -q bsc_es')), ecflow_suite_dir = ecflow_suite_dir, wait = TRUE) ``` -- GitLab From c85f2a1eaf00b16bed88aa7c54d81c859bac8857 Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 3 Apr 2025 10:45:06 +0200 Subject: [PATCH 13/19] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c8cf874..f85ec76 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ res <- Compute(wf, threads_compute = 4, cluster = list(queue_host = 'n4login1.bsc.es', queue_type = 'slurm', - temp_dir = /gpfs/scratch/bsc32/bsc032473/startR_tests/', + temp_dir = '/gpfs/scratch/bsc32/bsc032473/startR_tests/', r_module = 'R/4.1.2-foss-2019b', cores_per_job = 4, job_wallclock = '00:10:00', -- GitLab From 95d0f700e39b0e49b7ea53dfbfc8732d1fa56fad Mon Sep 17 00:00:00 2001 From: vagudets Date: Mon, 7 Apr 2025 10:52:37 +0200 Subject: [PATCH 14/19] Improve logic for special setup commands and initial commands --- R/ByChunks_ecflow.R | 40 +++++++++++------ R/Utils.R | 43 ++++++++++++------- inst/chunking/Autosubmit/startR_autosubmit.sh | 1 + inst/chunking/ecFlow/Chunk.ecf | 1 + 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/R/ByChunks_ecflow.R b/R/ByChunks_ecflow.R index 11d39c7..2b63b8e 100644 --- a/R/ByChunks_ecflow.R +++ b/R/ByChunks_ecflow.R @@ -510,8 +510,10 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', writeLines(chunk_script_lines, paste0(ecflow_suite_dir_suite, '/load_process_save_chunk_ecflow.R')) # Copy Chunk.ecf into shared folder - chunk_ecf_script <- file(system.file('chunking/ecFlow/Chunk.ecf', - package = 'startR')) + ## TODO: Remove + chunk_ecf_script <- file("/esarchive/scratch/vagudets/repos/startR/inst/chunking/ecFlow/Chunk.ecf") + # chunk_ecf_script <- file(system.file('chunking/ecFlow/Chunk.ecf', + # package = 'startR')) chunk_ecf_script_lines <- readLines(chunk_ecf_script) close(chunk_ecf_script) if (cluster[['queue_type']] == 'host') { @@ -524,24 +526,34 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', chunk_ecf_script_lines) } # Special setup: Nord4 singularity wrapper + special_setup_commands <- list("") if (cluster[['special_setup']] == "nord4") { - nord4_commands <- list(paste0("module load nord3-singu \n", - "module load bsc/current \n", - "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", - "unset PYTHONSTARTUP \n", - "alias cdo='nord3_singu_es cdo' \n", - "alias Rscript='nord3_singu_es Rscript' \n", - "alias ecflow_client='nord3_singu_es ecflow_client' \n", - "shopt -s expand_aliases \n", - "include_init_commands")) - chunk_ecf_script_lines <- gsub('^include_init_commands', - paste0(paste0(nord4_commands, collapse = '\n'), '\n'), - chunk_ecf_script_lines) + if ('conda_env' %in% names(cluster)) { + special_setup_commands <- list(paste0("module load nord3-singu \n", + "module load bsc/current \n", + "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", + "unset PYTHONSTARTUP \n", + "alias ecflow_client='nord3_singu_es ecflow_client' \n", + "shopt -s expand_aliases \n")) + } else { + special_setup_commands <- list(paste0("module load nord3-singu \n", + "module load bsc/current \n", + "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", + "unset PYTHONSTARTUP \n", + "alias cdo='nord3_singu_es cdo' \n", + "alias Rscript='nord3_singu_es Rscript' \n", + "alias ecflow_client='nord3_singu_es ecflow_client' \n", + "shopt -s expand_aliases \n")) + } } + chunk_ecf_script_lines <- gsub('^include_special_setup', + paste0(paste0(special_setup_commands, collapse = '\n'), '\n'), + chunk_ecf_script_lines) # Init commands chunk_ecf_script_lines <- gsub('^include_init_commands', paste0(paste0(cluster[['init_commands']], collapse = '\n'), '\n'), chunk_ecf_script_lines) + if ('conda_env' %in% names(cluster)) { chunk_ecf_script_lines <- gsub('^include_module_load', paste0('set +eu \n', diff --git a/R/Utils.R b/R/Utils.R index e4ffe5c..bb74ee8 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -885,8 +885,10 @@ write_autosubmit_bash <- function(chunks, cluster, autosubmit_suite_dir) { chunk_args[2, ] <- paste0('%JOBS.CHUNK_', n_chunk, '.', chunk_names, '_N%') chunk_args <- paste0('(', paste(c(chunk_args), collapse = ' '), ')') - bash_script_template <- file(system.file('chunking/Autosubmit/startR_autosubmit.sh', - package = 'startR')) + ## TODO: remove + bash_script_template <- file("/esarchive/scratch/vagudets/repos/startR/inst/chunking/Autosubmit/startR_autosubmit.sh") + # bash_script_template <- file(system.file('chunking/Autosubmit/startR_autosubmit.sh', + # package = 'startR')) bash_script_lines <- readLines(bash_script_template) close(bash_script_template) @@ -894,20 +896,29 @@ write_autosubmit_bash <- function(chunks, cluster, autosubmit_suite_dir) { bash_script_lines <- gsub('^chunk_args=*', paste0('chunk_args=', chunk_args), bash_script_lines) # Include Nord4 Singularity wrapper special setup + special_setup_commands <- list("") if (cluster[['special_setup']] == "nord4") { - nord4_commands <- list(paste0("module load nord3-singu \n", - "module load bsc/current \n", - "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", - "unset PYTHONSTARTUP \n", - "alias cdo='nord3_singu_es cdo' \n", - "alias Rscript='nord3_singu_es Rscript' \n", - "alias ecflow_client='nord3_singu_es ecflow_client' \n", - "shopt -s expand_aliases \n", - "include_init_commands")) - bash_script_lines <- gsub('^include_init_commands', - paste0(paste0(nord4_commands, collapse = '\n'), '\n'), - bash_script_lines) + if ('conda_env' %in% names(cluster)) { + special_setup_commands <- list(paste0("module load nord3-singu \n", + "module load bsc/current \n", + "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", + "unset PYTHONSTARTUP \n", + "alias ecflow_client='nord3_singu_es ecflow_client' \n", + "shopt -s expand_aliases \n")) + } else { + special_setup_commands <- list(paste0("module load nord3-singu \n", + "module load bsc/current \n", + "module use /gpfs/projects/bsc32/software/suselinux/11/modules/all \n", + "unset PYTHONSTARTUP \n", + "alias cdo='nord3_singu_es cdo' \n", + "alias Rscript='nord3_singu_es Rscript' \n", + "alias ecflow_client='nord3_singu_es ecflow_client' \n", + "shopt -s expand_aliases \n")) + } } + bash_script_lines <- gsub('^include_special_setup', + paste0(paste0(special_setup_commands, collapse = '\n'), '\n'), + bash_script_lines) # Include init commands bash_script_lines <- gsub('^include_init_commands', paste0(paste0(cluster[['init_commands']], collapse = '\n'), '\n'), @@ -962,8 +973,8 @@ write_autosubmit_confs <- function(chunks, cluster, autosubmit_suite_dir) { # Get config template files from package ## TODO: Change - # template_dir <- "/esarchive/scratch/vagudets/repos/startR/inst/chunking/Autosubmit/" - template_dir <- system.file('chunking/Autosubmit/', package = 'startR') + template_dir <- "/esarchive/scratch/vagudets/repos/startR/inst/chunking/Autosubmit/" + # template_dir <- system.file('chunking/Autosubmit/', package = 'startR') config_files <- list.files(template_dir, pattern = "*\\.yml$") for (i_file in config_files) { diff --git a/inst/chunking/Autosubmit/startR_autosubmit.sh b/inst/chunking/Autosubmit/startR_autosubmit.sh index 63d9e61..be8ce1d 100644 --- a/inst/chunking/Autosubmit/startR_autosubmit.sh +++ b/inst/chunking/Autosubmit/startR_autosubmit.sh @@ -11,6 +11,7 @@ chunknum="${chunknum:5}" #e.g., CHUNK_0 # e.g., chunk_args=(%JOBS."${chunknum}".dat% %JOBS."${chunknum}".dat_N% %JOBS."${chunknum}".var% %JOBS."${chunknum}".var_N% %JOBS."${chunknum}".sdate% %JOBS."${chunknum}".sdate_N%) chunk_args= +include_special_setup include_init_commands include_module_load diff --git a/inst/chunking/ecFlow/Chunk.ecf b/inst/chunking/ecFlow/Chunk.ecf index 5a265fb..3471781 100644 --- a/inst/chunking/ecFlow/Chunk.ecf +++ b/inst/chunking/ecFlow/Chunk.ecf @@ -3,6 +3,7 @@ include_queue_header date --rfc-3339=seconds > %REMOTE_ECF_HOME%/%ECF_NAME%.setup_time +include_special_setup include_init_commands %include "./head.h" -- GitLab From f198677c4829b3e0c456b7afa5315393aba603db Mon Sep 17 00:00:00 2001 From: vagudets Date: Tue, 8 Apr 2025 15:29:52 +0200 Subject: [PATCH 15/19] Add 'PROJECT_DESTINATION' to expdef.yml to work with autosubmit>4.1.x; remove unneeded lines --- R/Utils.R | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/R/Utils.R b/R/Utils.R index bb74ee8..fe73ba9 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -934,12 +934,6 @@ write_autosubmit_bash <- function(chunks, cluster, autosubmit_suite_dir) { } else { bash_script_lines <- gsub('^include_module_load', paste0('module load ', cluster[['r_module']]), - paste0('if [ $BSC_MACHINE == "nord4" ]; then \n', - ' unalias Rscript 2>/dev/null \n', - ' unalias cdo 2>/dev/null \n', - ' alias cdo="nord3_singu cdo" \n', - ' alias Rscript="nord3_singu Rscript" \n', - 'fi'), bash_script_lines) } # Rewrite cd run_dir @@ -997,10 +991,12 @@ write_autosubmit_confs <- function(chunks, cluster, autosubmit_suite_dir) { ############################################################ } else if (conf_type == "expdef") { + proj_dir <- paste0("STARTR_CHUNKING_", cluster$expid) conf$default$EXPID <- cluster$expid conf$default$HPCARCH <- cluster$queue_host # PROJECT_PATH should be where submit.sh and load....R stored --> local startR_autosubmit folder, under expID/ - conf$local$PROJECT_PATH <- file.path(autosubmit_suite_dir, paste0("STARTR_CHUNKING_", cluster$expid)) + conf$local$PROJECT_PATH <- file.path(autosubmit_suite_dir, proj_dir) + conf$project$PROJECT_DESTINATION <- proj_dir ############################################################ } else if (conf_type == "jobs") { -- GitLab From dbd8a54d24b6dc5a95c040e6c986d428e6226667 Mon Sep 17 00:00:00 2001 From: vagudets Date: Tue, 8 Apr 2025 15:30:34 +0200 Subject: [PATCH 16/19] Get 'autosubmit_version' from system when run locally --- R/ByChunks_autosubmit.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/R/ByChunks_autosubmit.R b/R/ByChunks_autosubmit.R index 5f1544f..9e66648 100644 --- a/R/ByChunks_autosubmit.R +++ b/R/ByChunks_autosubmit.R @@ -311,6 +311,8 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', if (!is.character(cluster[['autosubmit_version']])) { stop("The component 'autosubmit_version' of the parameter 'cluster' must be a character string.") } + } else if (is.null(autosubmit_server)) { + cluster[['autosubmit_version']] <- system("autosubmit --version", intern = TRUE) } else { cluster[['autosubmit_version']] <- stringr::str_extract(cluster['autosubmit_module'], "(?<=/)(.+)(?=\\-foss)") -- GitLab From c0080587c5eadb3d32d2ee7ec6a232a6a1f2894a Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 10 Apr 2025 14:53:05 +0200 Subject: [PATCH 17/19] Remove CTE-AMD and MN5 from list of supported HPCs --- R/ByChunks_autosubmit.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/ByChunks_autosubmit.R b/R/ByChunks_autosubmit.R index 9e66648..bbcac47 100644 --- a/R/ByChunks_autosubmit.R +++ b/R/ByChunks_autosubmit.R @@ -236,7 +236,7 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', cluster <- default_cluster ### queue_host - support_hpcs <- c('local', 'nord3', 'mn5', 'cte-amd', 'nord4') # names in platforms.yml + support_hpcs <- c('local', 'nord3', 'nord4') # names in platforms.yml if (is.null(cluster$queue_host) || !cluster$queue_host %in% support_hpcs) { stop("Cluster component 'queue_host' must be one of the following: ", paste(support_hpcs, collapse = ','), '.') -- GitLab From 440b9d2acef95849f1d92086a2cddcde6acd6682 Mon Sep 17 00:00:00 2001 From: vagudets Date: Mon, 14 Apr 2025 10:23:13 +0200 Subject: [PATCH 18/19] Parse autosubmit version from module instead of trying to use 'autosubmit --version' --- R/ByChunks_autosubmit.R | 2 -- 1 file changed, 2 deletions(-) diff --git a/R/ByChunks_autosubmit.R b/R/ByChunks_autosubmit.R index bbcac47..bf63878 100644 --- a/R/ByChunks_autosubmit.R +++ b/R/ByChunks_autosubmit.R @@ -311,8 +311,6 @@ ByChunks_autosubmit <- function(step_fun, cube_headers, ..., chunks = 'auto', if (!is.character(cluster[['autosubmit_version']])) { stop("The component 'autosubmit_version' of the parameter 'cluster' must be a character string.") } - } else if (is.null(autosubmit_server)) { - cluster[['autosubmit_version']] <- system("autosubmit --version", intern = TRUE) } else { cluster[['autosubmit_version']] <- stringr::str_extract(cluster['autosubmit_module'], "(?<=/)(.+)(?=\\-foss)") -- GitLab From 216aee901056370596996fe6d0c81a19b75c806f Mon Sep 17 00:00:00 2001 From: vagudets Date: Tue, 22 Apr 2025 11:52:15 +0200 Subject: [PATCH 19/19] Remove paths to personal scratch --- R/ByChunks_ecflow.R | 11 ++--------- R/Utils.R | 14 +++++--------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/R/ByChunks_ecflow.R b/R/ByChunks_ecflow.R index 2b63b8e..eab03fe 100644 --- a/R/ByChunks_ecflow.R +++ b/R/ByChunks_ecflow.R @@ -510,10 +510,8 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', writeLines(chunk_script_lines, paste0(ecflow_suite_dir_suite, '/load_process_save_chunk_ecflow.R')) # Copy Chunk.ecf into shared folder - ## TODO: Remove - chunk_ecf_script <- file("/esarchive/scratch/vagudets/repos/startR/inst/chunking/ecFlow/Chunk.ecf") - # chunk_ecf_script <- file(system.file('chunking/ecFlow/Chunk.ecf', - # package = 'startR')) + chunk_ecf_script <- file(system.file('chunking/ecFlow/Chunk.ecf', + package = 'startR')) chunk_ecf_script_lines <- readLines(chunk_ecf_script) close(chunk_ecf_script) if (cluster[['queue_type']] == 'host') { @@ -622,11 +620,6 @@ ByChunks_ecflow <- function(step_fun, cube_headers, ..., chunks = 'auto', writeLines(chunk_queue_header_lines, paste0(ecflow_suite_dir_suite, '/', cluster[['queue_type']], '.h')) # Copy headers - ## TODO: remove - # file.copy("/esarchive/scratch/vagudets/repos/startR/inst/chunking/ecFlow/head.h", - # to = ecflow_suite_dir_suite) - # file.copy("/esarchive/scratch/vagudets/repos/startR/inst/chunking/ecFlow/tail.h", - # to = ecflow_suite_dir_suite) file.copy(system.file('chunking/ecFlow/head.h', package = 'startR'), ecflow_suite_dir_suite) file.copy(system.file('chunking/ecFlow/tail.h', package = 'startR'), diff --git a/R/Utils.R b/R/Utils.R index fe73ba9..28840e1 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -885,10 +885,8 @@ write_autosubmit_bash <- function(chunks, cluster, autosubmit_suite_dir) { chunk_args[2, ] <- paste0('%JOBS.CHUNK_', n_chunk, '.', chunk_names, '_N%') chunk_args <- paste0('(', paste(c(chunk_args), collapse = ' '), ')') - ## TODO: remove - bash_script_template <- file("/esarchive/scratch/vagudets/repos/startR/inst/chunking/Autosubmit/startR_autosubmit.sh") - # bash_script_template <- file(system.file('chunking/Autosubmit/startR_autosubmit.sh', - # package = 'startR')) + bash_script_template <- file(system.file('chunking/Autosubmit/startR_autosubmit.sh', + package = 'startR')) bash_script_lines <- readLines(bash_script_template) close(bash_script_template) @@ -966,9 +964,7 @@ write_autosubmit_confs <- function(chunks, cluster, autosubmit_suite_dir) { # "autosubmit_suite_dir" should be the local folder that has R script, like ecflow_suite_dir in Compute() # Get config template files from package - ## TODO: Change - template_dir <- "/esarchive/scratch/vagudets/repos/startR/inst/chunking/Autosubmit/" - # template_dir <- system.file('chunking/Autosubmit/', package = 'startR') + template_dir <- system.file('chunking/Autosubmit/', package = 'startR') config_files <- list.files(template_dir, pattern = "*\\.yml$") for (i_file in config_files) { @@ -982,8 +978,8 @@ write_autosubmit_confs <- function(chunks, cluster, autosubmit_suite_dir) { conf$config$AUTOSUBMIT_VERSION <- cluster['autosubmit_version'] #Q: Should it be the total amount of chunk? ## TODO: Fix this; it is not generalizable - conf$config$AUTOSUBMIT_VERSION <- stringr::str_extract(cluster['autosubmit_module'], - "(?<=/)(.+)(?=\\-foss)") + # conf$config$AUTOSUBMIT_VERSION <- stringr::str_extract(cluster['autosubmit_module'], + # "(?<=/)(.+)(?=\\-foss)") conf$config$MAXWAITINGJOBS <- as.integer(prod(unlist(chunks))) # total amount of chunk #NOTE: Nord3 max. amount of queued jobs is 366 if (conf$config$MAXWAITINGJOBS > 366) conf$config$MAXWAITINGJOBS <- 366 -- GitLab