Any type of foreach adaptator
Hi @aho
I have received this suggestion by email from an external user (see below). Could you consider it, please?
(I'll sent you the email too)
Cheers,
Núria
Hi,
would you consider supporting other types of foreach parallel backends than the currently hard-coded doParallel package, e.g. from https://earth.bsc.es/gitlab/ces/multiApply/-/blob/master/R/Apply.R#L696-700:
# Execute in parallel if needed
parallel <- ncores > 1
if (parallel) registerDoParallel(ncores)
result <- llply(1:length(chunk_sizes), iteration, .parallel = parallel)
if (parallel) registerDoSEQ()
One way to do this, is to support whatever foreach adaptor is currently set when ncores = NA. Please see attached patch, but the gist is:
if (is.null(ncores)) {
ncores <- 1
parallel <- FALSE
} else if (is.na(ncores)) {
# Use whatever foreach adaptor is already registered
parallel <- NA
ncores <- getDoParWorkers() # number of parallel workers
} else if (is.numeric(ncores)) {
ncores <- round(ncores)
parallel <- (ncores > 1)
} else {
stop("Parameter 'ncores' must be numeric or NA.")
}
...
# Execute in parallel if needed
if (is.na(parallel)) {
parallel <- TRUE
} else if (parallel) {
registerDoParallel(ncores)
on.exit(registerDoSEQ())
}
result <- llply(1:length(chunk_sizes), iteration, .parallel = parallel)
This would allow users to use, for instance, any parallel backend supported by the futureverse, e.g.
library(multiApply)
data <- list(array(1:4, dim = c(A = 1, B = 2, C = 2)),
array(1:6, dim = c(a = 2, b = 3)))
test_fun <- function(x, y) {
str(list(x = x, y = y))
sum(x) / sum(y)
}
message("*** Sequential")
test <- Apply(data, target_dims = list(3, 2), test_fun)
test0 <- test
message("*** Parallel")
test <- Apply(data, target_dims = list(3, 2), test_fun, ncores = 2L)
stopifnot(identical(test, test0))
message("*** doMC with two forked workers")
library(doMC)
registerDoMC(2L)
test <- Apply(data, target_dims = list(3, 2), test_fun, ncores = NA)
stopifnot(identical(test, test0))
message("*** doFuture with sequential processing")
library(doFuture)
registerDoFuture()
plan(sequential)
test <- Apply(data, target_dims = list(3, 2), test_fun, ncores = NA)
stopifnot(identical(test, test0))
message("*** doFuture with two local workers")
library(doFuture)
registerDoFuture()
plan(multisession, workers = 2L)
test <- Apply(data, target_dims = list(3, 2), test_fun, ncores = NA)
stopifnot(identical(test, test0))
message("*** doFuture with parallel workers on two other machines")
library(doFuture)
registerDoFuture()
plan(cluster, workers = c("m1.example.org", "m2.example.org"))
test <- Apply(data, target_dims = list(3, 2), test_fun, ncores = NA)
stopifnot(identical(test, test0))
I've verified that this works.
Henrik