From a848d5f37bcdde2a65f80d1ccd947c94a7a94e2f Mon Sep 17 00:00:00 2001
From: Nicolau Manubens <nicolau.manubens@ic3.cat>
Date: Mon, 21 Jan 2019 00:41:03 +0100
Subject: [PATCH 1/3] Sending chunk indices to Step functions.

---
 inst/chunking/load_process_save_chunk.R | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/inst/chunking/load_process_save_chunk.R b/inst/chunking/load_process_save_chunk.R
index c552aa2..65abe45 100644
--- a/inst/chunking/load_process_save_chunk.R
+++ b/inst/chunking/load_process_save_chunk.R
@@ -82,7 +82,8 @@ res <- do.call("Apply",
                  list(data,
                       target_dims = attr(fun, 'TargetDims'),
                       fun = fun,
-                      output_dims = attr(fun, 'OutputDims'), 
+                      output_dims = attr(fun, 'OutputDims'),
+                      extra_info = list(chunk_indices = as.integer(chunk_indices)), 
                       ncores = threads_compute),
                  params
                 )
-- 
GitLab


From da56e9a928217e2237974b173d038ddd781fbaf5 Mon Sep 17 00:00:00 2001
From: Nicolau Manubens <nicolau.manubens@ic3.cat>
Date: Mon, 21 Jan 2019 00:57:52 +0100
Subject: [PATCH 2/3] Added feature to send attributes to Step functions and to
 load libraries remotely.

---
 R/Step.R                                | 34 ++++++++++++++++++++++++-
 inst/chunking/load_process_save_chunk.R |  6 +++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/R/Step.R b/R/Step.R
index d08595a..1f54d08 100644
--- a/R/Step.R
+++ b/R/Step.R
@@ -1,4 +1,5 @@
-Step <- function(fun, target_dims, output_dims) {
+Step <- function(fun, target_dims, output_dims, 
+                 use_libraries = NULL, use_attributes = NULL) {
   # Check fun
   if (!is.function(fun)) {
     stop("Parameter 'fun' must be a function.")
@@ -42,8 +43,39 @@ Step <- function(fun, target_dims, output_dims) {
     }
   }
 
+  # Check use_libraries
+  if (!is.null(use_libraries)) {
+    if (!is.character(use_libraries)) {
+      stop("Parameter 'use_libraries' must be a vector of character ",
+           "strings.")
+    }
+  }
+
+  # Check use_attributes
+  if (!is.null(use_attributes)) {
+    raise_error <- FALSE
+    if (!is.list(use_attributes)) {
+      raise_error <- TRUE
+    }
+    if (!all(sapply(use_attributes, 
+        function(x) {
+          is.character(x) ||
+          (is.list(x) && all(sapply(x, is.character)))
+        }))) {
+      raise_error <- TRUE
+    }
+    if (raise_error) {
+      stop("Parameter 'use_attributes' must be a list of vectors of ",
+           "character strings or of lists of vectors of character ",
+           "strings.")
+    }
+  }
+
   attr(fun, 'TargetDims') <- target_dims
   attr(fun, 'OutputDims') <- output_dims
+  attr(fun, 'UseLibraries') <- use_libraries
+  attr(fun, 'UseAttributes') <- use_attributes
+
   # TODO: Add provenance info
   class(fun) <- 'startR_step_fun'
 
diff --git a/inst/chunking/load_process_save_chunk.R b/inst/chunking/load_process_save_chunk.R
index 65abe45..f2834da 100644
--- a/inst/chunking/load_process_save_chunk.R
+++ b/inst/chunking/load_process_save_chunk.R
@@ -76,6 +76,11 @@ t_end_load <- Sys.time()
 t_load <- as.numeric(difftime(t_end_load, t_begin_load, units = 'secs'))
 
 t_begin_compute <- Sys.time()
+if (!is.null(attr(fun, 'UseLibraries'))) {
+  for (i in seq_along(attr(fun, 'UseLibraries'))) {
+    require(attr(fun, 'UseLibraries')[i])
+  }
+}
 Apply <- multiApply::Apply
 res <- do.call("Apply",
                c(
@@ -83,6 +88,7 @@ res <- do.call("Apply",
                       target_dims = attr(fun, 'TargetDims'),
                       fun = fun,
                       output_dims = attr(fun, 'OutputDims'),
+                      use_attributes = attr(fun, 'UseAttributes'),
                       extra_info = list(chunk_indices = as.integer(chunk_indices)), 
                       ncores = threads_compute),
                  params
-- 
GitLab


From 1725a56b4cb0fdc9b85fb5a1b500244fab8f01e7 Mon Sep 17 00:00:00 2001
From: Nicolau Manubens <nicolau.manubens@ic3.cat>
Date: Mon, 21 Jan 2019 01:02:39 +0100
Subject: [PATCH 3/3] Updated dependency to multiApply.

---
 DESCRIPTION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DESCRIPTION b/DESCRIPTION
index a7bdd2b..fce86c7 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -11,7 +11,7 @@ Imports:
     abind,
     bigmemory,
     future,
-    multiApply (>= 2.0.0),
+    multiApply (>= 2.1.1),
     parallel
 Suggests:
     easyNCDF,
-- 
GitLab