ByChunks <- function(fun,
                     chunks = list(dataset = 1, member = 1, sdate = 1, 
                                   ftime = 1,   lat = 1,    lon = 1), 
                     var, exp, obs, sdates, ...,
                     cluster = NULL, shared_dir = NULL, 
                     silent = FALSE, debug = FALSE){
  library(s2dverification)
  MergeArrays <- easyNCDF:::.MergeArrays

  params <- list(...)

  # Check fun
  if (!is.function(fun)) {
    stop("Parameter 'fun' must be a function.")
  }

  # Check cluster
  if (!is.null(cluster)) {
    cluster <- list(queue_host = 'bsceslogin01.bsc.es',
                    node_memory = NULL,
                    max_jobs = 6)
  }
  on_cluster <- !is.null(cluster)
  if (on_cluster) {
    if (Sys.which('ecflow_client') == '') {
      stop("ecFlow must be installed in order to run the computation on clusters.")
    }
  }

  # Check shared_dir
  if (on_cluster) {
    if (is.null(shared_dir)) {
      stop("Parameter 'shared_dir' must be specified when a cluster is specified.")
    } else {
      dir.create(shared_dir, recursive = TRUE)
      if (!dir.exists(shared_dir)) stop("Could not find or create the directory in ",
                                        "parameter 'shared_dir'.")
    }
  }

  # Check silent
  if (!is.logical(silent)) {
    stop("Parameter 'silent' must be logical.")
  }

  # Check debug
  if (!is.logical(debug)) {
    stop("Parameter 'debug' must be logical.")
  }
  if (silent) {
    debug <- FALSE
  }

  # Check chunks
  if (!is.list(chunks)) {
    stop("Parameter 'chunks' must be a named list.")
  }
  if (is.null(names(chunks))) {
    stop("Parameter 'chunks' must be a named list.")
  }
  new_chunks <- list(dataset = 1, member = 1, sdate = 1, 
                     ftime = 1,   lat = 1,    lon = 1)
  if (any(!(names(chunks) %in% names(new_chunks)))) {
    stop("All names in parameter 'chunks' must be one of 'dataset', 'member', ",
         "'sdate', 'ftime', 'lat' or 'lon'.")
  }
  if (any(!(((unlist(chunks) %% 1) == 0) | (unlist(chunks) == 'all')))) {
    stop("All values in parameter 'chunks' must take a numeric value or 'all'.")
  }
  if (any(unlist(chunks) < 1)) {
    stop("All values in parameter 'chunks' must be > 1.")
  }
  new_chunks[names(chunks)] <- chunks
  chunks <- new_chunks

  # Check no chunking along members requested
  if (chunks[['member']] != 1) {
    stop("Chunking along members not available yet.")
  }

  # Discover coordinates
  s2dverification:::.message(paste("Discovering grid size..."))
  params_to_leave <- c('leadtimemax', 'leadtimemin', 'nmember', 'nmemberobs')
  params_to_use <- params
  if (length(which(names(params) %in% params_to_leave)) > 0) {
    params_to_use <- params[-which(names(params) %in% params_to_leave)]
  }
  data_lonlat <- do.call("Load", 
                         c(
                           list(
                             var, exp[1], obs[1], 
                             sdates[1], leadtimemax = 1, 
                             nmember = 1, nmemberobs = 1, 
                             silent = !debug
                           ), 
                           params_to_use
                          )
                        )
  lons <- data_lonlat$lon
  lats <- data_lonlat$lat
  s2dverification:::.message(paste("Discovering forecast time steps..."))
  params_to_leave <- c('nmember', 'nmemberobs', 'lonmin', 'lonmax', 'latmin', 'latmax')
  params_to_use <- params
  if (length(which(names(params) %in% params_to_leave)) > 0) {
    params_to_use <- params[-which(names(params) %in% params_to_leave)]
  }
  data_ftime <- do.call("Load",
                        c(
                          list(
                            var, exp[1], obs[1], sdates[1], 
                            nmember = 1, nmemberobs = 1, 
                            lonmin = lons[1], lonmax = lons[1],
                            latmin = lats[1], latmax = lats[1], 
                            silent = !debug
                          ),
                          params_to_use
                         )
                       )
  s2dverification:::.message(paste("Discovering ensemble members..."))
  params_to_leave <- c('leadtimemax', 'leadtimemin', 
                       'lonmin', 'lonmax', 'latmin', 'latmax')
  params_to_use <- params
  if (length(which(names(params) %in% params_to_leave)) > 0) {
    params_to_use <- params[-which(names(params) %in% params_to_leave)]
  }
  data_members <- do.call("Load",
                          c(
                            list(
                              var, exp[1], obs[1], 
                              sdates[1], leadtimemax = 1,
                              lonmin = lons[1], lonmax = lons[1],
                              latmin = lats[1], latmax = lats[1], 
                              silent = !debug
                            ), 
                            params_to_use
                           )
                         )
  if (!is.null(exp)) {
    nftime <- dim(data_ftime$mod)['ftime']
    nmemb_exp <- dim(data_ftime$mod)['member']
  } else {
    nftime <- dim(data_ftime$obs)['ftime']
    nmemb_obs <- dim(data_ftime$obs)['member']
  }
  if (is.null(params[['leadtimemin']])) {
    ftimes <- 1:nftime
  } else {
    ftimes <- (1:nftime) + params[['leadtimemin']] - 1
  }

  # Replace 'all's
  if (chunks[['dataset']] == 'all') {
    chunks[['dataset']] <- length(exp)
  }
  if (chunks[['member']] == 'all') {
    chunks[['dataset']] <- nmemb_exp
  }
  if (chunks[['sdate']] == 'all') {
    chunks[['sdate']] <- length(sdates)
  }
  if (chunks[['ftime']] == 'all') {
    chunks[['ftime']] <- nftime
  }
  if (chunks[['lat']] == 'all') {
    chunks[['lat']] <- length(lats)
  }
  if (chunks[['lon']] == 'all') {
    chunks[['lon']] <- length(lons)
  }

  # Empty array to collect results
  result <- list(mod = NULL, obs = NULL)

  # Mount the ecFlow suite
  if (on_cluster) {
    # Copy load_process_save_chunk.R into shared folder
    chunk_script <- file(system.file('chunking/load_process_save_chunk.R', 
                                     package = 's2dverification'))
    chunk_script_lines <- readLines(chunk_script)
    chunk_script_lines <- gsub('^var <- *', paste0('var <- ', paste(deparse(var), collapse = '\n')), 
                               chunk_script_lines)
    chunk_script_lines <- gsub('^obs <- *', paste0('obs <- ', paste(deparse(obs), collapse = '\n')), 
                               chunk_script_lines)
    chunk_script_lines <- gsub('^shared_dir <- *', paste0('shared_dir <- ', 
                               paste(deparse(shared_dir), collapse = '\n')), chunk_script_lines)
    chunk_script_lines <- gsub('^debug <- *', paste0('debug <- ', paste(deparse(debug), collapse = '\n')), 
                               chunk_script_lines)
    params_to_leave <- c('leadtimemax', 'leadtimemin', 
                         'lonmin', 'lonmax', 'latmin', 'latmax')
    params_to_use <- params
    if (length(which(names(params) %in% params_to_leave)) > 0) {
      params_to_use <- params[-which(names(params) %in% params_to_leave)]
    }
    chunk_script_lines <- gsub('^params <- *', paste0('params <- ', 
                               paste(deparse(params_to_use), collapse = '\n')), chunk_script_lines)
    chunk_script_lines <- gsub('^fun <- *', paste0('fun <- ', paste(deparse(fun), collapse = '\n')), 
                               chunk_script_lines)
    original_selectors <- vector('list', 6)
    names(original_selectors) <- c('dataset', 'member', 'sdate', 'ftime', 'lat', 'lon')
    original_selectors[['dataset']] <- exp
    original_selectors[['sdate']] <- sdates
    original_selectors[['ftime']] <- ftimes
    original_selectors[['lat']] <- lats
    original_selectors[['lon']] <- lons
    chunk_script_lines <- gsub('^original_selectors <- *', 
                               paste0('original_selectors <- ', 
                                      paste(deparse(original_selectors), collapse = '\n')), 
                               chunk_script_lines)
    writeLines(chunk_script_lines, paste0(shared_dir, '/load_process_save_chunk.R'))
  
    # Copy Chunk.ecf into shared folder
    #TODO: Modify chain of params sent to r script in function of 
    #array dimensions: %DIM_CHUNK_INDEX% %DIM_MIN% %DIM_MAX%
    chunk_ecf_script <- file(system.file('chunking/Chunk.ecf',
                                         package = 's2dverification'))
    chunk_ecf_script_lines <- readLines(chunk_ecf_script)
    writeLines(chunk_ecf_script_lines, paste0(shared_dir, '/Chunk.ecf'))

    # Copy merge_chunks.R into tmp folder
    merge_script <- file(system.file('chunking/merge_chunks.R',
                                     package = 's2dverification'))
    merge_script_lines <- readLines(merge_script)
    merge_script_lines <- gsub('^shared_dir <- *', paste0('shared_dir <- ', 
                               paste(deparse(shared_dir), collapse = '\n')), merge_script_lines)
    writeLines(merge_script_lines, paste0(shared_dir, '/merge_chunks.R'))
  
    # Copy Merge.ecf into tmp folder
    #TODO: Modify chain of parameters sent to r script when merging 
    #chunks progressively
    merge_ecf_script <- file(system.file('chunking/Merge.ecf',
                                         package = 's2dverification'))
    merge_ecf_script_lines <- readLines(merge_ecf_script)
    writeLines(merge_ecf_script_lines, paste0(shared_dir, '/Merge.ecf')) 

    # Copy headers
    file.copy(system.file('chunking/head.h', package = 's2dverification'),
              shared_dir)
    file.copy(system.file('chunking/tail.h', package = 's2dverification'),
              shared_dir)
    file.copy(system.file('chunking/slurm.h', package = 's2dverification'),
              shared_dir)
 }

  add_line <- function(suite, line, tabs) {
    c(suite, paste0(paste(rep(' ', tabs), collapse = ''), line))
  }
  suite <- NULL
  tabs <- 0
  suite <- add_line(suite, 'suite S2DV_CHUNKING', tabs)
  tabs <- tabs + 2
  suite <- add_line(suite, paste0("edit ECF_HOME '", shared_dir, "'"), tabs)
  suite <- add_line(suite, paste0("edit ECF_FILES '", shared_dir, "'"), tabs)
  suite <- add_line(suite, paste0("edit ECF_INCLUDE '", shared_dir, "'"), tabs)
  suite <- add_line(suite, paste0("limit max_jobs ", cluster[['max_jobs']]), tabs)
  suite <- add_line(suite, paste0("inlimit max_jobs"), tabs)
  suite <- add_line(suite, paste0("edit HOST '", cluster[['queue_host']], "'"), tabs)
  suite <- add_line(suite, "edit ECF_JOB_CMD 'ssh %HOST% 'sbatch %ECF_JOB% > %ECF_JOBOUT% 2>&1 &''", tabs)
  suite <- add_line(suite, "family computation", tabs)
  tabs <- tabs + 2

  # Iterate through chunks
  chunk_counter <- 0
  if (chunks[['dataset']] > length(exp)) {
    warning("Too many 'dataset' chunks requested. Reducing from ",
            chunks[['dataset']], " to ", length(exp), ".")
    chunks[['dataset']] <- length(exp)
  }
  datasets_per_chunk <- ceiling(length(exp) / chunks[['dataset']])
  dataset_index <- 1
  result <- list(mod = NULL, obs = NULL)
  for (dataset in 1:chunks[['dataset']]) {
    dataset_min <- (1:length(exp))[dataset_index]
    dataset_max <- (1:length(exp))[min(length(exp), dataset_index + datasets_per_chunk - 1)]
    suite <- add_line(suite, paste0('family ', 'dataset_', dataset), tabs)
    tabs <- tabs + 2
    suite <- add_line(suite, paste0('edit ', 'DATASET ', dataset), tabs)
    suite <- add_line(suite, paste0('edit ', 'DATASET_N ', chunks[['dataset']]), tabs)
    suite <- add_line(suite, paste0('edit ', 'DATASET_MIN ', dataset_min), tabs)
    suite <- add_line(suite, paste0('edit ', 'DATASET_MAX ', dataset_max), tabs)

    if (chunks[['member']] > nmemb_exp) {
      warning("Too many 'member' chunks requested. Reducing from ",
              chunks[['member']], " to ", nmemb_exp, ".")
      chunks[['member']] <- nmemb_exp
    }
    members_per_chunk <- ceiling(nmemb_exp / chunks[['member']])
    member_index <- 1
    res_member <- list(mod = NULL, obs = NULL)
    for (member in 1:chunks[['member']]) {
      member_min <- (1:nmemb_exp)[member_index]
      member_max <- (1:nmemb_exp)[min(nmemb_exp, member_index + members_per_chunk - 1)]
      suite <- add_line(suite, paste0('family ', 'member_', member), tabs)
      tabs <- tabs + 2
      suite <- add_line(suite, paste0('edit ', 'MEMBER ', member), tabs)
      suite <- add_line(suite, paste0('edit ', 'MEMBER_N ', chunks[['member']]), tabs)
      suite <- add_line(suite, paste0('edit ', 'MEMBER_MIN ', member_min), tabs)
      suite <- add_line(suite, paste0('edit ', 'MEMBER_MAX ', member_max), tabs)
  
      if (chunks[['sdate']] > length(sdates)) {
        warning("Too many 'sdate' chunks requested. Reducing from ",
                chunks[['sdate']], " to ", length(sdates), ".")
                chunks[['sdate']] <- length(sdate)
      }
      sdates_per_chunk <- ceiling(length(sdates) / chunks[['sdate']])
      sdate_index <- 1
      res_sdate <- list(mod = NULL, obs = NULL)
      for (sdate in 1:chunks[['sdate']]) {
        sdate_min <- (1:length(sdates))[sdate_index]
        sdate_max <- (1:length(sdates))[min(length(sdates), sdate_index + sdates_per_chunk - 1)]    
        suite <- add_line(suite, paste0('family ', 'sdate_', sdate), tabs)
        tabs <- tabs + 2
        suite <- add_line(suite, paste0('edit ', 'SDATE ', sdate), tabs)
        suite <- add_line(suite, paste0('edit ', 'SDATE_N ', chunks[['sdate']]), tabs)
        suite <- add_line(suite, paste0('edit ', 'SDATE_MIN ', sdate_min), tabs)
        suite <- add_line(suite, paste0('edit ', 'SDATE_MAX ', sdate_max), tabs)

        if (chunks[['ftime']] > nftime) {
          warning("Too many 'ftime' chunks requested. Reducing from ",
                  chunks[['ftime']], " to ", nftime, ".")
          chunks[['ftime']] <- nftime
        }
        ftimes_per_chunk <- ceiling(nftime / chunks[['ftime']])
        ftime_index <- 1
        res_ftime <- list(mod = NULL, obs = NULL)
        for (ftime in 1:chunks[['ftime']]) {
          ftime_min <- (1:nftime)[ftime_index]
          ftime_max <- (1:nftime)[min(nftime, ftime_index + ftimes_per_chunk - 1)]
          suite <- add_line(suite, paste0('family ', 'ftime_', ftime), tabs)
          tabs <- tabs + 2
          suite <- add_line(suite, paste0('edit ', 'FTIME ', ftime), tabs)
          suite <- add_line(suite, paste0('edit ', 'FTIME_N ', chunks[['ftime']]), tabs)
          suite <- add_line(suite, paste0('edit ', 'FTIME_MIN ', ftime_min), tabs)
          suite <- add_line(suite, paste0('edit ', 'FTIME_MAX ', ftime_max), tabs)

          if (chunks[['lat']] > length(lats)) {
            warning("Too many 'lat' chunks requested. Reducing from ",
                    chunks[['lat']], " to ", length(lats), ".")
            chunks[['lat']] <- length(lats)
          }
          lats_per_chunk <- ceiling(length(lats) / chunks[['lat']])
          lat_index <- 1
          res_lat <- list(mod = NULL, obs = NULL)
          for (lat in 1:chunks[['lat']]) {
            lat_max <- lat_index
            lat_min <- min(length(lats), lat_index + lats_per_chunk - 1)
            suite <- add_line(suite, paste0('family ', 'lat_', lat), tabs)
            tabs <- tabs + 2
            suite <- add_line(suite, paste0('edit ', 'LAT ', lat), tabs)
            suite <- add_line(suite, paste0('edit ', 'LAT_N ', chunks[['lat']]), tabs)
            suite <- add_line(suite, paste0('edit ', 'LAT_MIN ', lat_min), tabs)
            suite <- add_line(suite, paste0('edit ', 'LAT_MAX ', lat_max), tabs)

            if (chunks[['lon']] > length(lons)) {
              warning("Too many 'lon' chunks requested. Reducing from ",
                      chunks[['lon']], " to ", length(lons), ".")
              chunks[['lon']] <- length(lons)
            }
            lons_per_chunk <- ceiling(length(lons) / chunks[['lon']])
            lon_index <- 1
            res_lon <- list(mod = NULL, obs = NULL)
            for (lon in 1:chunks[['lon']]) {
              lon_min <- lon_index
              lon_max <- min(length(lons), lon_index + lons_per_chunk - 1)
              suite <- add_line(suite, paste0('family ', 'lon_', lon), tabs)
              tabs <- tabs + 2
              suite <- add_line(suite, paste0('edit ', 'LON ', lon), tabs)
              suite <- add_line(suite, paste0('edit ', 'LON_N ', chunks[['lon']]), tabs)
              suite <- add_line(suite, paste0('edit ', 'LON_MIN ', lon_min), tabs)
              suite <- add_line(suite, paste0('edit ', 'LON_MAX ', lon_max), tabs)

              # ADD CHUNK SCRIPT TO SUITE
              suite <- add_line(suite, "task Chunk", tabs)
              tabs <- tabs - 2
              suite <- add_line(suite, paste0('endfamily'), tabs)

              if (!on_cluster) {
                if (!silent) {
                  s2dverification:::.message(paste("Loading chunk", chunk_counter + 1, 
                                             "out of", prod(unlist(chunks)), "..."))
                }
                params_to_leave <- c('leadtimemax', 'leadtimemin', 
                                     'latmin', 'latmax', 'lonmin', 'lonmax')
                params_to_use <- params
                if (length(which(names(params) %in% params_to_leave)) > 0) {
                  params_to_use <- params[-which(names(params) %in% params_to_leave)]
                }
                data <- do.call("Load",
                                c(
                                  list(
                                    var, exp[dataset_min:dataset_max], obs,
                                    sdates[sdate_min:sdate_max], 
                                    leadtimemin = ftimes[ftime_min], 
                                    leadtimemax = ftimes[ftime_max],
                                    latmin = lats[lat_min], 
                                    latmax = lats[lat_max],
                                    lonmin = lons[lon_min], 
                                    lonmax = lons[lon_max], 
                                    silent = !debug
                                  ), 
                                  params_to_use
                                 )
                               )
  
                if (!silent) {
                  s2dverification:::.message(paste("Processing..."))
                }
  
                res <- fun(data)
  
                rm(data)
                gc()
  
                if (chunks[['lon']] > 1) {
                  res_lon$mod <- MergeArrays(res_lon$mod, res$mod, 'lon')
                  res_lon$obs <- MergeArrays(res_lon$obs, res$obs, 'lon')
                } else {
                  res_lon <- res
                }
                rm(res)
                gc()
              }
              chunk_counter <- chunk_counter + 1
              lon_index <- lon_index + lons_per_chunk              
            }
            tabs <- tabs - 2
            suite <- add_line(suite, paste0('endfamily'), tabs)
            if (!on_cluster) {
              if (chunks[['lat']] > 1) {
                res_lat$mod <- MergeArrays(res_lat$mod, res_lon$mod, 'lat')
                res_lat$obs <- MergeArrays(res_lat$obs, res_lon$obs, 'lat')
              } else {
                res_lat <- res_lon
              }
              rm(res_lon)
              gc()
            }
            lat_index <- lat_index + lats_per_chunk 
          }
          tabs <- tabs - 2
          suite <- add_line(suite, paste0('endfamily'), tabs)
          if (!on_cluster) {
            if (chunks[['ftime']] > 1) {
              res_ftime$mod <- MergeArrays(res_ftime$mod, res_lat$mod, 'ftime')
              res_ftime$obs <- MergeArrays(res_ftime$obs, res_lat$obs, 'ftime')
            } else {
              res_ftime <- res_lat
            }
            rm(res_lat)
            gc()
          }
          ftime_index <- ftime_index + ftimes_per_chunk 
        }
        tabs <- tabs - 2
        suite <- add_line(suite, paste0('endfamily'), tabs)
        if (!on_cluster) {
          if (chunks[['sdate']] > 1) {
            res_sdate$mod <- MergeArrays(res_sdate$mod, res_ftime$mod, 'sdate')
            res_sdate$obs <- MergeArrays(res_sdate$obs, res_ftime$obs, 'sdate')
          } else {
            res_sdate <- res_ftime
          }
          rm(res_ftime)
          gc()
       }
       sdate_index <- sdate_index + sdates_per_chunk 
      }
      tabs <- tabs - 2
      suite <- add_line(suite, paste0('endfamily'), tabs)
      if (!on_cluster) {
        if (chunks[['member']] > 1) {
          res_member$mod <- MergeArrays(res_member$mod, res_sdate$mod, 'member')
          res_member$obs <- MergeArrays(res_member$obs, res_sdate$obs, 'member')
        } else {
          res_member <- res_sdate
        }
        rm(res_sdate)
        gc()
      }
      member_index <- member_index + members_per_chunk 
    }
    tabs <- tabs - 2
    suite <- add_line(suite, paste0('endfamily'), tabs)
    if (!on_cluster) {
      if (chunks[['dataset']] > 1) {
        result$mod <- MergeArrays(result$mod, res_member$mod, 'dataset')
        result$obs <- MergeArrays(result$obs, res_member$obs, 'dataset')
      } else {
        result <- res_member
      }
      rm(res_member)
      gc()
    }
    dataset_index <- dataset_index + datasets_per_chunk 
  }

  tabs <- tabs - 2
  suite <- add_line(suite, paste0('endfamily'), tabs)

  # Close the ecFlow suite
  suite <- add_line(suite, "family merge", tabs)
  tabs <- tabs + 2
  suite <- add_line(suite, "trigger computation == complete", tabs)
  suite <- add_line(suite, "edit ECF_JOB_CMD 'bash %ECF_JOB% > %ECF_JOBOUT% 2>&1 &'", tabs)
  suite <- add_line(suite, "task Merge", tabs)
  tabs <- tabs - 2
  suite <- add_line(suite, paste0('endfamily'), tabs)

  tabs <- tabs - 2
  suite <- add_line(suite, "endsuite", tabs)

  # Run ecFlow suite if needed
  if (on_cluster) {
    suite_file <- paste0(shared_dir, '/s2dv_chunking.def')
    suite_file_o <- file(suite_file)
    writeLines(suite, suite_file_o)
    close(suite_file_o)
    system("ecflow_start.sh -p 5678")
    system(paste0("ecflow_client --load=", suite_file, ' --port=5678'))
    system("ecflow_client --begin=S2DV_CHUNKING --port=5678")
    done <- FALSE
    while (!done) {
      Sys.sleep(1)
      status <- system("ecflow_client --get_state=S2DV_CHUNKING --port=5678", 
                       intern = TRUE)
      if (any(grepl("suite S2DV_CHUNKING #.* state:complete", status))) {
        done <- TRUE
      }
    }
    system("ecflow_client --delete=/S2DV_CHUNKING --port=5678")
    #system("ecflow_client --shutdown --port=5678")
    #system("ecflow_stop.sh -p 5678")
    result <- readRDS(paste0(shared_dir, '/result.Rds'))
    file.remove(paste0(shared_dir, '/result.Rds'))
  }
  result[['lon']] <- lons
  result[['lat']] <- lats

  result
}
