compute_workflow <- function(recipe, hcst, obs, fcst = NULL,
                             nchunks = chunk_indices,
                             expected_output_dims) {
  # Load modules
  source("modules/Preprocessing/Preprocessing.R")
  source("modules/Units/Units.R")
  source("modules/Aggregation/Aggregation.R")
  source("tools/tmp/as.s2dv_cube.R")

  recipe$Run$logger <- log4r::logger(threshold = recipe$Run$Loglevel,
                                     appenders = list(console_appender(layout = .custom_log_layout())))
  modules <- tolower(strsplit(recipe$Run$startR_workflow$modules,
                              ", | |,")[[1]])
  return_datasets <- strsplit(recipe$Run$startR_workflow$return, ", | |,")[[1]]

  ## TODO: Decide on file path
  # for (module in modules) {
  #   module_path <- file.path("modules", "Crossval", paste0(module, ".R"))
  #   source(module_path)
  # }
  source("modules/Crossval/Crossval_anomalies.R")
  source("modules/Crossval/Crossval_metrics.R")
  source("modules/old_modules/Statistics/Statistics.R")
  # Define appender with custom log layout so that it knows not to append
  # within Compute().
  # Create data list
  data <- list(hcst = hcst,
               obs = obs,
               fcst = fcst)
  
  data <- data[!sapply(data, is.null)]
  var_name <- as.vector(attributes(hcst)$FileSelectors[[1]]$var[[1]])
  # Remove duplicated objects
  rm(hcst, obs, fcst)
  # Pre-process datasets
  data <- preprocess_datasets(recipe, data)
  # Transform data to s2dv_cube
  data <- lapply(data, 
                 function(x) {
                   class(x) <- "startR_array"
                   as.s2dv_cube(x)
                 })
  for (cube_idx in seq(1:length(data))) {
    data[[cube_idx]]$attrs$Variable$varName <- var_name
  }
  ## TODO: Find a way to dynamically define outputs
  skill_metrics <- NULL
  probabilities <- NULL
  statistics <- NULL
  # Change units
  data <- Units(recipe, data, retrieve = F)
  # Call workflows
  for (module in modules) {
    if (module == "aggregation") {
      data <- Aggregation(recipe, data, retrieve = F, nchunks = nchunks)
    } else if (module == "crossval_anomalies") {
      data <- Crossval_anomalies(recipe = recipe, data = data, retrieve = FALSE)
    } else if (module == "crossval_metrics") {
      ## TODO: These parameters need to be in the recipe
      skill_metrics <- Crossval_metrics(recipe = recipe, data_crossval = data,
                                        retrieve = FALSE,
                                        fair = FALSE, nmemb = NULL,
                                        nmemb_ref = NULL, nchunks = nchunks)
    } else if (module == "statistics") {
      statistics <- Statistics(recipe = recipe, data = data, retrieve = FALSE,
                               nchunks = nchunks)
    }
  }

  ## TODO: Define what to return depending the modules called + the recipe
  return_list <- list(hcst = data$hcst$data,
                      fcst = data$fcst$data,
                      obs = data$obs$data,
                      skill = skill_metrics,
                      statistics = statistics,
                      probabilities = probabilities)
  # Eliminate NULL elements from the return list
  return_list <- return_list[!sapply(return_list, is.null)]
  for (element in names(data)[!names(data) %in% return_datasets]) {
    return_list[element] <- NULL
  }
  # Reorder dimensions as expected
  for (element in names(return_list)) {
    return_list[[element]] <- s2dv::Reorder(data = return_list[[element]],
                                            order = expected_output_dims[[element]])
  }
  return(return_list)
}

convert_to_s2dv_cube <- function(new_cube, original_cube) {
  attr(new_cube, "Variables") <- attr(original_cube, "Variables")
  attr(new_cube, "FileSelectors") <- attr(original_cube, "FileSelectors")
  attr(new_cube, "Files") <- attr(original_cube, "Files")

  class(new_cube) <- "startR_array"
  new_cube <- as.s2dv_cube(new_cube)
  return(new_cube)
}

run_compute_workflow <- function(recipe, data, last_module = NULL) {
  
  # ---------------------------------------------------------------------------
  # Step 1: Retrieve the modules that will be called inside the workflow

  modules <- tolower(strsplit(recipe$Run$startR_workflow$modules, ", | |,")[[1]])
  return_datasets <- strsplit(recipe$Run$startR_workflow$return, ", | |,")[[1]]
  
  # ---------------------------------------------------------------------------
  # Step 2: Define the inputs and outputs for Compute()
  
  # Reorder data: hcst, obs, fcst
  data <- data[c("hcst", "obs", "fcst")]
  data <- data[!sapply(data, is.null)]
  # Retrieve original dimensions
  chunking_dims <- names(recipe$Run$startR_workflow$chunk_along)
  target_dims <- sapply(names(data), function(x) NULL)
  inputs <- sapply(names(data), function(x) NULL)
  input_attributes <- sapply(names(data), function(x) NULL)
  STARTR_CUBE_ATTRS <- c("Variables", "Dimensions", "FileSelectors", "Files")
  # Create output dimensions
  ## TODO: Improve this condition
  if ("indices" %in% modules) {
    spatial_output_dims <- c("region")
  } else {
    spatial_output_dims <- c("latitude", "longitude")
  }
  
  FIXED_OUTPUT_DIMS <- c("dat", "var", "syear", "sday", "sweek", "time", "ensemble")
  expected_output_dims <- c(FIXED_OUTPUT_DIMS, spatial_output_dims)
  expected_output_dims <- expected_output_dims[!expected_output_dims %in% chunking_dims]
  # Default outputs: hindcast and observations
  output_dims <- sapply(names(data), function(x) NULL)
  # Iterate over datasets to define inputs and outputs 
  for (dataset in names(data)) {
    # Retrieve original dimensions
    dataset_dims <- names(attr(data[[dataset]], "Dimensions"))
    # Remove the chunking dimensions from the original dimensions in the arrays,
    # to create the target dimensions
    target_dims[[dataset]] <- dataset_dims[!dataset_dims %in% chunking_dims]
    # Define inputs
    inputs[[dataset]] <- data[[dataset]]
    # Define input attributes 
    input_attributes[[dataset]] <- STARTR_CUBE_ATTRS
    # Define output dimensions
    if (dataset %in% return_datasets) {
      output_dims[[dataset]] <- expected_output_dims
    } else {
      output_dims[[dataset]] <- NULL
    }
  }

  ## TODO: Automatize and refactor
  # Add skill metrics if skill module is called
  if (any(c("skill", "crossval_metrics") %in% modules)) {
    skill_dims <- c('metric', 'var', 'time', spatial_output_dims)
    skill_output_dims <- skill_dims[!skill_dims %in% chunking_dims]
    output_dims <- c(output_dims,
                     list(skill = skill_output_dims))
  }
  # Add statistics if statistics module is called
  if ("statistics" %in% modules) {
    statistics_dims <- c('metric', 'var', 'time', spatial_output_dims)
    statistics_output_dims <- statistics_dims[!statistics_dims %in% chunking_dims]
    output_dims <- c(output_dims,
                     list(statistics = statistics_output_dims))
  }
  ## TODO: Add Indices (Niño1+2, Niño3, Niño3.4, Niño4, NAO)
  ## Instead of having the index dimension, we could add a second
  ## loop and then define outputs dynamically based on the recipe;
  ## one s2dv_cube per index. At least in the case of ENSO.
  # if ("indices" %in% modules) {
  #   idx_dims <- c('index', 'var', 'syear', 'time', 'region')
  # }

  # ---------------------------------------------------------------------------
  # Step 3: Generate the Step and call Compute()
  # Libraries to be loaded when running Compute()
  LIBS_LOAD <- c("log4r", "docopt", "ClimProjDiags", "multiApply", "yaml",
                 "s2dv", "abind", "easyNCDF", "CSTools", "lubridate", "PCICt",
                 "RColorBrewer", "configr", "sf", "ggplot2", "rnaturalearth",
                 "cowplot", "stringr", "pryr")

  step <- Step(fun = compute_workflow,
               target_dims = target_dims,
               output_dims = output_dims,
               use_attributes = input_attributes, 
               use_libraries = LIBS_LOAD)

  wf <- AddStep(inputs = inputs,
                step = step,
                recipe = recipe,
                expected_output_dims = output_dims)
  ## TODO: Remove when startR is fixed (startR issue #109)
  if (!any(c('startR_cube', 'startR_workflow') %in% class(wf))) {
    wf <- wf[[1]]
  }

  #---------------------------------------
  ## TODO: Create function to generate call to Compute()?
  # Compute locally
  run_on <- recipe$Run$startR_workflow$run_on

  r_module_ver <- list(local = "R/4.1.2-foss-2015a-bare",
                       as_machine = "R/4.1.2-foss-2015a-bare",
                       nord3 = "R/4.1.2-foss-2019b")
  cdo_module_ver <- list(local = "CDO/1.9.8-foss-2015a",
                         as_machine = "CDO/1.9.8-foss-2015a",
                         nord3 = "CDO/1.9.8-foss-2019b")

  if (run_on == 'local') {
    # Compute locally, in serial
    res <- Compute(wf,
                   chunks = recipe$Run$startR_workflow$chunk_along,
                   threads_compute = recipe$Analysis$ncores,
                   threads_load = 1) # recipe$Analysis$ncores/2)
  } else if (run_on %in% c('nord3', 'as_machine')) {
    #NOTE: autosubmit_suite_dir can be anywhere
    ## TODO: Change code_dir to autosubmit dir when necessary
    autosubmit_suite_dir <- recipe$Run$code_dir
    if (!is.null(recipe$Run$startR_workflow$slurm_directives)) {
      extra_queue_params <- as.list(recipe$Run$startR_workflow$slurm_directives)
    } else {
      extra_queue_params <- NULL
    }
    ## TODO: Review the core/thread setting
    # Compute in cluster/AS machine, in parallel, with autosubmit
    res <- Compute(wf,
                   chunks = recipe$Run$startR_workflow$chunk_along,
                   threads_compute = recipe$Analysis$ncores, 
                   threads_load = 2,
                   cluster = list(
                     queue_host = run_on, # name in platforms.yml
                     r_module = r_module_ver[[run_on]],
                     CDO_module = cdo_module_ver[[run_on]],
                     init_commands = list('module load GDAL PROJ GEOS'),
                     autosubmit_module = 'autosubmit/4.0.0b-foss-2015a-Python-3.7.3',
                     cores_per_job = recipe$Analysis$ncores,
                     job_wallclock = recipe$Run$startR_workflow$chunk_wallclock,
                     max_jobs = prod(unlist(recipe$Run$startR_workflow$chunk_along)),
                     polling_period = 10,
                     extra_queue_params = extra_queue_params,
                     expid = recipe$Run$startR_workflow$expid,
                     hpc_user = recipe$Run$startR_workflow$hpc_user,
                     run_dir = recipe$Run$code_dir
                   ),
                   workflow_manager = 'autosubmit', # 'ecFlow'
                   autosubmit_suite_dir = autosubmit_suite_dir,
                   autosubmit_server = NULL, #or 'bscesautosubmit02',
                   wait = TRUE
                   )
  }
  info(recipe$Run$logger, retrieve = TRUE,
       "##### COMPUTE SECTION ENDED, REFORMATTING OUTPUT #####")

  # ---------------------------------------------------------------------------
  # Step 4: Convert results to s2dv_cube/list format
  source("tools/retrieve_metadata.R")
  for (cube in c("hcst", "obs", "fcst")) {
   if (!is.null(res[[cube]])) {
     if (!is.null(last_module)) {
       tmp_dir <- file.path(recipe$Run$output_dir, "outputs", "tmp",
                            str_to_title(last_module))
       res$data[[cube]]  <- retrieve_metadata(tmp_dir = tmp_dir,
                                              chunks = recipe$Run$startR_workflow$chunk_along,
                                              array_dims = dim(res[[cube]]),
                                              metadata_file_pattern = paste0(last_module,
                                                                             "_metadata_",
                                                                             cube))
       res$data[[cube]]$data <- res[[cube]]
     } else {
        res$data[[cube]] <- convert_to_s2dv_cube(res[[cube]], data[[cube]])
     }
   } else {
     res$data[[cube]] <- NULL
   }
   res[[cube]] <- NULL
  }

  ## TODO: Replicate for probabilities and other modules; refactor
  if (!is.null(res$skill)) {
    tmp_dir <- file.path(recipe$Run$output_dir, "outputs", "tmp", "Skill")
    metric_list <- readRDS(file.path(tmp_dir, "metric_names.Rds"))
    res$skill <- ClimProjDiags::ArrayToList(res$skill,
                                            dim = 'metric',
                                            level = 'list',
                                            names = metric_list)
    # Put chunked metadata back together
    res$skill$metadata <- retrieve_metadata(tmp_dir = tmp_dir,
                                            chunks = recipe$Run$startR_workflow$chunk_along,
                                            array_dims = dim(res$skill[[1]]),
                                            metadata_file_pattern = "metric_metadata")
  }
  if (!is.null(res$statistics)) {
    tmp_dir <- file.path(recipe$Run$output_dir, "outputs", "tmp", "Statistics")
    metric_list <- readRDS(file.path(tmp_dir, "metric_names.Rds"))
    res$statistics <- ClimProjDiags::ArrayToList(res$statistics,
                                                 dim = 'metric',
                                                 level = 'list',
                                                 names = metric_list)
    res$statistics$metadata <- retrieve_metadata(tmp_dir = tmp_dir,
                                                 chunks = recipe$Run$startR_workflow$chunk_along,
                                                 array_dims = dim(res$statistics[[1]]),
                                                 metadata_file_pattern = "metric_metadata")
  }
   
  # ---------------------------------------------------------------------------
  # Step 5: Remove temporary files
  unlink(file.path(recipe$Run$output_dir, "outputs", "tmp"), recursive = TRUE)
  
  # ---------------------------------------------------------------------------
  # Step 6: Save data
  ## PROBLEM: Defining the output dir
  ## TODO: Generalize
  # Saving(recipe = recipe, data = res$data, skill_metrics = res$skill)
  if (!is.null(res$skill) && recipe$Analysis$Workflow$Skill$save == 'all') {
    save_metrics(recipe = recipe, metrics = res$skill,
                 agg = "global", module = "Skill")
  }
  # ---------------------------------------------------------------------------
  # Step 7: Return outputs
  res <- res[!sapply(res, is.null)]
  info(recipe$Run$logger, retrieve = TRUE,
       "##### DATA RETURNED AS A NAMED LIST #####")
  info(recipe$Run$logger, retrieve = TRUE,
       paste0("Steps performed: ", recipe$Run$startR_workflow$modules))
  return(res)
}
