source('~/s2dverification/R/Utils.R')
library(ncdf4)
library(abind)

.CheckSelectors <- function(selectors, var = NULL, return_indices = TRUE) {
  if (length(selectors) == 0) {
    stop("No selectors provided in 'selectors'.")
  }
  if (return_indices) {
    if (is.list(selectors)) {
      if (length(selectors) != 2) {
        stop("'selectors' provided in a wrong format.")
      }
      for (i in 1:length(selectors)) {
        if (is.null(var)) {
          stop("No selector values provided in 'var'.")
        } else if (is.na(selectors[[i]])) {
          if (i == 1) {
            selectors[[i]] <- 1
          } else {
            selectors[[i]] <- length(var)
          }
        } else if (is.character(selectors[[i]])) {
          if (is.character(var)) {
            candidate <- which(var == selectors[[i]])
            if (length(candidate) > 0) {
              selectors[[i]] <- candidate[1]
            } else {
              stop("Selector value not found in 'var'.")
            }
          } else {
            stop("Character selectors provided but possible values in 'var' are not character.")
          }
        } else if (is.numeric(selectors[[i]])) {
          if (is.numeric(var)) {
            if (sum(var == selectors[[i]], na.rm = TRUE) > 0) {
              selectors[[i]] <- which(var == selectors[[i]])[1]
            } else {
              print(paste0("Warning: a numeric selector has been ",
                           "provided for a dimension defined along a ",
                           "numeric variable, but no exact match ",
                           "found. Taking the index of the nearest ",
                           "value."))
              selectors[[i]] <- which.min(abs(var - selectors[[i]]))
            }
          } else {
            stop("Numeric selectors provided but possible values in 'var' are not numeric.")
          }
        }
      }
      selectors[[1]]:selectors[[2]]
    } else if (is.numeric(selectors)) {
      if (is.null(var)) {
        selectors
      } else {
        if (is.numeric(var)) {
          if (!all(selectors %in% var)) {
            print(paste0("Warning: numeric selectors have been ",
               "provided for a dimension defined along a ",
               "numeric variable, but no exact match ",
               "found for all the selectors. Taking the index of the ",
               "nearest values."))
          }
          sapply(selectors, function(x) which.min(abs(var - x))[1])
        } else {
          stop("Numeric selectors provided but possible values in 'var' are not numeric.")
        }
      }
    } else if (is.character(selectors)) {
      if (is.null(var)) {
        stop("No selector values provided in 'var'.")
      } else {
        if ((length(selectors) == 1) && 
            (selectors %in% c('all', 'first', 'last'))) {
          if (selectors == 'all') {
            1:length(var)
          } else if (selectors == 'first') {
            1
          } else {
            length(var)
          }
        } else {
          if (!is.character(var)) {
            stop("Character selectors provided but possible values in 'var' are not character.")
          }
          candidates <- which(var == selectors)
          if (length(candidates) == 0) {
            stop("Selectors do not match values in 'var'.")
          } else if (length(candidates) != length(selectors)) {
            stop("Some selectors do not match values in 'var'.")
          }
          candidates
        }
      }
    }
  } else {
    if (is.list(selectors)) {
    } else if (is.numeric(selectors)) {
      if (length(selectors) > 0) {
        if (!is.null(var)) {
          var[selectors]
        }
      } else {
        stop("No selectors provided.")
      }
    } else if (is.character(selectors)) {
      if ((length(selectors) == 1) && 
          (selectors %in% c('all', 'first', 'last'))) {
        if (selectors == 'all') {
          var
        } else if (selectors == 'first') {
          head(var, 1)
        } else {
          tail(var, 1)
        }
      } else {
        selectors
      }
    }
  }
}

## TODO: Somehow reorder lons and lats
.NcOpener <- function(file_name) {
  result <- NULL
  
  try({
    result <- nc_open(file_name)
  }, silent = TRUE)
  
  result
}

.NcVarReader <- function(file_name = NULL, file_object = NULL, var_name = NULL) {
  if (is.null(file_object)) {
    if (is.null(file_name)) {
      stop("Either 'file_name' or 'file_object' must be provided.")
    }
    file_object <- .NcOpener(file_name)
  } else {
    file_name <- file_object$filename
  }
  if (is.null(var_name)) {
    if (length(fnc$var) == 0) {
      stop(paste0("No variables found in ", file_name, "."))
    } else if (length(fnc$var) > 1) {
      stop("A 'var_name' must be provided if the file contains more than one variables.")
    } else {
      var_name <- names(fnc$var)[1]
    }
  }

  result <- NULL
  try({
    result <- ncvar_get(file_object, var_name)
    units <- file_object$var[[var_name]]$units
    if (units %in% c('seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years')) {
      if (units == 'seconds') {
        units <- 'secs'
      } else if (units == 'minutes') {
        units <- 'mins'
      }
      result <- paste(result, units)
    } else if (grep(' since ', units)) {
      parts <- strsplit(units, ' since ')[[1]]
      units <- parts[1]
      if (units == 'seconds') {
        units <- 'secs'
      } else if (units == 'minutes') {
        units <- 'mins'
      }
      result <- seq(as.POSIXct(parts[2]), length = max(result, na.rm = TRUE), by = units)
    }
  }, silent = TRUE)

  result
}

.NcDimReader <- function(file_name, file_object, selectors) {
  if (is.null(file_object)) {
    if (is.null(file_name)) {
      stop("Either 'file_name' or 'file_object' must be provided.")
    }
    file_object <- .NcOpener(file_name)
  } else {
    file_name <- file_object$filename
  }

  var_name <- NULL
  if ('var' %in% names(selectors)) {
    if (is.character(selectors[['var']])) {
      #if (length(selectors[['var']]) == 1) {
        var_name <- selectors[['var']][1]
      #} else {
    }
  }
  if (is.null(var_name) && (length(file_object$var) == 1)) {
    var_name <- names(file_object$var)[1]
  }
  if (!is.null(var_name)) {
    if (var_name %in% names(file_object$var)) {
      dims <- file_object$var[[var_name]]$size
      names(dims) <- sapply(file_object$var[[var_name]]$dim, '[[', 'name')
      dims
    } else {
      stop(paste0("Could not find the variable '", var_name, "' in the ",
                  "file ", file_name))
    }
  } else {
    stop(paste0("Could not find the dimensions of the file. Either ",
                "provide the parameters 'var'/'variable' or force the ", 
                "file to contain only one variable. ", file_name))
  }
}

.NcDataReader <- function(file_name, file_object, indices, vars) {#, file_dims) {
  if (is.null(variable_name)) {
  }
  ## TODO: check performance of wrapping opener functions
  ncvar_get(file_object, variable_name)
}

.NcCloser <- function(file_pointer) {
  result <- NULL

  try({
    result <- nc_close(file_pointer)
  })

  invisible(result)
}

.Load <- function(..., pattern_dims = NULL, 
                  inner_dims_across_files = NULL,   # ex list(ftime = 'chunk')
                  return_vars = NULL,               # ex list('ftime_var', 'lat_var', lon = c('lon', 'longitude'), ftime = 'extra_var'). 
                                                    # Allow only one dimension per variable? No .
                                                    # Here return the names without the _var.
                                                    # ex list(ftime_var = NULL, lat_var = NULL, lon = NULL, date = c('sdate', 'ftime'))
                                                    #                                                       date = list(var = 'time',
                                                    #                                                                   dims = c('ftime'),
                                                    #                                                                   for_each = c('sdate', 'ftime'))
                                                    # by default should take NULL.
                  selector_checker  = .CheckSelectors,
                  file_opener = .NcOpener,
                  file_var_reader = .NcVarReader,
                  file_dim_reader = .NcDimReader,
                  file_data_reader = .NcDataReader,
                  file_closer = .NcCloser) {
                  #, config_file = NULL
                  #, variable_dim = , 
                  #file_dimnames = , 
                  #) { This one is needed if user wants to get values for a variable that is not defined as a *_var.
  dim_params <- list(...)
  # Split *_var parameters
  var_params_ind <- grep('_var$', names(dim_params))
  var_params <- dim_params[var_params_ind]
  # Check all *_var are NULL or vectors of character strings, and 
  # that they all have a matching dimension param.
  i <- 1
  for (var_param in var_params) {
    if (!is.null(var_param)) {
      if (!is.character(var_param)) {
        stop("All '*_var' parameters must be vectors of character strings or NULL.")
      } else if (!any(grepl(paste0('^', strsplit(names(var_params)[i], 
                                                 '_var$')[[1]][1], '$'),
                            names(dim_params)))) {
        stop(paste0("All '*_var' parameters must be associated to a dimension parameter. Found parameter '", 
                    names(var_params)[i], "' but no parameter '", 
                    strsplit(names(var_params)[i], '_var$')[[1]][1], "'."))
      }
    }
    i <- i + 1
  }
  # Split *_checker parameters
  check_params_ind <- grep('_checker$', names(dim_params))
  check_params <- dim_params[check_params_ind]
  # Check all *_checker are NULL or functions, and that they all have 
  # a matching dimension param.
  i <- 1
  for (check_param in check_params) {
    if (!is.null(check_param)) {
      if (!is.function(check_param)) {
        stop("All '*_checker' parameters must be functions or NULL.")
      } else if (!any(grepl(paste0('^', strsplit(names(check_params)[i], 
                                                 '_checker$')[[1]][1], '$'),
                            names(dim_params)))) {
        stop(paste0("All '*_checker' parameters must be associated to a dimension parameter. Found parameter '", 
                    names(check_params)[i], "' but no parameter '", 
                    strsplit(names(check_params)[i], '_checker$')[[1]][1], "'."))
      }
    }
    i <- i + 1
  }
  # Leave alone the dimension parameters
  if (length(c(var_params_ind, check_params_ind)) > 0) {
    dim_params <- dim_params[-c(var_params_ind, check_params_ind)]
  }
  dim_names <- names(dim_params)
  # Make the keys of 'var_params' and 'check_params' to be the name of 
  # the corresponding dimension.
  names(var_params) <- gsub('_var$', '', names(var_params))
  names(check_params) <- gsub('_checker$', '', names(check_params))

  # Check pattern_dims
  if (is.null(pattern_dims)) {
    print(paste0("Parameter 'pattern_dims' not specified. Taking the first dimension, '", 
                 dim_names[1], "' as 'pattern_dims'."))
    pattern_dims <- dim_names[1]
  } else if (is.character(pattern_dims) && (length(pattern_dims) > 0)) {
    pattern_dims <- unique(pattern_dims)
  } else {
    stop("Parameter 'pattern_dims' must be a vector of character strings.")
  }
  if (any(names(var_params) %in% pattern_dims)) {
    stop("'*_var' parameters specified for pattern dimensions. Remove or fix them.")
  }

  ## TODO: varmin and varmax
  # Find the pattern dimension with the pattern specifications
  found_pattern_dim <- NULL
  for (pattern_dim in pattern_dims) {
    # Check all specifications in pattern_dim are valid
    dat <- datasets <- dim_params[[pattern_dim]]
    if (is.null(dat) || !(is.character(dat) && all(nchar(dat) > 0)) && !is.list(dat)) {
      stop(paste0("Parameter '", pattern_dim, 
                  "' must be a list of lists with pattern specifications or a vector of character strings."))
    }
    if (!is.null(check_params[[pattern_dim]])) {
      print(paste0("A checker for the selectors of '", pattern_dim, 
                   "' has been specified, but it is a pattern dimension and the checker will be ignored."))
    }
    if (any(sapply(dat, is.list))) {
      if (is.null(found_pattern_dim)) {
        found_pattern_dim <- pattern_dim
      } else {
        stop("Found more than one pattern dim with pattern specifications (list of lists). One and only one pattern dim must contain pattern specifications.")
      }
    }
  }
  if (is.null(found_pattern_dim)) {
    print(paste0("Taking the first pattern dim, '", pattern_dims[1], "', as dimension with pattern specifications (to be fetched in configuration file)."))
    found_pattern_dim <- pattern_dims[1]
  }
  dat <- datasets <- dim_params[[found_pattern_dim]]
  dat_info_names <- c('name', 'path')#, 'nc_var_name', 'suffix', 'var_min', 'var_max', 'dimnames')
  dat_to_fetch <- c()
  dat_names <- c()
  if (!is.list(dat)) {
    dat <- lapply(dat, function (x) list(name = x))
  }
  for (i in 1:length(dat)) {
    if (is.character(dat[[i]]) && length(dat[[i]]) == 1 && nchar(dat[[i]]) > 0) {
      dat[[i]] <- list(name = dat[[i]])
    } else if (!is.list(dat[[i]])) {
      stop(paste0("Parameter '", pattern_dim, 
                  "' is incorrect. It must be a list of lists or character strings."))
    }
    #if (!(all(names(dat[[i]]) %in% dat_info_names))) {
    #  stop("Error: parameter 'dat' is incorrect. There are unrecognized components in the information of some of the dateriments. Check 'dat' in ?Load for details.")
    #}
    if (!('name' %in% names(dat[[i]]))) {
      dat[[i]][['name']] <- paste0('dat', i)
      if (!('path' %in% names(dat[[i]]))) {
        stop(paste0("Parameter '", found_pattern_dim, 
                    "' is incorrect. A 'path' should be provided for each dataset if no 'name' is provided."))
      }
    } else if (!('path' %in% names(dat[[i]]))) {
      dat_to_fetch <- c(dat_to_fetch, i)
    }
    #if ('path' %in% names(dat[[i]])) {
    #  if (!('nc_var_name' %in% names(dat[[i]]))) {
    #    dat[[i]][['nc_var_name']] <- '$var_name$'
    #  }
    #  if (!('suffix' %in% names(dat[[i]]))) {
    #    dat[[i]][['suffix']] <- ''
    #  }
    #  if (!('var_min' %in% names(dat[[i]]))) {
    #    dat[[i]][['var_min']] <- ''
    #  }
    #  if (!('var_max' %in% names(dat[[i]]))) {
    #    dat[[i]][['var_max']] <- ''
    #  }
    #}
    dat_names <- c(dat_names, dat[[i]][['name']])
  }
  if ((length(dat_to_fetch) > 0) && (length(dat_to_fetch) < length(dat))) {
    print("'path' has been provided for some datasets. Any information in the configuration file related to these will be ignored.")
  }

  ## TODO: Check opener, reader, etc... functions are specified

  ## TODO: check config_file
  ## TODO: Get rid of this default
  #if (is.null(config_file)) {
  #  config_file <- system.file("config", "BSC.conf", package = "s2dverification")
  #} else if (!is.character(config_file) || !(nchar(config_file) > 0)) {
  #  stop("Error: parameter 'config_file' must be a character string with the path to a .Load() configuration file, if provided.")
  #}

  ## If not all data has been provided for any of the datasets in the 
  ## pattern_dim, configuration file is read.
  #if (length(dat_to_fetch) > 0) {
  #  print(paste0("Some 'path's not explicitly provided in '", pattern_dim, 
  #               "', so will now proceed to open the configuration file.")
  #  data_info <- ConfigFileOpen(config_file, silent, TRUE)
  #
  #  # Check that the var, exp and obs parameters are right and keep the entries
  #  # that match for each dataset.
  #  # Afterwards, the matching entries are applied sequentially (as specified 
  #  # in ?ConfigFileOpen) and the replace_values are applied to the result.
  #  # Finally a path pattern for each dataset is provided.
  #  matches <- ConfigApplyMatchingEntries(data_info, var, sapply(exp[exps_to_fetch], '[[', 'name'),
  #               sapply(obs[obs_to_fetch], '[[', 'name'), show_entries = FALSE, show_result = FALSE)
  #  # 'replace_values' is a named list that associates a variable name to an 
  #  # associated value. Initially it is filled with variables and values parsed
  #  # from the configuration file, but we can add or modify some values during
  #  # the execution to choose for example which start date we want to load.
  #  # When '.ConfigReplaceVariablesInString' is called, all the variable accesses 
  #  # ($VARIABLE_NAME$) that appear in the string given as parameter are 
  #  # replaced by the associated value in 'replace_values'.
  #  replace_values <- data_info$definitions
  #  if (!is.null(exp) && length(exps_to_fetch) > 0) {
  #    counter <- 1
  #    exp[exps_to_fetch] <- lapply(matches$exp_info,
  #      function (x) {
  #        x[names(exp[[exps_to_fetch[counter]]])] <- exp[[exps_to_fetch[counter]]]
  #        x[['path']] <- paste0(x[['main_path']], x[['file_path']])
  #        counter <<- counter + 1
  #        x
  #      })
  #  }
  #  if (!is.null(obs) && length(obs_to_fetch) > 0) {
  #    counter <- 1
  #    obs[obs_to_fetch] <- lapply(matches$obs_info,
  #      function (x) {
  #        x[names(obs[[obs_to_fetch[counter]]])] <- obs[[obs_to_fetch[counter]]]
  #        x[['path']] <- paste0(x[['main_path']], x[['file_path']])
  #        counter <<- counter + 1
  #        x
  #      })
  #  }
  #  if (!silent) {
  #    .message("All pairs (var, exp) and (var, obs) have matching entries.")
  #  }
  #} else {
  #  replace_values <- list(DEFAULT_NC_VAR_NAME = '$VAR_NAME$',
  #                         DEFAULT_VAR_MIN = '',
  #                         DEFAULT_VAR_MAX = '',
  #                         DEFAULT_SUFFIX = '',
  #                         DEFAULT_DIM_NAME_LONGITUDES = 'longitude',
  #                         DEFAULT_DIM_NAME_LATITUDES = 'latitude',
  #                         DEFAULT_DIM_NAME_MEMBERS = 'ensemble')
  #}
  ## We take the dimnames that haven't been explicitly specified from the 
  ## configuration file.
  ## If the configuration file wasn't opened, we take the default values from
  ## the dictionary 'replace_values'.
  #dimnames <- list(lon = ifelse(is.null(dimnames[["lon"]]),
  #                              replace_values[["DEFAULT_DIM_NAME_LONGITUDES"]],
  #                              dimnames[['lon']]),
  #                 lat = ifelse(is.null(dimnames[["lat"]]),
  #                              replace_values[["DEFAULT_DIM_NAME_LATITUDES"]],
  #                              dimnames[['lat']]),
  #                 member = ifelse(is.null(dimnames[["member"]]),
  #                                 replace_values[["DEFAULT_DIM_NAME_MEMBERS"]],
  #                                 dimnames[['member']]))
  #mandatory_defaults <- c('DEFAULT_EXP_MAIN_PATH', 'DEFAULT_EXP_FILE_PATH',
  #                        'DEFAULT_OBS_MAIN_PATH', 'DEFAULT_OBS_FILE_PATH',
  #                        'DEFAULT_NC_VAR_NAME', 'DEFAULT_SUFFIX',
  #                        'DEFAULT_VAR_MIN', 'DEFAULT_VAR_MAX',
  #                        'DEFAULT_DIM_NAME_LONGITUDES',
  #                        'DEFAULT_DIM_NAME_LATITUDES',
  #                        'DEFAULT_DIM_NAME_MEMBERS')
  #extra_vars_with_default_ind <- (1:length(replace_values))[grep('^DEFAULT_', names(replace_values))]
  #extra_vars_with_default_ind <- extra_vars_with_default_ind[
  #                                 grep(paste0(paste0('^', mandatory_defaults),
  #                                             collapse = '|'),
  #                                      names(replace_values)[extra_vars_with_default_ind],
  #                                      invert = TRUE)
  #                               ]
  #extra_vars_with_default <- gsub('^DEFAULT_', '',
  #                                names(replace_values)[extra_vars_with_default_ind])
  #if (!is.null(exp)) {
  #  exp <- lapply(exp, function (x) {
  #    if (!('dimnames' %in% names(x))) {
  #      x[['dimnames']] <- dimnames
  #    } else {
  #      dimnames2 <- dimnames
  #      dimnames2[names(x[['dimnames']])] <- x[['dimnames']]
  #      x[['dimnames']] <- dimnames2
  #    }
  #    i <- 1
  #    while (i <= length(extra_vars_with_default)) {
  #      if (!(extra_vars_with_default[i] %in% names(x))) {
  #        x[[extra_vars_with_default[i]]] <- replace_values[[extra_vars_with_default_ind[i]]]
  #      }
  #      i <- i + 1
  #    }
  #    x
  #  })
  #}
  #if (!is.null(obs)) {
  #  obs <- lapply(obs, function (x) {
  #    if (!('dimnames' %in% names(x))) {
  #      x[['dimnames']] <- dimnames
  #    } else {
  #      dimnames2 <- dimnames
  #      dimnames2[names(x[['dimnames']])] <- x[['dimnames']]
  #      x[['dimnames']] <- dimnames2
  #    }
  #    i <- 1
  #    while (i <= length(extra_vars_with_default)) {
  #      if (!(extra_vars_with_default[i] %in% names(x))) {
  #        x[[extra_vars_with_default[i]]] <- replace_values[[extra_vars_with_default_ind[i]]]
  #      }
  #      i <- i + 1
  #    }
  #    x
  #  })
  #}
  #single_dataset <- (length(obs) + length(exp) == 1)
  dim_params[[found_pattern_dim]] <- dat_names

  # Check that no unrecognized variables are present in the path patterns
  # and also that no file dimensions are requested to THREDDs catalogs.
  # And in the mean time, build all the work pieces and look for the 
  # first available file of each dataset.
  array_of_files_to_load <- NULL
  indices_of_first_files_with_data <- vector('list', length(dat))
  found_file_dims <- vector('list', length(dat))
  expected_inner_dims <- vector('list', length(dat))
print("A")
  for (i in 1:length(dat)) {
print("B")
    dat_selectors <- dim_params
    dat_selectors[[found_pattern_dim]] <- dat_selectors[[found_pattern_dim]][i]
    dim_vars <- paste0('$', dim_names, '$')
    file_dims <- dim_names[which(sapply(dim_vars, grepl, dat[[i]][['path']], fixed = TRUE))]
    found_file_dims[[i]] <- file_dims
    file_dims <- unique(c(pattern_dims, file_dims))
    expected_inner_dims[[i]] <- dim_names[which(!(dim_names %in% file_dims))]
    replace_values <- vector('list', length = length(file_dims))
    names(replace_values) <- file_dims
    dataset_has_files <- FALSE
    # Take the first selector for all possible file dimensions
    for (file_dim in file_dims) {
      if (file_dim %in% names(var_params)) {
        print(paste0("The '", file_dim, "_var' param will be ignored since '", 
                     file_dim, "' is a file dimension (for the dataset with pattern ", 
                     dat[[i]][['path']], ")."))
      }
      if (is.character(dat_selectors[[file_dim]]) && 
          !((length(dat_selectors[[file_dim]]) == 1) && 
            (dat_selectors[[file_dim]] %in% c('all', 'first', 'last')))) {
        dat_selectors[[file_dim]] <- selector_checker(selectors = dat_selectors[[file_dim]],
                                                      return_indices = FALSE)
        replace_values[[file_dim]] <- dat_selectors[[file_dim]][1]
      }
    }
print("C")
    #replace_values[[found_pattern_dim]] <- dat_selectors[[found_pattern_dim]][i]
    # Now we know which dimensions whose selectors are provided non-explicitly.
    undefined_file_dims <- file_dims[which(sapply(replace_values, is.null))]
    defined_file_dims <- file_dims[which(!(file_dims %in% undefined_file_dims))]
    # Find the possible values for the selectors that are provided as
    # indices. If the requested file is on server, impossible operation.
    if (length(grep("^http", dat[[i]][['path']])) > 0) {
      if (length(undefined_file_dims) > 0) {
        stop(paste0("All selectors for the file dimensions must be ",
                    "character strings if requesting data to a remote ",
                    "server. Found invalid selectors for the file dimensions ",
                    paste(paste0("'", undefined_file_dims, "'"), collapse = ', '), "."))
      }
      dataset_has_files <- TRUE
    } else {
      # Iterate over the known dimensions to find the first existing file.
      # The path to the first existing file will be used to find the 
      # values for the non explicitly defined selectors.
      first_file <- NULL
      first_file_selectors <- NULL
      if (length(undefined_file_dims) > 0) {
        replace_values[undefined_file_dims] <- '*'
      }
      ## TODO: What if length of defined_file_dims is 0? code might crash (in practice it worked for an example case)
      files_to_check <- sapply(dat_selectors[defined_file_dims], length)
      #files_to_check[found_pattern_dim] <- 1
      sub_array_of_files_to_check <- array(1:prod(files_to_check), dim = files_to_check)
      j <- 1
print("D")
      while (j <= prod(files_to_check) && is.null(first_file)) {
        selector_indices <- which(sub_array_of_files_to_check == j, arr.ind = TRUE)[1, ]
        selectors <- sapply(1:length(defined_file_dims), function (x) dat_selectors[defined_file_dims][[x]][selector_indices[x]])
        #selectors[[which(defined_file_dims == found_pattern_dim)]] <- dat_selectors[[found_pattern_dim]][i]
        replace_values[defined_file_dims] <- selectors
        file_path <- .ConfigReplaceVariablesInString(dat[[i]][['path']], replace_values)
        file_path <- Sys.glob(file_path)
        if (length(file_path) > 0) {
          first_file <- file_path[1]
          first_file_selectors <- selectors
        }
        j <- j + 1
      }
print("E")
      # Start looking for values for the non-explicitly defined selectors.
      if (is.null(first_file)) {
        print(paste0("Warning: no found files for the datset '", dat[[i]][['name']], 
                     "'. Check its path pattern: ", dat[[i]][['path']]))
      } else {
        dataset_has_files <- TRUE
        ## TODO: Improve message here if no variable found:
        if (length(undefined_file_dims) > 0) {
          # Looking for the first values, parsed from first_file.
          first_values <- vector('list', length = length(undefined_file_dims))
          names(first_values) <- undefined_file_dims
          found_values <- 0
          stop <- FALSE
          try_dim <- 1
          last_success <- 1
print("F")
          while ((found_values < length(undefined_file_dims)) && !stop) {
            u_file_dim <- undefined_file_dims[try_dim]
            if (is.null(first_values[[u_file_dim]])) {
              path_with_globs_and_tag <- .ConfigReplaceVariablesInString(dat[[i]][['path']], 
                                           replace_values[-which(file_dims == u_file_dim)],
                                           allow_undefined_key_vars = TRUE)
              found_value <- .FindTagValue(path_with_globs_and_tag, 
                                           first_file, u_file_dim)
              if (!is.null(found_value)) {
                found_values <- found_values + 1
                last_success <- try_dim
                first_values[[u_file_dim]] <- found_value
                replace_values[[u_file_dim]] <- found_value
              }
            }
            try_dim <- (try_dim %% length(undefined_file_dims)) + 1
            if (try_dim == last_success) {
              stop <- TRUE
            }
          }
print("G")
          if (found_values < length(undefined_file_dims)) {
            stop(paste0("Path pattern of dataset '", dat[[i]][['name']], 
                        "' is too complex. Could not automatically ",
                        "detect values for all non-explicitly defined ",
                        "indices. Check its pattern: ", dat[[i]][['path']]))
          }
          ## TODO: Replace ReplaceGlobExpressions by looped call to FindTagValue? As done above
          ##       Maybe it can solve more cases actually. I got warnings in ReplGlobExp with a typical
          ##       cmor case, requesting all members and chunks for fixed var and sdate. Not fixing
          ##       sdate raised 'too complex' error.
          # Replace shell globs in path pattern and keep the file_dims as tags
          dat[[i]][['path']] <- .ReplaceGlobExpressions(dat[[i]][['path']], first_file, replace_values, 
                                                        file_dims, dat[[i]][['name']], FALSE)
          # Now time to look for the available values for the non 
          # explicitly defined selectors.
print("H")
          for (u_file_dim in undefined_file_dims) {
            parsed_values <- c()
            replace_values[undefined_file_dims] <- first_values
            replace_values[[u_file_dim]] <- '*'
            path_with_globs <- .ConfigReplaceVariablesInString(dat[[i]][['path']], replace_values)
            found_files <- Sys.glob(path_with_globs)
            if (length(found_files) == 0) {
              stop("Error 3.")
            }
            for (found_file in found_files) {
              path_with_globs_and_tag <- .ConfigReplaceVariablesInString(dat[[i]][['path']],
                                           replace_values[-which(file_dims == u_file_dim)],
                                           allow_undefined_key_vars = TRUE)
              parsed_values <- c(parsed_values, 
                                 .FindTagValue(path_with_globs_and_tag, found_file, 
                                               u_file_dim))
            }
            dat_selectors[[u_file_dim]] <- selector_checker(selectors = dat_selectors[[u_file_dim]],
                                                            var = parsed_values,
                                                            return_indices = FALSE)
          }
print("I")
        } else {
          dat[[i]][['path']] <- .ReplaceGlobExpressions(dat[[i]][['path']], first_file, replace_values, 
                                                        defined_file_dims, dat[[i]][['name']], FALSE)
        }
      }
    }
    if (dataset_has_files) {
      # Now fetch for the first available file
      replace_values <- vector('list', length = length(file_dims))
      names(replace_values) <- file_dims
      files_to_load <- sapply(dat_selectors[file_dims], length)
      files_to_load[found_pattern_dim] <- 1
      sub_array_of_files_to_load <- array(1:prod(files_to_load), 
                                          dim = files_to_load,
                                          dimnames = file_dims)
      j <- 1
      while (j <= prod(files_to_load)) {
        selector_indices <- which(sub_array_of_files_to_load == j, arr.ind = TRUE)[1, ]
        names(selector_indices) <- file_dims
        selectors <- sapply(1:length(file_dims), function (x) dat_selectors[file_dims][[x]][selector_indices[x]])
        replace_values[file_dims] <- selectors
        file_path <- .ConfigReplaceVariablesInString(dat[[i]][['path']], replace_values)
        sub_array_of_files_to_load[j] <- file_path
        if (is.null(indices_of_first_files_with_data[[i]])) {
          if (!(length(grep("^http", file_path)) > 0)) {
            if (!file.exists(file_path)) {
              file_path <- NULL
            }
          }
          if (!is.null(file_path)) {
            test_file <- NULL
            ## TODO: suppress error messages
            test_file <- file_opener(file_path)
            if (!is.null(test_file)) {
              selector_indices[which(file_dims == found_pattern_dim)] <- i
              indices_of_first_files_with_data[[i]] <- selector_indices
              file_closer(test_file)
            }
          }
        }
        j <- j + 1
      }
      ## TODO: Put names in arrays and extend arrays as needed progressively
      if (is.null(array_of_files_to_load)) {
        array_of_files_to_load <- sub_array_of_files_to_load
      } else {
        if (!(identical(names(dim(array_of_files_to_load)), names(dim(sub_array_of_files_to_load))) &&
            identical(dim(array_of_files_to_load)[-which(names(dim(array_of_files_to_load)) == found_pattern_dim)], 
                      dim(sub_array_of_files_to_load)[-which(names(dim(sub_array_of_files_to_load)) == found_pattern_dim)]))) {
          dims1 <- dim(array_of_files_to_load)
          dims2 <- dim(sub_array_of_files_to_load)
          new_dims1 <- c()
          new_dims2 <- c()
          while (length(dims1) > 0) {
            if (names(dims1)[1] %in% names(dims2)) {
              pos <- which(names(dims2) == names(dims1)[1])
              dims_to_add <- rep(1, pos - 1)
              if (length(dims_to_add) > 0) {
                names(dims_to_add) <- names(dims2[1:(pos - 1)])
              }
              new_dims1 <- c(new_dims1, dims_to_add, dims1[1])
              new_dims2 <- c(new_dims2, dims2[1:pos])
              dims1 <- dims1[-1]
              dims2 <- dims2[-c(1:pos)]
            } else {
              new_dims1 <- c(new_dims1, dims1[1])
              new_dims2 <- c(new_dims2, 1)
              names(new_dims2)[length(new_dims2)] <- names(dims1)[1]
              dims1 <- dims1[-1]
            }
          }
          if (length(dims2) > 0) {
            dims_to_add <- rep(1, length(dims2))
            names(dims_to_add) <- names(dims2)
            new_dims1 <- c(new_dims1, dims_to_add)
            new_dims2 <- c(new_dims2, dims2)
          }
          dim(array_of_files_to_load) <- new_dims1
          dim(sub_array_of_files_to_load) <- new_dims2
          for (j in 1:length(dim(array_of_files_to_load))) {
            if (names(dim(array_of_files_to_load))[j] != found_pattern_dim) {
              if (dim(array_of_files_to_load)[j] != dim(sub_array_of_files_to_load)[j]) {
                if (which.max(c(dim(array_of_files_to_load)[j], dim(sub_array_of_files_to_load)[j])) == 1) {
                  na_array_dims <- dim(sub_array_of_files_to_load)
                  na_array_dims[j] <- dim(array_of_files_to_load)[j] - dim(sub_array_of_files_to_load)[j]
                  na_array <- array(dim = na_array_dims)
                  sub_array_of_files_to_load <- abind(sub_array_of_files_to_load, na_array, along = j)
                  names(dim(sub_array_of_files_to_load)) <- names(na_array_dims)
                } else {
                  na_array_dims <- dim(array_of_files_to_load)
                  na_array_dims[j] <- dim(sub_array_of_files_to_load)[j] - dim(array_of_files_to_load)[j]
                  na_array <- array(dim = na_array_dims)
                  array_of_files_to_load <- abind(array_of_files_to_load, na_array, along = j)
                  names(dim(array_of_files_to_load)) <- names(na_array_dims)
                }
              }
            }
          }
        }
        ## TODO: file_dims, and variables like that.. are still ok now? I don't think so
        array_of_files_to_load <- abind(array_of_files_to_load, 
                                        sub_array_of_files_to_load,
                                        along = which(names(dim(array_of_files_to_load)) == found_pattern_dim))
        names(dim(array_of_files_to_load)) <- names(dim(sub_array_of_files_to_load))
      }
    }
    dat[[i]][['selectors']] <- dat_selectors
  }
  if (is.null(indices_of_first_files_with_data)) {
    stop("No data files found for any of the specified datasets.")
  }
print("J")
  ## TODO: To be run in parallel (local multi-core)
  # Now time to work out the inner file dimensions
  for (i in 1:length(dat)) {
    indices <- indices_of_first_files_with_data[[i]]
    if (!is.null(indices)) {
      # First take all the variables requested.
      # For each of the dimensions across files
      # TODO: dimensions_across_files is now inverted
      # TODO: Avoid looking for the variable if the selectors are
      #       explicitly provided and the variable is not requested in 
      #       return_vars.
      picked_vars <- NULL
      picked_vars_chunk_lengths <- NULL
      picked_constant_vars <- NULL
      for (file_dim in names(inner_dims_across_files)) {
        vars_to_pick <- NULL
        # Find the names of the requested variables that go over the 
        # dimension in var_params.
        for (inner_dim in inner_dims_across_files[[file_dim]]) {
          detected_var <- grep(inner_dim, names(var_params))[1]
          if (length(detected_var) > 0) {
            vars_to_pick <- c(vars_to_pick, var_params[[detected_var]])
          }
        }
        if (length(vars_to_pick) > 0) {
          # And take their values
          #if (indices[which(found_file_dims[[i]] == file_dim)] != 1) {
          #  stop(paste0("Some variables requested for the dataset '", 
          #              dat[[i]][['name']], "' go along the '", file_dim, 
          #              "' dimension, which goes across multiple files. ",
          #              "However could not find all the files for this ",
          #              "dimension. Either generate these files or ",
          #              "request only for existing indices."))
          #}
          picked_vars <- vector('list', length = length(vars_to_pick))
          names(picked_vars) <- vars_to_pick
          picked_vars_chunk_lengths <- vector('list', length = length(vars_to_pick))
          names(picked_vars_chunk_lengths) <- vars_to_pick
          for (j in 1:(dim(array_of_files_to_load)[file_dim])) {
            indices[file_dim] <- j
            file_path <- do.call("[", c(list(array_of_files_to_load), as.list(indices)))
            file_object <- file_opener(file_path)
            if (is.null(file_object)) {
              if ((j == dim(array_of_files_to_load)[file_dim]) && 
                  (length(picked_vars[[1]]) == 0)) {
                stop(paste0("No files found for the dimension '", 
                            file_dim, "'. Can not read the variable(s) ", 
                            paste(paste0("'", vars_to_pick, "'") , 
                                  collapse = ', '), "."))          
              }
              for (var_to_pick in vars_to_pick) {
                if (is.null(picked_vars_chunk_lengths[[var_to_pick]])) {
                  picked_vars[[var_to_pick]] <- c(picked_vars[[var_to_pick]], NA)
                } else {
                  picked_vars[[var_to_pick]] <- c(picked_vars[[var_to_pick]], 
                                                  rep(NA, picked_vars_chunk_lengths[[var_to_pick]][1]))
                }
              }
            } else {
              for (var_to_pick in vars_to_pick) {
                found_values <- file_var_reader(file_path, file_object, var_to_pick)
                if (is.null(found_values)) {
                  stop(paste0("Can not read the variable '", var_to_pick, 
                              "' from the file ", file_path))
                }
                if (is.null(picked_vars_chunk_lengths[[var_to_pick]])) {
                  picked_vars[[var_to_pick]] <- rep(NA, length(picked_vars[[var_to_pick]]) * length(found_values))
                  picked_vars_chunk_lengths[[var_to_pick]] <- rep(length(found_values), dim(array_of_files_to_load)[file_dim])
                }
                picked_vars_chunk_lengths[[var_to_pick]][j] <- length(found_values)
                picked_vars[[var_to_pick]] <- c(picked_vars[[var_to_pick]], found_values)
              }
            }
          }
        }
      }
      # Take also the variables that are not going across files.
      if (is.null(picked_constant_vars)) {
        constant_vars_to_pick <- unlist(var_params, use.names = FALSE)
        constant_vars_to_pick <- constant_vars_to_pick[which(!(unlist(var_params) %in% names(picked_vars)))]
        if (length(constant_vars_to_pick) > 0) {
          file_path <- do.call("[", c(list(array_of_files_to_load), as.list(indices_of_first_files_with_data[[i]])))
          file_object <- file_opener(file_path)
          for (var_to_pick in constant_vars_to_pick) {
            found_values <- file_var_reader(file_path, file_object, var_to_pick)
            if (is.null(found_values)) {
              stop(paste0("Can not read the variable '", var_to_pick, 
                          "' from the file ", file_path))
            }
            picked_vars[[var_to_pick]] <- c(picked_vars[[var_to_pick]], found_values)
          }
          picked_constant_vars <- picked_vars[constant_vars_to_pick]
        } else {
          picked_constant_vars <- list()
        }
      } else if (length(picked_constant_vars) > 0) {
        picked_vars[constant_vars_to_pick] <- picked_constant_vars
      }
      # Once we have the variable values, we can work out the indices
      # for the implicitly defined selectors.
      for (inner_dim in expected_inner_dims[[i]]) {
        picked_var <- NULL
        picked_var_chunk_lengths <- NULL
        var_with_selectors <- var_params[[inner_dim]]
        if (!is.null(var_with_selectors)) {
          picked_var <- picked_vars[[var_with_selectors]]
          picked_var_chunk_lengths <- picked_vars_chunk_lengths[[var_with_selectors]]
        }
        if (is.null(picked_var) && !is.numeric(dat[[i]][['selectors']][[inner_dim]])) {
          stop(paste0("Can not translate the provided selectors for '", inner_dim, 
                      "' to numeric indices. Provide numeric indices or a '", 
                      inner_dim, "_var' in order to calculate the indices."))
        }
        found_indices <- selector_checker(dat[[i]][['selectors']][[inner_dim]], var = picked_var)
        if (inner_dim %in% unlist(inner_dims_across_files)) {
          ## TODO: translate dimension names to the actual names inside
          if (is.null(picked_var_chunk_lengths)) {
            file_path <- do.call("[", c(list(array_of_files_to_load), as.list(indices_of_first_files_with_data[[i]])))
            file_object <- file_opener(file_path)
            data_dims <- file_dim_reader(file_path, file_object, dat[[i]][['selectors']])
            if (inner_dim %in% names(data_dims)) {
              picked_var_chunk_lengths <- rep(data_dims[inner_dim], length(dat[[i]][['selectors']][[file_dim]]))
            } else {
              stop(paste0("Could not find the dimension '", inner_dim, "' in ",
                          "the file. Either change the dimension name in ",
                          "your request, adjust the parameter ",
                          "'dim_names_in_files' or fix the dimension name in ",
                          "the file. ", file_path))
            }
          }
          ## TODO: The [1] after the which is in case more than one file dimensions
          ##       have the inner_dim associated (in inner_dims_across_files), but 
          ##       this should be forbidden above.
          file_dim <- names(inner_dims_across_files)[which(sapply(inner_dims_across_files, function(x) inner_dim %in% x))[1]]
          dat[[i]][['selectors']][[inner_dim]] <- vector('list', 
            length = length(dat[[i]][['selectors']][[file_dim]]))
          names(dat[[i]][['selectors']][[inner_dim]]) <- dat[[i]][['selectors']][[file_dim]]
          taken_indices <- 0
          j <- 1
          dat[[i]][['selectors']][[inner_dim]] <- lapply(dat[[i]][['selectors']][[inner_dim]], function(x) {
            r <- found_indices[which((found_indices > taken_indices) & (found_indices <= taken_indices + picked_var_chunk_lengths[j]))]
            r <- r - taken_indices
            taken_indices <<- taken_indices + picked_var_chunk_lengths[j]
            j <<- j + 1
            r
          })
        } else {
          dat[[i]][['selectors']][[inner_dim]] <- found_indices
        }
      }
    }
  }

  #it should work even if inner_dim_var is not specified

  ## TODO: Close all opened files...

  # Crop the dimensions that go across files if needed
  for (i in 1:length(dat)) {
    #and then remove files from the array of files if needed (dont know)
  }

  # Take variables in return_vars.
  ## TODO: Move outisde the loops. It has to be recomputed since
  ##       return_vars may be defined over more than 1 dimension.
  ## Actually it should be sent to each work piece, otherwise
  ## it would require opening many files for only taking the 
  ## variable values.
  #tmp_rv <- return_vars
  #return_dim_vars <- which(unlist(return_vars) %in% names(var_params))
  #if (length(return_dim_vars) > 0) {
  #  tmp_rv <- tmp_rv[-return_dim_vars]
  #}
  #vars_to_pick <- c(vars_to_pick, unlist(tmp_rv[which(names(tmp_rv) == inner_dim)], use.names = FALSE))

  ## TODO: Crop the found variables? Only those requested in return_vars?

  list(dat, array_of_files_to_load, picked_vars)
}
