#' How to use:
#' 1. Allow to read multiple datasets with different selectors
#' 2. A list of list is used when different selectors are required, just like 'dat'.
#'    The items in each list must be 'name' and '<inner_dim_name>'. 'name' is the 
#'    dataset name, same as the one in 'dat'.
#' 3. Only indices() is accepted. (If for values(), use selectorchecker to transform?)
#' 4. Allow to have NA in the selectors. Those NAs will be removed when being fed in
#'    Start(), and be inserted back after Start().
#'
#'NOTE: 
#' 1. No reshape (merge_across_dim yes)
#' 2. The input check is not comprehensive
#'
#'@import abind multiApply
#'@export
multiStart <- function(..., # dim = indices/selectors, 
                  # dim_var = 'var', 
                  # dim_reorder = Sort/CircularSort, 
                  # dim_tolerance = number, 
                  # dim_depends = 'file_dim', 
                  # dim_across = 'file_dim', 
                  return_vars = NULL,
                  synonims = NULL,
                  file_opener = NcOpener,
                  file_var_reader = NcVarReader,
                  file_dim_reader = NcDimReader,
                  file_data_reader = NcDataReader,
                  file_closer = NcCloser,
                  transform = NULL,
                  transform_params = NULL,
                  transform_vars = NULL,
                  transform_extra_cells = 2,
                  apply_indices_after_transform = FALSE,
                  pattern_dims = NULL,
                  metadata_dims = NULL,
                  selector_checker = SelectorChecker,
                  merge_across_dims = FALSE,
                  merge_across_dims_narm = FALSE,
                  split_multiselected_dims = FALSE,
                  path_glob_permissive = FALSE,
                  largest_dims_length = FALSE,
                  retrieve = FALSE,
                  num_procs = 1,
                  ObjectBigmemory = NULL,
                  silent = FALSE, debug = FALSE) {

  dim_params_all <- list(...)
  inner_dims_across_files <- take_var_across(dim_params_all)
  dim_params <- rebuild_dim_params(dim_params_all, merge_across_dims,
                                   inner_dims_across_files)
  extra_params <- dim_params_all[!dim_params_all %in% dim_params]
  dim_params_list <- NULL
  dim_names <- names(dim_params)

#NOTE: The check can be removed. When entering Start(), the check will run as well.
#===============line 1020 (down)=========================
  # Check pattern_dims
  if (is.null(pattern_dims)) {
    .warning(paste0("Parameter 'pattern_dims' not specified. Taking the first dimension, '",
                    dim_names[1], "' as 'pattern_dims'."))
    pattern_dims <- dim_names[1]
  } else if (is.character(pattern_dims) && (length(pattern_dims) > 0)) {
    pattern_dims <- unique(pattern_dims)
  } else {
    stop("Parameter 'pattern_dims' must be a vector of character strings.")
  }

  # Find the pattern dimension with the pattern specifications
  found_pattern_dim <- NULL
  for (pattern_dim in pattern_dims) {
    # Check all specifications in pattern_dim are valid
    dat <- datasets <- dim_params[[pattern_dim]]
    if (is.list(dat) || any(sapply(dat, is.list))) {
      if (is.null(found_pattern_dim)) {
        found_pattern_dim <- pattern_dim
      } else {
        stop("Found more than one pattern dim with pattern specifications (list of lists). One and only one pattern dim must contain pattern specifications.")
      }
    }
  }
  if (is.null(found_pattern_dim)) {
    .warning(paste0("Could not find any pattern dim with explicit data set descriptions (in the form of list of lists). Taking the first pattern dim, '", pattern_dims[1], "', as dimension with pattern specifications."))
    found_pattern_dim <- pattern_dims[1]
  }
#===============line 1056 (up)=========================
  ndat <- length(dim_params[[found_pattern_dim]])  # 2
  dat_names <- sapply(dat, '[[', 'name')  #c(hadgem3, mpi_esm)
  dim_params_list <- vector('list', ndat)
  dim_params_list <- lapply(dim_params_list, '[<-', vector('list', length(dim_params)))
  names(dim_params_list) <- dat_names
  dim_names <- names(dim_params) #c(dat, var, sdate,...)
  
  # Create an array for NA record 
  na_pos_record <- vector('list', ndat)
  na_pos_record <- lapply(na_pos_record, '[<-', vector('list', length(dim_params)))
  names(na_pos_record) <- dat_names #c(hadgem3, mpi_esm)
  for (n_dat in 1:ndat) {
    na_pos_record[[n_dat]] <- as.list(rep(0, length(dim_params)))
    names(na_pos_record[[n_dat]]) <- dim_names
  }
  
  for (param_ind in 1:length(dim_params)) {
    dim_param <- dim_params[[param_ind]] # $ dat  :List of 2; $ var : chr "tas"
    if (is.list(dim_param) & all(lapply(lapply(dim_param, names), '[', 1) == 'name') &
        (names(dim_params)[param_ind] != found_pattern_dim)) {
      for (dat_ind in 1:ndat) {
        # If NA exists, take it out and record in an array
        if (any(is.na(dim_param[[dat_ind]][[2]]))) {
          dim_params_list[[dat_ind]][[param_ind]] <- dim_param[[dat_ind]][[2]][!is.na(dim_param[[dat_ind]][[2]])]
          attributes(dim_params_list[[dat_ind]][[param_ind]]) <- attributes(dim_param[[dat_ind]][[2]])
          na_pos_record[[dat_ind]][[param_ind]] <- which(is.na(dim_param[[dat_ind]][[2]]))
        } else {
          dim_params_list[[dat_ind]][[param_ind]] <- dim_param[[dat_ind]][[2]] # 2nd item is value, 1st item is dat name
        }
      }
    } else {
      if (names(dim_params)[param_ind] == found_pattern_dim) {
        for (dat_ind in 1:ndat) {
          dim_params_list[[dat_ind]][[param_ind]] <- list(dim_param[[dat_ind]])
        }
      } else {
        for (dat_ind in 1:ndat) {
          dim_params_list[[dat_ind]][[param_ind]] <- dim_param  # vector
        }
      }
    }
  }
  # Put dim name back
  for (dat_ind in 1:ndat) {
    names(dim_params_list[[dat_ind]]) <- dim_names
  }
  
  # Combine ... with other params
  dim_params_list <- lapply(dim_params_list, c,
             c(extra_params,
               list(return_vars = return_vars,
                    synonims = synonims,
                    file_opener = file_opener,
                    file_var_reader = file_var_reader,
                    file_dim_reader = file_dim_reader,
                    file_data_reader = file_data_reader,
                    file_closer = file_closer,
                    transform = transform,
                    transform_params = transform_params,
                    transform_vars = transform_vars,
                    transform_extra_cells = transform_extra_cells,
                    apply_indices_after_transform = apply_indices_after_transform,
                    pattern_dims = pattern_dims,
                    metadata_dims = metadata_dims,
                    selector_checker = selector_checker,
                    merge_across_dims = merge_across_dims,
                    merge_across_dims_narm = merge_across_dims_narm,
                    split_multiselected_dims = split_multiselected_dims,
                    path_glob_permissive = path_glob_permissive,
                    largest_dims_length = largest_dims_length,
                    retrieve = retrieve,
                    num_procs = num_procs,
                    ObjectBigmemory = ObjectBigmemory,
                    silent = silent,
                    debug = debug)))
  
  #===============Put in Start()=====================
  res <- vector('list', ndat)  # A list saving datasets separately
  ## matchcall needs to be modified and put in Start call. 
  ## Because do.call is used below, match.call() in Start() will generate incorrect
  ## call argument (it is correct only if Start(...) is used directly.)
  matchcall <- match.call()
  matchcall[[1]] <- as.name('Start')

  for (n_dat in 1:ndat) {

   # Revise matchcall for each Start call
    for (item in dim_names) { #only need to check '...'
      if (!identical(matchcall[[item]], dim_params_list[[n_dat]][[item]])) {
        matchcall[[item]] <- dim_params_list[[n_dat]][[item]]
      }
    }

    message("* Use Start() to get dataset '", dat_names[n_dat], "' now...")
    res[[n_dat]] <- do.call(Start, c(dim_params_list[[n_dat]], matchcall = quote(matchcall)))
    message("* Start() has successfully gotten dataset '", dat_names[n_dat], "'.")
  }
  names(res) <- dat_names #e.g., c(hadgem3, mpi_esm)
  
  #=====================================================
  
  if (retrieve) {
    have_na <- as.logical(lapply(lapply(na_pos_record, unlist), sum)) # a logical vector
  
    for (n_dat in 1:ndat) {
      if (have_na[n_dat]) {  #need to insert NA
  
        # Save metadata for later use 
        saved_metadata <- attributes(res[[n_dat]]) #attr(res[[n_dat]], 'Variables')
  
        no_na_dim_name <- names(unlist(lapply(lapply(na_pos_record[[n_dat]], '==', 0), which)))
        na_dim_name_ind <- which(!names(na_pos_record[[n_dat]]) %in% no_na_dim_name)
        na_dims <- na_pos_record[[n_dat]][na_dim_name_ind] # list
        correct_dim_order <- names(dim(res[[n_dat]])) # if reshape, here is diff from dim_names
        for (na_dim in 1:length(na_dims)) {
          na_dim_name <- names(na_dims)[na_dim]  # 'time'
          res[[n_dat]] <- Apply(res[[n_dat]],
                                target_dims = na_dim_name,
                                output_dims = na_dim_name,
                                fun = .insertNAdim,
                                na_pos = na_dims[[na_dim]] # a vector. b = c(2, 4)
                          )$output1
          res[[n_dat]] <- s2dv::Reorder(res[[n_dat]], correct_dim_order)
        }
        # Put metadata back
        ## res[[n_dat]] already has 'dim' attributes. The rest is lost
        lost_list_ind <- which(!names(saved_metadata) %in% names(attributes(res[[n_dat]])))
        attributes(res[[n_dat]]) <- c(attributes(res[[n_dat]]),
                                      saved_metadata[lost_list_ind])
      }
  
      # Add na_pos_record list to metadata
      attr(res[[n_dat]], 'na_pos_record')[[dat_names[n_dat]]] <- na_pos_record[[n_dat]]
    }
  
    # If > 1 dataset, combine datasets.
    if (length(res) > 1) {
      bind_along <- which(dim_names == found_pattern_dim)
      # Check if the dimensions are identical
      array_dims <- lapply(res, dim)
      names(array_dims) <- NULL
  
      if (!do.call(identical, array_dims)) {
        stop("The expected dimensions of datasets are not identical. Cannot combine them together.")
      }
      abind_input_list <- c(res, along = bind_along)
      combine_array <- do.call(abind::abind, abind_input_list)
      names(dim(combine_array)) <- dim_names
  
      if (!silent) {
        .message("Final dimension sizes after combination:")
        final_dims_combined <- dim(combine_array)
        longest_dim_len <- max(sapply(names(final_dims_combined), nchar))
        longest_size_len <- max(sapply(paste0(final_dims_combined, ''), nchar))
        sapply(names(final_dims_combined), 
          function(x) {
            message(paste0("*   ", paste(rep(' ', longest_dim_len - nchar(x)), collapse = ''), 
                           x, ": ", paste(rep(' ', longest_size_len - nchar(paste0(final_dims_combined[x], ''))), collapse = ''), 
                           final_dims_combined[x]))
        })
        bytes <- prod(c(final_dims_combined, 8))

        .message(paste("Total size of combined data:"))
        dim_sizes <- paste(final_dims_combined, collapse = ' x ')
        .message(paste(dim_sizes, " x 8 bytes =", 
                       format(structure(bytes, class = "object_size"), units = "auto")), 
                 indent = 2)
      }

      #---- restore metadata ----
      ## combine_array already has 'dim' attributes. The rest is lost
      lost_list_ind <- which(!names(attributes(res[[1]])) %in% names(attributes(combine_array)))
      attributes(combine_array) <- c(attributes(combine_array),
                                     attributes(res[[1]])[lost_list_ind])
  
      #---- Combine metadata which is not common ----
      #NOTE: $dim, $PatternDim, $class are common attributes.
      attr_list <- vector('list', length = ndat)
      for (n_dat in 1:ndat) {
        attr_list[[n_dat]] <- attributes(res[[n_dat]])
      }
  
      # For the value is array
      ## $Files
      sublist <- lapply(attr_list, '[[', 'Files') # list(dat1 = files1, dat2 = files2)
      bind_along <- which(names(dim(attr(res[[1]], 'Files'))) == found_pattern_dim) # 1
      attr(combine_array, 'Files') <- do.call(abind::abind, c(sublist, along = bind_along))
      names(dim(attr(combine_array, 'Files'))) <- names(dim(attr_list[[1]]$Files))
  
      ## $NotFoundFiles
      sublist <- lapply(attr_list, '[[', 'NotFoundFiles')
      if (!is.null(unlist(sublist))) { #Some of the dats have NotFoundFiles
        attr(combine_array, 'NotFoundFiles') <- do.call(abind::abind, c(sublist, along = bind_along)) #bind_along is from $Files above
        names(attr(combine_array, 'NotFoundFiles')) <- names(dim(attr_list[[1]]$Files))
      }
      # For the value is list
      ## $FileSelectors
      attr(combine_array, 'FileSelectors') <- lapply(lapply(attr_list, '[', 'FileSelectors'),
                                                     '[[', 1)
      names(attr(combine_array, 'FileSelectors')) <- dat_names
      ## $ObjectBigmemory
      attr(combine_array, 'ObjectBigmemory') <- lapply(lapply(attr_list, '[', 'ObjectBigmemory'),
                                                       '[[', 1)
      names(attr(combine_array, 'ObjectBigmemory')) <- dat_names
      ## $na_pos_record
      attr(combine_array, 'na_pos_record') <- lapply(lapply(attr_list, '[[', 'na_pos_record'),
                                                     '[[', 1)
      names(attr(combine_array, 'na_pos_record')) <- dat_names
  
      for (n_dat in 2:ndat) {
        # $Variables
        ## For $common, use the 1st data. Therefore, just add 2nd data $dat2 into 
        ## the metadata of 1st data
        if (!all(names(attr(res[[n_dat]], 'Variables')) %in% 'common')) {
          dat_list_name <- names(attr(res[[n_dat]], 'Variables'))[which(!names(attr(res[[n_dat]], 'Variables')) %in% 'common')] #'dat2'
          attr(combine_array, 'Variables')[[dat_list_name]] <- attr(res[[n_dat]], 'Variables')[[dat_list_name]]
        }
      }
  
      return(combine_array)
  
    } else { # only 1 dataset
      return(res[[1]])
    }
  
  } else { # retrieve = FALSE
    for (n_dat in 1:ndat) {
      # Add na_pos_record list to metadata
      attr(res[[n_dat]], 'na_pos_record')[[dat_names[n_dat]]] <- na_pos_record[[n_dat]]
  
      # Correct dimensions if NA exists
      if (sum(unlist(na_pos_record[[n_dat]])) != 0) {
        have_na_dims <- which(unlist(lapply(lapply(na_pos_record[[n_dat]], '!=', 0), any)))
        for (have_na_dim in 1:length(have_na_dims)) {
          attr(res[[n_dat]], 'Dimensions')[have_na_dims[have_na_dim]] <- attr(res[[n_dat]], 'Dimensions')[have_na_dims[have_na_dim]] + length(na_pos_record[[n_dat]][[have_na_dims[have_na_dim]]])
        }
      }
    }
    # Check if the datasets can be combined
    dim_list <- lapply(1:ndat, function(x) { attr(res[[x]], 'Dimensions') } )
    found_pattern_dim_ind <-lapply(lapply(
                                     lapply(dim_list, names), 
                                     '==', found_pattern_dim), which)
    dim_list_without_dat <- lapply(1:ndat, function(x) {
                                     dim_list[[x]] <- dim_list[[x]][-found_pattern_dim_ind[[x]]]})
    if (!do.call(identical, dim_list_without_dat)) {
      .warning(paste0("The datasets are detected to have inconsistent dimensions. ",
                      "They may not be able to combine for the later operation."))
    } else {
      if (!silent) {
        dim_list_dat <- lapply(1:ndat, function(x) {
                               dim_list[[x]] <- dim_list[[x]][found_pattern_dim_ind[[x]]]})
        combined_dat_length <- do.call(sum, dim_list_dat)
        final_dims_combined <- attr(res[[1]], 'Dimensions')
        final_dims_combined[found_pattern_dim] <- do.call(sum, dim_list_dat)
        .message("Detected final dimension sizes after combination:")
        longest_dim_len <- max(sapply(names(final_dims_combined), nchar))
        longest_size_len <- max(sapply(paste0(final_dims_combined, ''), nchar))
        sapply(names(final_dims_combined),
          function(x) {
            message(paste0("*   ", paste(rep(' ', longest_dim_len - nchar(x)), collapse = ''), 
                           x, ": ", paste(rep(' ', longest_size_len - nchar(paste0(final_dims_combined[x], ''))), collapse = ''),
                           final_dims_combined[x]))
        })
        bytes <- prod(c(final_dims_combined, 8))
   
        .message(paste("Total size of combined data:"))
        dim_sizes <- paste(final_dims_combined, collapse = ' x ')
        .message(paste(dim_sizes, " x 8 bytes =",
                       format(structure(bytes, class = "object_size"), units = "auto")),
                 indent = 2)
      }
    }
  
    return(res)
  }

}

